diff --git a/tests/sbr-operator/README.md b/tests/sbr-operator/README.md index 0dfedd6c41..0788a0540e 100644 --- a/tests/sbr-operator/README.md +++ b/tests/sbr-operator/README.md @@ -135,3 +135,17 @@ independently using a short-lived privileged hostPID pod per node. - **Environment**: Connected or disconnected - **Standalone**: `ginkgo --label-filter="sbr" --focus="Verify watchdog device" ./tests/sbr-operator/...` - **Pass criteria**: All hardware watchdog devices are character devices; nodes without hardware watchdog have softdog.ko present in the kernel module tree + +### 9. Verify SBR Must-Gather Collects Diagnostic Data ([OCP-88733](https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-88733)) + +Validates that `oc adm must-gather` with the RHWA image collects +SBR-related diagnostic data: node manifests, CRD definitions, and +MachineHealthCheck resources. + +- **Operators**: SBR v0.3.0, NHC (for must-gather image resolution) +- **Cluster**: Any topology +- **Storage**: None +- **Environment**: Connected by default; disconnected clusters can set + `MUST_GATHER_IMAGE` to an accessible mirrored image +- **Standalone**: `ginkgo --label-filter="sbr" --focus="must-gather" ./tests/sbr-operator/...` +- **Pass criteria**: SBR deployment is Ready; must-gather completes successfully; output contains node YAMLs for all cluster nodes, all 3 SBR CRD definition files, and MachineHealthCheck data diff --git a/tests/sbr-operator/internal/sbrparams/const.go b/tests/sbr-operator/internal/sbrparams/const.go index 3c9a0b2647..96c7c77d89 100644 --- a/tests/sbr-operator/internal/sbrparams/const.go +++ b/tests/sbr-operator/internal/sbrparams/const.go @@ -97,8 +97,20 @@ const ( SBRAgentDaemonSetPrefix = "sbr-agent-" // SBRCReadyTimeout is the time allowed for the SBRC's agent DaemonSet to have all scheduled - // pods reach Ready before a functional test begins. - SBRCReadyTimeout = 7 * time.Minute + // pods reach Ready before a functional test begins. The first SBRC created in a suite pays a + // one-time cold-start cost (first CephFS PVC bind plus SBD device-init Job on a freshly + // provisioned ODF cluster); later SBRCs reuse the warmed CSI path and become ready in ~90s. + // The ceiling is sized for that worst-case cold start - a ready DaemonSet returns immediately, + // so healthy runs never wait this long. + SBRCReadyTimeout = 10 * time.Minute + + // SBRCReadyDiagMaxEvents caps how many recent Warning events are included in the on-timeout + // readiness diagnostics dumped by waitForSBRCReady, to keep the failure message readable. + SBRCReadyDiagMaxEvents = 15 + + // SBRCReadyDiagTimeout bounds the API calls made while collecting on-timeout readiness + // diagnostics, so a slow or wedged apiserver cannot hang the already-failed test. + SBRCReadyDiagTimeout = 30 * time.Second // WatchdogProbeLogTimeout is the deadline for reading a completed watchdog probe pod's logs // via the in-cluster API. @@ -251,6 +263,21 @@ const ( // AgentMetricsPort is the port on which SBR agent pods expose custom Prometheus metrics. // Port 8080 is controller-runtime's built-in metrics; port 8082 is the SBR agent's own metrics. AgentMetricsPort = "8082" + + // MustGatherOCTimeout is the --timeout flag passed to oc adm must-gather so it cleans up gracefully. + MustGatherOCTimeout = 14 * time.Minute + + // MustGatherContextTimeout is the outer Go context timeout, strictly greater than MustGatherOCTimeout. + MustGatherContextTimeout = 16 * time.Minute + + // MustGatherImageRepo is the registry path for the RHWA must-gather image (without tag). + MustGatherImageRepo = "registry.redhat.io/workload-availability/node-healthcheck-must-gather-rhel9" + + // MustGatherDefaultTag is the fallback image tag when dynamic resolution fails. + MustGatherDefaultTag = "v0.9.0" + + // MustGatherCleanupTimeout is the deadline for best-effort cleanup of leftover must-gather namespaces. + MustGatherCleanupTimeout = 2 * time.Minute ) // AgentExpectedMetricNames lists the Prometheus metric names that must be present in the agent output. diff --git a/tests/sbr-operator/internal/sbrparams/sbrvars.go b/tests/sbr-operator/internal/sbrparams/sbrvars.go index 6fdc8f2fde..152318a62e 100644 --- a/tests/sbr-operator/internal/sbrparams/sbrvars.go +++ b/tests/sbr-operator/internal/sbrparams/sbrvars.go @@ -35,6 +35,13 @@ var ( "StorageBasedRemediationTemplate", } + // SBRCRDNames lists the full CRD names (plural.group) for all SBR custom resource definitions. + SBRCRDNames = []string{ + "storagebasedremediationconfigs." + CRDGroup, + "storagebasedremediations." + CRDGroup, + "storagebasedremediationtemplates." + CRDGroup, + } + // RequiredAnnotations defines the required annotations and expected values for SBR CSV. RequiredAnnotations = map[string]string{ "features.operators.openshift.io/tls-profiles": "false", diff --git a/tests/sbr-operator/tests/must_gather.go b/tests/sbr-operator/tests/must_gather.go new file mode 100644 index 0000000000..0c6972feb2 --- /dev/null +++ b/tests/sbr-operator/tests/must_gather.go @@ -0,0 +1,272 @@ +package tests + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + oplmV1alpha1 "github.com/rh-ecosystem-edge/eco-goinfra/pkg/schemes/olm/operators/v1alpha1" + + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/deployment" + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/olm" + "github.com/rh-ecosystem-edge/eco-goinfra/pkg/reportxml" + + "github.com/medik8s/system-tests/tests/internal/labels" + . "github.com/medik8s/system-tests/tests/internal/medik8sinittools" + "github.com/medik8s/system-tests/tests/internal/medik8sparams" + "github.com/medik8s/system-tests/tests/sbr-operator/internal/sbrparams" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe( + "SBR Must-Gather Diagnostics", + Serial, + Ordered, + Label(labels.OperatorSBR), func() { + It("Verify SBR must-gather collects diagnostic data", + reportxml.ID("88733"), + Label( + labels.DisruptionNonDestructive, + labels.TierAcceptance, + labels.PlatformAny, + labels.ComponentController, + labels.FrequencyWeekly, + ), func() { + By("Verifying SBR deployment is Ready") + + sbrDeployment, err := deployment.Pull( + APIClient, sbrparams.OperatorDeploymentName, medik8sparams.OperatorNs) + Expect(err).ToNot(HaveOccurred(), "Failed to get SBR deployment") + Expect(sbrDeployment.IsReady(medik8sparams.DefaultTimeout)).To(BeTrue(), + "SBR deployment is not Ready") + + By("Resolving the RHWA must-gather image") + + mustGatherImage := resolveMustGatherImage() + Expect(mustGatherImage).To(ContainSubstring(":"), + "must-gather image %q should contain a tag separator", mustGatherImage) + GinkgoWriter.Printf("Using must-gather image: %s\n", mustGatherImage) + + By("Creating artifact directory for must-gather output") + + destDir := createMustGatherDestDir() + + By("Capturing cluster state before must-gather for validation") + + listCtx, listCancel := context.WithTimeout(context.Background(), medik8sparams.DefaultTimeout) + defer listCancel() + + nodeList, err := APIClient.CoreV1Interface.Nodes().List(listCtx, metav1.ListOptions{}) + Expect(err).ToNot(HaveOccurred(), "Failed to list cluster nodes") + Expect(nodeList.Items).ToNot(BeEmpty(), "Cluster has no nodes") + + var nodeNames []string + for i := range nodeList.Items { + nodeNames = append(nodeNames, nodeList.Items[i].Name) + } + + By("Running oc adm must-gather") + + testStartTime := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), sbrparams.MustGatherContextTimeout) + defer cancel() + + DeferCleanup(func() { + cleanupMustGatherNamespaces(context.Background(), testStartTime) + }) + + runMustGather(ctx, mustGatherImage, destDir) + + By("Collecting gathered file paths") + + collectedFiles, walkErr := collectRelativePaths(destDir) + Expect(walkErr).ToNot(HaveOccurred(), "Failed to walk must-gather output directory") + Expect(collectedFiles).ToNot(BeEmpty(), "No files collected by must-gather") + + if writeErr := os.WriteFile(filepath.Join(destDir, "collected-paths.txt"), + []byte(strings.Join(collectedFiles, "\n")+"\n"), 0o644); writeErr != nil { + GinkgoWriter.Printf("Warning: failed to write collected-paths.txt: %v\n", writeErr) + } + + By("Validating node YAMLs for all cluster nodes") + + for _, nodeName := range nodeNames { + Expect(hasMatchingFile(collectedFiles, "/nodes/"+nodeName+".yaml")).To(BeTrue(), + "must-gather should contain YAML for node %s", nodeName) + } + + By("Validating SBR CRD definitions are present") + + for _, crdName := range sbrparams.SBRCRDNames { + Expect(hasMatchingFile(collectedFiles, crdName+".yaml")).To(BeTrue(), + "must-gather should contain CRD definition for %s", crdName) + } + + By("Validating MachineHealthCheck data is collected") + + Expect(hasMatchingFile(collectedFiles, "machinehealthchecks")).To(BeTrue(), + "must-gather should contain MachineHealthCheck data") + }) + }) + +func resolveMustGatherImage() string { + if envImg := os.Getenv("MUST_GATHER_IMAGE"); envImg != "" { + GinkgoWriter.Printf("must-gather image resolved from MUST_GATHER_IMAGE env var\n") + + return envImg + } + + nhcCSVs, err := olm.ListClusterServiceVersionWithNamePattern( + APIClient, "node-healthcheck", medik8sparams.OperatorNs) + if err != nil { + GinkgoWriter.Printf("Warning: failed to list NHC CSVs for must-gather image resolution: %v\n", err) + } else { + for _, csv := range nhcCSVs { + phase, phaseErr := csv.GetPhase() + if phaseErr != nil || phase != oplmV1alpha1.CSVPhaseSucceeded { + continue + } + + version := csv.Object.Spec.Version.String() + if version != "" { + image := fmt.Sprintf("%s:v%s", sbrparams.MustGatherImageRepo, version) + GinkgoWriter.Printf("must-gather image resolved from NHC CSV version: %s\n", image) + + return image + } + } + } + + GinkgoWriter.Printf("WARNING: must-gather image using hardcoded fallback tag %s\n", + sbrparams.MustGatherDefaultTag) + + return fmt.Sprintf("%s:%s", sbrparams.MustGatherImageRepo, sbrparams.MustGatherDefaultTag) +} + +func createMustGatherDestDir() string { + base := os.Getenv("ARTIFACT_DIR") + if base == "" { + base = GinkgoT().TempDir() + } + + dir, mkdirErr := os.MkdirTemp(base, "sbr-must-gather-") + ExpectWithOffset(1, mkdirErr).ToNot(HaveOccurred(), "Failed to create must-gather output directory") + + return dir +} + +func runMustGather(ctx context.Context, image, destDir string) { + ocTimeout := fmt.Sprintf("%ds", int(sbrparams.MustGatherOCTimeout.Seconds())) + + cmd := exec.CommandContext(ctx, "oc", "adm", "must-gather", + "--image="+image, + "--dest-dir="+destDir, + "--timeout="+ocTimeout, + ) + + env := os.Environ() + if os.Getenv("HOME") == "" { + env = append(env, "HOME=/tmp") + } + + cmd.Env = env + + output, err := cmd.CombinedOutput() + + logFile := filepath.Join(destDir, "oc-adm-must-gather.log") + + if writeErr := os.WriteFile(logFile, output, 0o644); writeErr != nil { + GinkgoWriter.Printf("Warning: failed to write must-gather log to %s: %v\n", logFile, writeErr) + } + + GinkgoWriter.Printf("must-gather output saved to %s\n", logFile) + + if ctx.Err() != nil { + Fail(fmt.Sprintf("must-gather timed out after %s:\n%s", + sbrparams.MustGatherContextTimeout, string(output))) + } + + ExpectWithOffset(1, err).ToNot(HaveOccurred(), "must-gather failed:\n%s", string(output)) +} + +func collectRelativePaths(root string) ([]string, error) { + var paths []string + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + + paths = append(paths, filepath.ToSlash(rel)) + + return nil + }) + + return paths, err +} + +func hasMatchingFile(files []string, pattern string) bool { + lowerPattern := strings.ToLower(pattern) + for _, f := range files { + if strings.Contains(strings.ToLower(f), lowerPattern) { + return true + } + } + + return false +} + +func cleanupMustGatherNamespaces(ctx context.Context, testStartTime time.Time) { + cleanupCtx, cleanupCancel := context.WithTimeout(ctx, sbrparams.MustGatherCleanupTimeout) + defer cleanupCancel() + + out, err := exec.CommandContext(cleanupCtx, "oc", "get", "ns", + "-l", "openshift.io/run-level", + "-o", "jsonpath={range .items[*]}{.metadata.name} {.metadata.creationTimestamp}{\"\\n\"}{end}", + ).CombinedOutput() + if err != nil { + GinkgoWriter.Printf("Warning: failed to list namespaces for must-gather cleanup: %v\n", err) + + return + } + + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + fields := strings.Fields(line) + if len(fields) < 1 || !strings.HasPrefix(fields[0], "openshift-must-gather-") { + continue + } + + namespaceName := fields[0] + + if len(fields) >= 2 { + createdAt, parseErr := time.Parse(time.RFC3339, fields[1]) + if parseErr == nil && createdAt.Before(testStartTime) { + continue + } + } + + GinkgoWriter.Printf("Cleaning up leftover must-gather namespace: %s\n", namespaceName) + + cleanupOut, cleanupErr := exec.CommandContext(cleanupCtx, "oc", "delete", "ns", namespaceName, + "--ignore-not-found", "--wait=false").CombinedOutput() + if cleanupErr != nil { + GinkgoWriter.Printf("Warning: failed to delete namespace %s: %v\n%s\n", + namespaceName, cleanupErr, string(cleanupOut)) + } + } +} diff --git a/tests/sbr-operator/tests/sbr.go b/tests/sbr-operator/tests/sbr.go index 338e3c62fe..18c4726823 100644 --- a/tests/sbr-operator/tests/sbr.go +++ b/tests/sbr-operator/tests/sbr.go @@ -3,7 +3,9 @@ package tests import ( "context" "fmt" + "sort" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -556,7 +558,247 @@ func waitForSBRCReady(sbrcName string) { return nil }, sbrparams.SBRCReadyTimeout, sbrparams.DefaultPollInterval).Should(Succeed(), - "SBRC %q agent DaemonSet must have all pods ready before functional tests begin", sbrcName) + // Gomega calls this lazily, only on timeout, so the diagnostics land directly in the + // Ginkgo [FAILED] block (and therefore in the Prow log) without any happy-path cost. + func() string { + return fmt.Sprintf( + "SBRC %q agent DaemonSet must have all pods ready before functional tests begin\n%s", + sbrcName, sbrcReadinessDiagnostics(sbrcName, dsName)) + }) +} + +// sbrcReadinessDiagnostics gathers the cluster state that explains why an SBRC agent DaemonSet +// has not become ready. The usual bottleneck is the first SBRC's cold start on a freshly +// provisioned ODF/CephFS cluster: the SBD device PVC bind and the SBD device-init Job can lag, +// leaving agent pods Pending. It lists (by dynamic discovery, never by assumed operator-internal +// names) the DaemonSet status, the agent pods' container/scheduling state, every PVC and Job in +// the operator namespace, and the most recent Warning events. Returned as a string so it can be a +// lazily-evaluated Gomega failure description; every lookup is best-effort so one failure does not +// mask the rest. +func sbrcReadinessDiagnostics(sbrcName, dsName string) string { + // Bound the diagnostic API calls so a slow or wedged apiserver cannot hang the already-failed + // test while the failure description is being built. + ctx, cancel := context.WithTimeout(context.Background(), sbrparams.SBRCReadyDiagTimeout) + defer cancel() + + namespace := medik8sparams.OperatorNs + + var report strings.Builder + + fmt.Fprintf(&report, "=== SBRC %q readiness diagnostics (namespace %s) ===\n", sbrcName, namespace) + + agentDS, dsErr := APIClient.DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) + if dsErr != nil { + fmt.Fprintf(&report, "DaemonSet %s: GET failed: %v\n", dsName, dsErr) + } else { + dsStatus := agentDS.Status + fmt.Fprintf(&report, + "DaemonSet %s: desired=%d current=%d ready=%d available=%d updated=%d misscheduled=%d\n", + dsName, dsStatus.DesiredNumberScheduled, dsStatus.CurrentNumberScheduled, dsStatus.NumberReady, + dsStatus.NumberAvailable, dsStatus.UpdatedNumberScheduled, dsStatus.NumberMisscheduled) + } + + // Prefer the DaemonSet's own label selector (discovered, not assumed) to find its agent pods; + // fall back to the "-" name prefix when the DaemonSet itself could not be fetched. + podSelector := "" + + if dsErr == nil && agentDS.Spec.Selector != nil { + if selector, selErr := metav1.LabelSelectorAsSelector(agentDS.Spec.Selector); selErr == nil { + podSelector = selector.String() + } + } + + report.WriteString(agentPodDiagnostics(ctx, namespace, dsName, podSelector)) + report.WriteString(pvcDiagnostics(ctx, namespace)) + report.WriteString(jobDiagnostics(ctx, namespace)) + report.WriteString(recentWarningEvents(ctx, namespace)) + + return report.String() +} + +// agentPodDiagnostics reports the SBRC agent pods' scheduling/container state and the PVC each one +// mounts. Pods are selected by the DaemonSet's label selector when known, otherwise by the +// "-" name prefix (DaemonSet pods are named "-"). +func agentPodDiagnostics(ctx context.Context, namespace, dsName, podSelector string) string { + listOptions := metav1.ListOptions{} + if podSelector != "" { + listOptions.LabelSelector = podSelector + } + + podList, err := APIClient.Pods(namespace).List(ctx, listOptions) + if err != nil { + return fmt.Sprintf("Pods: LIST failed: %v\n", err) + } + + var report strings.Builder + + found := false + + for idx := range podList.Items { + agentPod := &podList.Items[idx] + if podSelector == "" && !strings.HasPrefix(agentPod.Name, dsName+"-") { + continue + } + + found = true + + fmt.Fprintf(&report, "Pod %s: phase=%s node=%q\n", + agentPod.Name, agentPod.Status.Phase, agentPod.Spec.NodeName) + + for _, cond := range agentPod.Status.Conditions { + if cond.Status != corev1.ConditionTrue { + fmt.Fprintf(&report, " condition %s=%s reason=%s msg=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + for _, vol := range agentPod.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + fmt.Fprintf(&report, " volume %s -> PVC %s\n", vol.Name, vol.PersistentVolumeClaim.ClaimName) + } + } + + for _, initStatus := range agentPod.Status.InitContainerStatuses { + report.WriteString(" init " + containerStateSummary(initStatus)) + } + + for _, ctrStatus := range agentPod.Status.ContainerStatuses { + report.WriteString(" " + containerStateSummary(ctrStatus)) + } + } + + if !found { + fmt.Fprintf(&report, "Pods: none found for DaemonSet %s\n", dsName) + } + + return report.String() +} + +// pvcDiagnostics reports every PVC in the namespace. The first SBRC's SBD device PVC bind is the +// usual cold-start bottleneck, and a Pending PVC keeps the agent pods from starting. +func pvcDiagnostics(ctx context.Context, namespace string) string { + pvcList, err := APIClient.PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Sprintf("PVCs: LIST failed: %v\n", err) + } + + if len(pvcList.Items) == 0 { + return "PVCs: none in namespace\n" + } + + var report strings.Builder + + for idx := range pvcList.Items { + pvc := &pvcList.Items[idx] + + storageClass := "" + if pvc.Spec.StorageClassName != nil { + storageClass = *pvc.Spec.StorageClassName + } + + fmt.Fprintf(&report, "PVC %s: phase=%s storageClass=%q\n", pvc.Name, pvc.Status.Phase, storageClass) + } + + return report.String() +} + +// jobDiagnostics reports every Job in the namespace. The SBD device-init Job runs once per SBRC, +// and a stuck or failed Job leaves the agent pods not ready. +func jobDiagnostics(ctx context.Context, namespace string) string { + jobList, err := APIClient.K8sClient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Sprintf("Jobs: LIST failed: %v\n", err) + } + + if len(jobList.Items) == 0 { + return "Jobs: none in namespace\n" + } + + var report strings.Builder + + for idx := range jobList.Items { + job := &jobList.Items[idx] + fmt.Fprintf(&report, "Job %s: active=%d succeeded=%d failed=%d\n", + job.Name, job.Status.Active, job.Status.Succeeded, job.Status.Failed) + } + + return report.String() +} + +// containerStateSummary renders a single container's current state on one line for diagnostics. +func containerStateSummary(containerStatus corev1.ContainerStatus) string { + switch { + case containerStatus.State.Waiting != nil: + return fmt.Sprintf("container %s: Waiting reason=%s msg=%s\n", + containerStatus.Name, containerStatus.State.Waiting.Reason, containerStatus.State.Waiting.Message) + case containerStatus.State.Terminated != nil: + return fmt.Sprintf("container %s: Terminated reason=%s exit=%d\n", + containerStatus.Name, containerStatus.State.Terminated.Reason, containerStatus.State.Terminated.ExitCode) + case containerStatus.State.Running != nil: + return fmt.Sprintf("container %s: Running ready=%t restarts=%d\n", + containerStatus.Name, containerStatus.Ready, containerStatus.RestartCount) + default: + return fmt.Sprintf("container %s: state unknown ready=%t\n", containerStatus.Name, containerStatus.Ready) + } +} + +// recentWarningEvents returns up to SBRCReadyDiagMaxEvents of the most recent Warning events in +// the namespace, newest first with age, formatted one per line for diagnostics. +func recentWarningEvents(ctx context.Context, namespace string) string { + evList, err := APIClient.Events(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Sprintf("Events: LIST failed: %v\n", err) + } + + warnings := make([]corev1.Event, 0, len(evList.Items)) + + for idx := range evList.Items { + if evList.Items[idx].Type == corev1.EventTypeWarning { + warnings = append(warnings, evList.Items[idx]) + } + } + + if len(warnings) == 0 { + return "Events: no Warning events\n" + } + + sort.Slice(warnings, func(i, j int) bool { + return eventTimestamp(warnings[i]).After(eventTimestamp(warnings[j])) + }) + + limit := len(warnings) + if limit > sbrparams.SBRCReadyDiagMaxEvents { + limit = sbrparams.SBRCReadyDiagMaxEvents + } + + var report strings.Builder + + fmt.Fprintf(&report, "Warning events (most recent %d of %d):\n", limit, len(warnings)) + + for idx := 0; idx < limit; idx++ { + event := warnings[idx] + age := time.Since(eventTimestamp(event)).Round(time.Second) + fmt.Fprintf(&report, " %s %s/%s (%s ago): %s\n", + event.Reason, event.InvolvedObject.Kind, event.InvolvedObject.Name, age, event.Message) + } + + return report.String() +} + +// eventTimestamp returns the most representative timestamp for an Event, preferring the newer +// Series/EventTime fields and falling back to the legacy timestamps, so "most recent" ordering is +// correct on clusters that leave LastTimestamp unset. +func eventTimestamp(event corev1.Event) time.Time { + switch { + case event.Series != nil && !event.Series.LastObservedTime.IsZero(): + return event.Series.LastObservedTime.Time + case !event.LastTimestamp.IsZero(): + return event.LastTimestamp.Time + case !event.EventTime.IsZero(): + return event.EventTime.Time + default: + return event.FirstTimestamp.Time + } } var _ = Describe(