diff --git a/tests/far-operator/README.md b/tests/far-operator/README.md index 19dad8684..456f87c88 100644 --- a/tests/far-operator/README.md +++ b/tests/far-operator/README.md @@ -198,4 +198,4 @@ Validates the full customer upgrade path: install GA FAR from redhat-operators o - **CI prerequisite**: `medik8s-catalogsource` step must run before the test (creates the `medik8s-catalog` CatalogSource) - **Env vars (optional, have defaults)**: `MEDIK8S_OPERATOR_PACKAGE` (default: `fence-agents-remediation`), `MEDIK8S_TARGET_CHANNEL` (default: `stable`) - **Standalone**: `ginkgo --label-filter="far && tier:upgrade" ./tests/far-operator/...` -- **Pass criteria**: FAR deployment Ready on OCP N-1, OCP upgrade completes (Progressing=False, Available=True, Degraded=False), FAR deployment Ready after OCP upgrade, FAR CSV in Succeeded phase after both upgrades, controller image changes after operator upgrade, remediation succeeds with both GA and pre-GA operator versions (node rebooted via boot ID change, node recovers to Ready), workload pods evicted after each fencing cycle +- **Pass criteria**: FAR deployment Ready on OCP N-1, OCP upgrade completes (Progressing=False, Available=True, Degraded=False), FAR deployment Ready after OCP upgrade, FAR CSV in Succeeded phase after catalog switch (new CSV if Konflux version is higher than GA, same CSV if versions match), controller image changes after operator upgrade (skipped on version parity), remediation succeeds after OCP upgrade and after catalog switch (node rebooted via boot ID change, node recovers to Ready), workload pods evicted after each fencing cycle diff --git a/tests/far-operator/tests/upgrade.go b/tests/far-operator/tests/upgrade.go index 1fed02d4d..648a73033 100644 --- a/tests/far-operator/tests/upgrade.go +++ b/tests/far-operator/tests/upgrade.go @@ -35,16 +35,17 @@ var _ = Describe("FAR Operator Upgrade", labels.ComponentOLM), func() { var ( - ctx context.Context - previousCSV *olm.ClusterServiceVersionBuilder - preUpgradeImage string - platform configv1.PlatformType - region string - fenceAgent string - sharedParams map[string]interface{} - nodeParams map[string]interface{} - leaderNode string - currentFARName string + ctx context.Context + previousCSV *olm.ClusterServiceVersionBuilder + preUpgradeImage string + platform configv1.PlatformType + region string + fenceAgent string + sharedParams map[string]interface{} + nodeParams map[string]interface{} + leaderNode string + currentFARName string + operatorUpgraded bool ) BeforeAll(func() { @@ -260,60 +261,137 @@ var _ = Describe("FAR Operator Upgrade", Expect(err).NotTo(HaveOccurred(), "Failed to switch Subscription to target catalog") - By("Step 9: Wait for new CSV and verify it reached Succeeded") + By("Step 9: Wait for operator upgrade or version parity after catalog switch") Eventually(func() error { - csvs, listErr := olm.ListClusterServiceVersionWithNamePattern( - APIClient, medik8sparams.OperatorPackage, medik8sparams.OperatorNs) - if listErr != nil { - return listErr + sub, subErr := olm.PullSubscription( + APIClient, farparams.UpgradeSubName, medik8sparams.OperatorNs) + if subErr != nil { + return fmt.Errorf("pulling subscription %s/%s: %w", + medik8sparams.OperatorNs, farparams.UpgradeSubName, subErr) } - for _, csv := range csvs { - csvPhase, _ := csv.GetPhase() - if csvPhase == olmV1alpha1.CSVPhaseSucceeded && - csv.Object.Name != previousCSV.Object.Name { - GinkgoWriter.Printf("New CSV: %s (was: %s)\n", - csv.Object.Name, previousCSV.Object.Name) + if sub == nil || sub.Object == nil { + return fmt.Errorf( + "subscription %s/%s returned without error but Object is nil", + medik8sparams.OperatorNs, farparams.UpgradeSubName) + } + + if sub.Object.Spec.CatalogSource != medik8sparams.UpgradeCatalogName { + return fmt.Errorf("subscription source not yet updated to %s", + medik8sparams.UpgradeCatalogName) + } + + currentCSV := sub.Object.Status.CurrentCSV + if currentCSV == "" { + return fmt.Errorf("subscription has no currentCSV yet") + } + + installedCSV := sub.Object.Status.InstalledCSV + if installedCSV != currentCSV { + return fmt.Errorf( + "OLM still reconciling (installed: %s, current: %s)", + installedCSV, currentCSV) + } - return nil + for _, cond := range sub.Object.Status.Conditions { + if cond.Type == olmV1alpha1.SubscriptionCatalogSourcesUnhealthy && + cond.Status == corev1.ConditionTrue { + return fmt.Errorf( + "catalog unhealthy: %s", cond.Message) } } - return fmt.Errorf("new FAR CSV not yet in Succeeded phase") - }, medik8sparams.OperatorUpgradeTimeout, farparams.DefaultPollInterval).Should(Succeed(), - "Operator upgrade did not complete") + catalogHealthy := false - By("Step 10: Verify FAR controller pods restarted with new image") + for _, ch := range sub.Object.Status.CatalogHealth { + if ch.CatalogSourceRef != nil && + ch.CatalogSourceRef.Name == medik8sparams.UpgradeCatalogName && + ch.Healthy { + catalogHealthy = true - Eventually(func() error { - currentImage, imgErr := farutils.GetFARControllerImage(APIClient) - if imgErr != nil { - return imgErr + break + } + } + + if !catalogHealthy { + return fmt.Errorf( + "catalog %s not yet healthy in subscription CatalogHealth", + medik8sparams.UpgradeCatalogName) + } + + csv, csvErr := olm.PullClusterServiceVersion( + APIClient, currentCSV, medik8sparams.OperatorNs) + if csvErr != nil { + return fmt.Errorf("CSV %s not found: %w", currentCSV, csvErr) } - if currentImage == preUpgradeImage { - return fmt.Errorf("controller still running old image %s", preUpgradeImage) + csvPhase, phaseErr := csv.GetPhase() + if phaseErr != nil { + return fmt.Errorf("failed to get phase for CSV %s: %w", + currentCSV, phaseErr) } - GinkgoWriter.Printf("Controller image updated: %s\n", currentImage) + if csvPhase != olmV1alpha1.CSVPhaseSucceeded { + return fmt.Errorf("CSV %s in phase %s, waiting for Succeeded", + currentCSV, csvPhase) + } + + if currentCSV != previousCSV.Object.Name { + GinkgoWriter.Printf( + "Operator upgraded: new CSV %s (was: %s)\n", + currentCSV, previousCSV.Object.Name) + + operatorUpgraded = true + } else { + GinkgoWriter.Printf( + "Version parity: Konflux catalog offers same "+ + "version %s as GA; subscription healthy on "+ + "new catalog\n", currentCSV) + } return nil }, medik8sparams.OperatorUpgradeTimeout, farparams.DefaultPollInterval).Should(Succeed(), - "FAR controller pods did not restart with new image") + "Operator upgrade or catalog switch verification failed") + + if operatorUpgraded { + By("Step 10: Verify FAR controller pods restarted with new image") + + Eventually(func() error { + currentImage, imgErr := farutils.GetFARControllerImage(APIClient) + if imgErr != nil { + return imgErr + } + + if currentImage == preUpgradeImage { + return fmt.Errorf("controller still running old image %s", + preUpgradeImage) + } + + GinkgoWriter.Printf("Controller image updated: %s\n", currentImage) - By("Step 11: Validate pre-GA FAR on OCP N (post-operator-upgrade remediation)") + return nil + }, medik8sparams.OperatorUpgradeTimeout, + farparams.DefaultPollInterval).Should(Succeed(), + "FAR controller pods did not restart with new image") + } else { + GinkgoWriter.Println( + "Step 10: Skipped (no operator upgrade occurred, " + + "Konflux and GA catalogs at same version)") + } + + By("Step 11: Validate FAR on OCP N (post-catalog-switch remediation)") fenceAgent, sharedParams, nodeParams, leaderNode, err = upgradeProvisionRemediationResources(ctx, platform, region) Expect(err).NotTo(HaveOccurred(), "Failed to set up remediation resources") currentFARName, err = upgradeRunRemediationCycle( - ctx, fenceAgent, sharedParams, nodeParams, leaderNode, "post-operator-upgrade") + ctx, fenceAgent, sharedParams, nodeParams, leaderNode, "post-catalog-switch") Expect(err).NotTo(HaveOccurred(), - "Post-operator-upgrade remediation failed with pre-GA operator") + "Post-catalog-switch remediation failed") - cleanupPostRemediation(ctx, ¤tFARName, "post-operator-upgrade") + cleanupPostRemediation(ctx, ¤tFARName, "post-catalog-switch") }) }) @@ -489,7 +567,12 @@ func verifyFAROperatorReady( APIClient, medik8sparams.OperatorPackage, medik8sparams.OperatorNs) if csvListErr == nil { for _, c := range csvs { - phase, _ := c.GetPhase() + phase, phaseErr := c.GetPhase() + if phaseErr != nil { + return fmt.Errorf("failed to get phase for CSV %s: %w", + c.Object.Name, phaseErr) + } + phaseStr := string(phase) if phaseStr != lastCSVPhase { GinkgoWriter.Printf("[OLM] CSV %s: phase=%s reason=%s message=%s\n",