diff --git a/cmd/service-ca-operator-tests-ext/main.go b/cmd/service-ca-operator-tests-ext/main.go index 56eaa2440..cee38eede 100644 --- a/cmd/service-ca-operator-tests-ext/main.go +++ b/cmd/service-ca-operator-tests-ext/main.go @@ -69,14 +69,23 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { registry := oteextension.NewRegistry() extension := oteextension.NewExtension("openshift", "payload", "service-ca-operator") - // The following suite runs tests that verify the operator's behaviour. - // This suite is executed only on pull requests targeting this repository. - // Tests tagged with both [Operator] and [Serial] are included in this suite. + // Non-disruptive tests run with default (Stable) cluster health monitoring. extension.AddSuite(oteextension.Suite{ Name: "openshift/service-ca-operator/operator/serial", Parallelism: 1, Qualifiers: []string{ - `name.contains("[Operator]") && name.contains("[Serial]")`, + `name.contains("[Operator]") && name.contains("[Serial]") && !name.contains("[Disruptive]")`, + }, + }) + + // Disruptive tests (e.g. CA rotation) that cause expected cluster-wide TLS disruption. + // Monitors will relax thresholds for this suite. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/service-ca-operator/operator/serial-disruptive", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Operator]") && name.contains("[Serial]") && name.contains("[Disruptive]")`, }, }) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index e6adf2e2b..4ca89965f 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -70,6 +70,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { }) }) + g.Context("headless-stateful-serving-cert-secret-delete-data", func() { + g.It("[Operator][Serial] should regenerate deleted serving cert secrets for StatefulSet with headless service", func() { + testHeadlessStatefulServingCertSecretDeleteData(g.GinkgoTB()) + }) + }) + g.Context("ca-bundle-injection-configmap", func() { g.It("[Operator][Serial] should inject CA bundle into annotated configmaps", func() { testCABundleInjectionConfigMap(g.GinkgoTB()) @@ -88,12 +94,6 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { }) }) - g.Context("headless-stateful-serving-cert-secret-delete-data", func() { - g.It("[Operator][Serial] should regenerate deleted serving cert secrets for StatefulSet with headless service", func() { - testHeadlessStatefulServingCertSecretDeleteData(g.GinkgoTB()) - }) - }) - g.Context("metrics", func() { g.It("[Operator][Serial] should collect metrics from the operator", func() { testMetricsCollection(g.GinkgoTB()) @@ -103,6 +103,12 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() { testServiceCAMetrics(g.GinkgoTB()) }) }) + + g.Context("refresh-CA", func() { + g.It("[Operator][Serial][Disruptive] should regenerate serving certs and configmaps when CA is deleted and recreated", func() { + testRefreshCA(g.GinkgoTB()) + }) + }) }) // testServingCertAnnotation checks that services with the serving-cert annotation @@ -1220,3 +1226,124 @@ func getSampleForPromQueryGinkgo(t testing.TB, promClient prometheusv1.API, quer } return res[0], nil } + +// testRefreshCA verifies that when the CA secret is deleted and recreated, +// all serving certs and configmaps get updated with the new CA. +// +// This test uses testing.TB interface for dual-compatibility with both +// standard Go tests and Ginkgo tests. +// +// This situation is temporary until we test the new e2e jobs with OTE. +// Eventually all tests will be run only as part of the OTE framework. +func testRefreshCA(t testing.TB) { + adminClient, err := getKubeClient() + if err != nil { + t.Fatalf("error getting kube client: %v", err) + } + + ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) + if err != nil { + t.Fatalf("could not create test namespace: %v", err) + } + defer cleanup() + + // create secrets + testServiceName := "test-service-" + randSeq(5) + testSecretName := "test-secret-" + randSeq(5) + testHeadlessServiceName := "test-headless-service-" + randSeq(5) + testHeadlessSecretName := "test-headless-secret-" + randSeq(5) + + err = createServingCertAnnotatedService(adminClient, testSecretName, testServiceName, ns.Name, false) + if err != nil { + t.Fatalf("error creating annotated service: %v", err) + } + if err = createServingCertAnnotatedService(adminClient, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil { + t.Fatalf("error creating annotated headless service: %v", err) + } + + secret, err := pollForServiceServingSecretWithReturn(adminClient, testSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching created serving cert secret: %v", err) + } + secretCopy := secret.DeepCopy() + headlessSecret, err := pollForServiceServingSecretWithReturn(adminClient, testHeadlessSecretName, ns.Name) + if err != nil { + t.Fatalf("error fetching created serving cert secret: %v", err) + } + headlessSecretCopy := headlessSecret.DeepCopy() + + // create configmap + testConfigMapName := "test-configmap-" + randSeq(5) + + err = createAnnotatedCABundleInjectionConfigMap(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error creating annotated configmap: %v", err) + } + + configmap, err := pollForCABundleInjectionConfigMapWithReturn(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error fetching ca bundle injection configmap: %v", err) + } + configmapCopy := configmap.DeepCopy() + err = checkConfigMapCABundleInjectionData(adminClient, testConfigMapName, ns.Name) + if err != nil { + t.Fatalf("error when checking ca bundle injection configmap: %v", err) + } + + // delete ca secret + err = adminClient.CoreV1().Secrets("openshift-service-ca").Delete(context.TODO(), "signing-key", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("error deleting signing key: %v", err) + } + + // make sure it's recreated + err = pollForCARecreation(adminClient) + if err != nil { + t.Fatalf("signing key was not recreated: %v", err) + } + + err = pollForConfigMapChange(t, adminClient, configmapCopy, api.InjectionDataKey) + if err != nil { + t.Fatalf("configmap bundle did not change: %v", err) + } + + err = pollForSecretChangeGinkgo(t, adminClient, secretCopy, v1.TLSCertKey, v1.TLSPrivateKeyKey) + if err != nil { + t.Fatalf("secret cert did not change: %v", err) + } + if err := pollForSecretChangeGinkgo(t, adminClient, headlessSecretCopy); err != nil { + t.Fatalf("headless secret cert did not change: %v", err) + } +} + +// pollForCABundleInjectionConfigMapWithReturn polls for a CA bundle injection configmap and returns it. +func pollForCABundleInjectionConfigMapWithReturn(client *kubernetes.Clientset, configMapName, namespace string) (*v1.ConfigMap, error) { + var configmap *v1.ConfigMap + err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + cm, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + configmap = cm + return true, nil + }) + return configmap, err +} + +// pollForCARecreation polls for the signing secret to be re-created in +// response to CA secret deletion. +func pollForCARecreation(client *kubernetes.Clientset) error { + return wait.PollImmediate(time.Second, rotationPollTimeout, func() (bool, error) { + _, err := client.CoreV1().Secrets("openshift-service-ca").Get(context.TODO(), "signing-key", metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil + }) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4fe9d75e6..e51a6c6b3 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -93,22 +93,6 @@ func editServingSecretData(t *testing.T, client *kubernetes.Clientset, secretNam return pollForSecretChange(t, client, scopy, keyName) } -func pollForCABundleInjectionConfigMapWithReturn(client *kubernetes.Clientset, configMapName, namespace string) (*v1.ConfigMap, error) { - var configmap *v1.ConfigMap - err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - cm, err := client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - configmap = cm - return true, nil - }) - return configmap, err -} - func pollForSecretChange(t *testing.T, client *kubernetes.Clientset, secret *v1.Secret, keysToChange ...string) error { return wait.PollImmediate(pollInterval, rotationPollTimeout, func() (bool, error) { s, err := client.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}) @@ -352,19 +336,6 @@ func pollForCARotation(t *testing.T, client *kubernetes.Clientset, caCertPEM, ca // pollForCARecreation polls for the signing secret to be re-created in // response to CA secret deletion. -func pollForCARecreation(client *kubernetes.Clientset) error { - return wait.PollImmediate(time.Second, rotationPollTimeout, func() (bool, error) { - _, err := client.CoreV1().Secrets(serviceCAControllerNamespace).Get(context.TODO(), signingKeySecretName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil - }) -} - // pollForUpdatedServingCert returns the cert and key PEM if it changes from // that provided before the polling timeout. @@ -723,80 +694,12 @@ func TestE2E(t *testing.T) { }) }) + // test CA refresh - delete and recreate CA secret, verify serving certs and configmaps are updated + // NOTE: This test is also available in the OTE framework (test/e2e/e2e.go). + // This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs. + // Eventually, all tests will run only through the OTE framework. t.Run("refresh-CA", func(t *testing.T) { - ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5)) - if err != nil { - t.Fatalf("could not create test namespace: %v", err) - } - defer cleanup() - - // create secrets - testServiceName := "test-service-" + randSeq(5) - testSecretName := "test-secret-" + randSeq(5) - testHeadlessServiceName := "test-headless-service-" + randSeq(5) - testHeadlessSecretName := "test-headless-secret-" + randSeq(5) - - err = createServingCertAnnotatedService(adminClient, testSecretName, testServiceName, ns.Name, false) - if err != nil { - t.Fatalf("error creating annotated service: %v", err) - } - if err = createServingCertAnnotatedService(adminClient, testHeadlessSecretName, testHeadlessServiceName, ns.Name, true); err != nil { - t.Fatalf("error creating annotated headless service: %v", err) - } - - secret, err := pollForServiceServingSecretWithReturn(adminClient, testSecretName, ns.Name) - if err != nil { - t.Fatalf("error fetching created serving cert secret: %v", err) - } - secretCopy := secret.DeepCopy() - headlessSecret, err := pollForServiceServingSecretWithReturn(adminClient, testHeadlessSecretName, ns.Name) - if err != nil { - t.Fatalf("error fetching created serving cert secret: %v", err) - } - headlessSecretCopy := headlessSecret.DeepCopy() - - // create configmap - testConfigMapName := "test-configmap-" + randSeq(5) - - err = createAnnotatedCABundleInjectionConfigMap(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error creating annotated configmap: %v", err) - } - - configmap, err := pollForCABundleInjectionConfigMapWithReturn(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error fetching ca bundle injection configmap: %v", err) - } - configmapCopy := configmap.DeepCopy() - err = checkConfigMapCABundleInjectionData(adminClient, testConfigMapName, ns.Name) - if err != nil { - t.Fatalf("error when checking ca bundle injection configmap: %v", err) - } - - // delete ca secret - err = adminClient.CoreV1().Secrets(serviceCAControllerNamespace).Delete(context.TODO(), signingKeySecretName, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("error deleting signing key: %v", err) - } - - // make sure it's recreated - err = pollForCARecreation(adminClient) - if err != nil { - t.Fatalf("signing key was not recreated: %v", err) - } - - err = pollForConfigMapChange(t, adminClient, configmapCopy, api.InjectionDataKey) - if err != nil { - t.Fatalf("configmap bundle did not change: %v", err) - } - - err = pollForSecretChange(t, adminClient, secretCopy, v1.TLSCertKey, v1.TLSPrivateKeyKey) - if err != nil { - t.Fatalf("secret cert did not change: %v", err) - } - if err := pollForSecretChange(t, adminClient, headlessSecretCopy); err != nil { - t.Fatalf("headless secret cert did not change: %v", err) - } + testRefreshCA(t) }) // This test triggers rotation by updating the CA to have an