diff --git a/ray-operator/test/e2e/raycluster_tls_test.go b/ray-operator/test/e2e/raycluster_tls_test.go index a6af57365c3..e0da2434feb 100644 --- a/ray-operator/test/e2e/raycluster_tls_test.go +++ b/ray-operator/test/e2e/raycluster_tls_test.go @@ -112,58 +112,46 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) { t.Skip("cert-manager CRDs not found; skipping TLS auto-generate e2e test") } - namespace := test.NewTestNamespace() - - t.Run("mTLS auto-generate cluster becomes Ready", func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - clusterName := "tls-autogen-mtls" - rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name). - WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion())) - - rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions) - g.Expect(err).NotTo(HaveOccurred()) - LogWithTimestamp(t, "Created RayCluster %s/%s with mTLS auto-generate", rayCluster.Namespace, rayCluster.Name) - - // Wait for cluster to become Ready - g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong). - Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) - LogWithTimestamp(t, "RayCluster %s is Ready", clusterName) - - // Verify head pod is running - headPod, err := GetHeadPod(test, rayCluster) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(headPod).NotTo(BeNil()) - g.Expect(IsPodRunningAndReady(headPod)).To(BeTrue(), "head pod should be running and ready") - - // Verify worker pods are running - workerPods, err := GetWorkerPods(test, rayCluster) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(workerPods).NotTo(BeEmpty()) - g.Expect(AllPodsRunningAndReady(workerPods)).To(BeTrue(), "all worker pods should be running and ready") - }) + g := NewWithT(t) - t.Run("mTLS auto-generate cert-manager resources created", func(t *testing.T) { - t.Parallel() + namespace := test.NewTestNamespace() + clusterName := "raycluster-tls-autogen" + + rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name). + WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion())) + rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions) + g.Expect(err).NotTo(HaveOccurred()) + LogWithTimestamp(t, "Created RayCluster %s/%s successfully with mTLS auto-generate", rayCluster.Namespace, rayCluster.Name) + + // Wait for cluster to become Ready + g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) + LogWithTimestamp(t, "RayCluster %s is Ready", clusterName) + + // Verify head pod is running + headPod, err := GetHeadPod(test, rayCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(headPod).NotTo(BeNil()) + g.Expect(IsPodRunningAndReady(headPod)).To(BeTrue(), "head pod should be running and ready") + + // Verify worker pods are running + workerPods, err := GetWorkerPods(test, rayCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(workerPods).NotTo(BeEmpty()) + g.Expect(AllPodsRunningAndReady(workerPods)).To(BeTrue(), "all worker pods should be running and ready") + + headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName) + workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName) + + test.T().Run("cert-manager resources created", func(t *testing.T) { + LogWithTimestamp(t, "Testing cert-manager resources") g := NewWithT(t) - clusterName := "tls-certmgr-res" - rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name). - WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion())) - - rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions) - g.Expect(err).NotTo(HaveOccurred()) - - g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong). - Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) - - // Re-fetch to get the UID assigned by the API server. - rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), clusterName, metav1.GetOptions{}) + refetchedRayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), clusterName, metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred()) // Verify cert-manager CA secret was created. - caSecretName := utils.GetCASecretName(clusterName, rayCluster.UID) + caSecretName := utils.GetCASecretName(clusterName, refetchedRayCluster.UID) caSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), caSecretName, metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred(), "CA secret %s should exist", caSecretName) g.Expect(caSecret.Data).To(HaveKey("tls.crt"), "CA secret should have tls.crt") @@ -171,7 +159,6 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) { g.Expect(caSecret.Data).To(HaveKey("ca.crt"), "CA secret should have ca.crt") // Verify head certificate secret was created by cert-manager. - headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName) headSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), headSecretName, metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred(), "head secret %s should exist", headSecretName) g.Expect(headSecret.Data).To(HaveKey("tls.crt"), "head secret should have tls.crt") @@ -179,7 +166,6 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) { g.Expect(headSecret.Data).To(HaveKey("ca.crt"), "head secret should have ca.crt") // Verify worker certificate secret was created by cert-manager. - workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName) workerSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), workerSecretName, metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred(), "worker secret %s should exist", workerSecretName) g.Expect(workerSecret.Data).To(HaveKey("tls.crt"), "worker secret should have tls.crt") @@ -189,33 +175,15 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) { LogWithTimestamp(t, "Cert-manager CA, head, and worker secrets verified for cluster %s", clusterName) }) - t.Run("mTLS auto-generate pod TLS configuration", func(t *testing.T) { - t.Parallel() + test.T().Run("pod TLS configuration", func(t *testing.T) { + LogWithTimestamp(t, "Testing pod TLS configuration") g := NewWithT(t) - clusterName := "tls-pod-config" - rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name). - WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion())) - - rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions) - g.Expect(err).NotTo(HaveOccurred()) - - g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong). - Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) - - // Auto-generate mode: head pod mounts the head secret, workers mount the worker secret. - headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName) - workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName) - // Verify head pod configuration - headPod, err := GetHeadPod(test, rayCluster) - g.Expect(err).NotTo(HaveOccurred()) verifyContainerTLSEnvVars(g, &headPod.Spec.Containers[utils.RayContainerIndex]) verifyTLSVolumeMount(g, headPod, headSecretName) // Verify worker pod configuration - workerPods, err := GetWorkerPods(test, rayCluster) - g.Expect(err).NotTo(HaveOccurred()) g.Expect(workerPods).NotTo(BeEmpty()) for i := range workerPods { verifyContainerTLSEnvVars(g, &workerPods[i].Spec.Containers[utils.RayContainerIndex]) @@ -225,23 +193,10 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) { LogWithTimestamp(t, "Pod TLS configuration verified for cluster %s", clusterName) }) - t.Run("mTLS auto-generate Ray job submission succeeds", func(t *testing.T) { - t.Parallel() + test.T().Run("Ray job submission succeeds", func(t *testing.T) { + LogWithTimestamp(t, "Testing Ray job submission with auto-generated") g := NewWithT(t) - clusterName := "tls-job-submit" - rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name). - WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion())) - - rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions) - g.Expect(err).NotTo(HaveOccurred()) - - g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong). - Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) - - headPod, err := GetHeadPod(test, rayCluster) - g.Expect(err).NotTo(HaveOccurred()) - // Submit a simple Ray job and verify it completes. // This proves head-worker mTLS communication is functional. submissionID := fmt.Sprintf("mtls-test-job-%d", time.Now().Unix())