Skip to content

[Refactor] Share RayCluster across mTLS e2e checks #5097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HsiaoHungKai wants to merge 5 commits into
base: master
Choose a base branch
from
Open

[Refactor] Share RayCluster across mTLS e2e checks #5097

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c0f445f
[Refactor] Share RayCluster across TLS e2e checks
HsiaoHungKai Jul 31, 2026
39dec75
[Refactor] Initialize Gomega inside parallel TLS subtest
HsiaoHungKai Aug 6, 2026
a10bbcc
[Refactor] Modify log messages
HsiaoHungKai Aug 7, 2026
be5ecae
[Refactor] Flatten auto-generated tls test structure
HsiaoHungKai Aug 8, 2026
b40e0fc
[Refactor] Remove parallel for auto-generated tls test to avoid timeo…
HsiaoHungKai Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 38 additions & 83 deletions ray-operator/test/e2e/raycluster_tls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,74 +112,60 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) {
t.Skip("cert-manager CRDs not found; skipping TLS auto-generate e2e test")
}

namespace := test.NewTestNamespace()

t.Run("mTLS auto-generate cluster becomes Ready", func(t *testing.T) {
t.Parallel()
g := NewWithT(t)

clusterName := "tls-autogen-mtls"
rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name).
WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion()))

rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
g.Expect(err).NotTo(HaveOccurred())
LogWithTimestamp(t, "Created RayCluster %s/%s with mTLS auto-generate", rayCluster.Namespace, rayCluster.Name)

// Wait for cluster to become Ready
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
LogWithTimestamp(t, "RayCluster %s is Ready", clusterName)

// Verify head pod is running
headPod, err := GetHeadPod(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(headPod).NotTo(BeNil())
g.Expect(IsPodRunningAndReady(headPod)).To(BeTrue(), "head pod should be running and ready")

// Verify worker pods are running
workerPods, err := GetWorkerPods(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(workerPods).NotTo(BeEmpty())
g.Expect(AllPodsRunningAndReady(workerPods)).To(BeTrue(), "all worker pods should be running and ready")
})
g := NewWithT(t)

t.Run("mTLS auto-generate cert-manager resources created", func(t *testing.T) {
t.Parallel()
namespace := test.NewTestNamespace()
clusterName := "raycluster-tls-autogen"

rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name).
WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion()))
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
g.Expect(err).NotTo(HaveOccurred())
LogWithTimestamp(t, "Created RayCluster %s/%s successfully with mTLS auto-generate", rayCluster.Namespace, rayCluster.Name)

// Wait for cluster to become Ready
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
LogWithTimestamp(t, "RayCluster %s is Ready", clusterName)

// Verify head pod is running
headPod, err := GetHeadPod(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(headPod).NotTo(BeNil())
g.Expect(IsPodRunningAndReady(headPod)).To(BeTrue(), "head pod should be running and ready")

// Verify worker pods are running
workerPods, err := GetWorkerPods(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(workerPods).NotTo(BeEmpty())
g.Expect(AllPodsRunningAndReady(workerPods)).To(BeTrue(), "all worker pods should be running and ready")

headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName)
workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName)

test.T().Run("cert-manager resources created", func(t *testing.T) {
LogWithTimestamp(t, "Testing cert-manager resources")
g := NewWithT(t)
Comment thread
HsiaoHungKai marked this conversation as resolved.

clusterName := "tls-certmgr-res"
rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name).
WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion()))

rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
g.Expect(err).NotTo(HaveOccurred())

g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))

// Re-fetch to get the UID assigned by the API server.
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), clusterName, metav1.GetOptions{})
refetchedRayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Get(test.Ctx(), clusterName, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred())

// Verify cert-manager CA secret was created.
caSecretName := utils.GetCASecretName(clusterName, rayCluster.UID)
caSecretName := utils.GetCASecretName(clusterName, refetchedRayCluster.UID)
caSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), caSecretName, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred(), "CA secret %s should exist", caSecretName)
g.Expect(caSecret.Data).To(HaveKey("tls.crt"), "CA secret should have tls.crt")
g.Expect(caSecret.Data).To(HaveKey("tls.key"), "CA secret should have tls.key")
g.Expect(caSecret.Data).To(HaveKey("ca.crt"), "CA secret should have ca.crt")

// Verify head certificate secret was created by cert-manager.
headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName)
headSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), headSecretName, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred(), "head secret %s should exist", headSecretName)
g.Expect(headSecret.Data).To(HaveKey("tls.crt"), "head secret should have tls.crt")
g.Expect(headSecret.Data).To(HaveKey("tls.key"), "head secret should have tls.key")
g.Expect(headSecret.Data).To(HaveKey("ca.crt"), "head secret should have ca.crt")

// Verify worker certificate secret was created by cert-manager.
workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName)
workerSecret, err := test.Client().Core().CoreV1().Secrets(namespace.Name).Get(test.Ctx(), workerSecretName, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred(), "worker secret %s should exist", workerSecretName)
g.Expect(workerSecret.Data).To(HaveKey("tls.crt"), "worker secret should have tls.crt")
Expand All @@ -189,33 +175,15 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) {
LogWithTimestamp(t, "Cert-manager CA, head, and worker secrets verified for cluster %s", clusterName)
})

t.Run("mTLS auto-generate pod TLS configuration", func(t *testing.T) {
t.Parallel()
test.T().Run("pod TLS configuration", func(t *testing.T) {
LogWithTimestamp(t, "Testing pod TLS configuration")
g := NewWithT(t)

clusterName := "tls-pod-config"
rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name).
WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion()))

rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
g.Expect(err).NotTo(HaveOccurred())

g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))

// Auto-generate mode: head pod mounts the head secret, workers mount the worker secret.
headSecretName := fmt.Sprintf("%s-%s", utils.RayHeadSecretPrefix, clusterName)
workerSecretName := fmt.Sprintf("%s-%s", utils.RayWorkerSecretPrefix, clusterName)

// Verify head pod configuration
headPod, err := GetHeadPod(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
verifyContainerTLSEnvVars(g, &headPod.Spec.Containers[utils.RayContainerIndex])
verifyTLSVolumeMount(g, headPod, headSecretName)

// Verify worker pod configuration
workerPods, err := GetWorkerPods(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(workerPods).NotTo(BeEmpty())
for i := range workerPods {
verifyContainerTLSEnvVars(g, &workerPods[i].Spec.Containers[utils.RayContainerIndex])
Expand All @@ -225,23 +193,10 @@ func TestRayClusterTLSAutoGenerate(t *testing.T) {
LogWithTimestamp(t, "Pod TLS configuration verified for cluster %s", clusterName)
})

t.Run("mTLS auto-generate Ray job submission succeeds", func(t *testing.T) {
t.Parallel()
test.T().Run("Ray job submission succeeds", func(t *testing.T) {
LogWithTimestamp(t, "Testing Ray job submission with auto-generated")
g := NewWithT(t)

clusterName := "tls-job-submit"
rayClusterAC := rayv1ac.RayCluster(clusterName, namespace.Name).
WithSpec(NewRayClusterSpecWithMTLS().WithRayVersion(GetRayVersion()))

rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
g.Expect(err).NotTo(HaveOccurred())

g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))

headPod, err := GetHeadPod(test, rayCluster)
g.Expect(err).NotTo(HaveOccurred())

// Submit a simple Ray job and verify it completes.
// This proves head-worker mTLS communication is functional.
submissionID := fmt.Sprintf("mtls-test-job-%d", time.Now().Unix())
Expand Down
Loading