CLOUDP-321067: update health check (#268)

# Summary

This pull request introduces health check functionality for the `mongot`
service, updates readiness and liveness probes, and removes a test
decorator for local testing. The changes enhance the monitoring and
reliability of the `mongot` service and simplify the test suite.

### Enhancements to `mongot` service health checks:

* Added a new constant `MongotDefautHealthCheckPort` set to port `8080`
in `mongodbsearch_types.go`. A helper method `GetMongotHealthCheckPort`
was also introduced to retrieve this value.
* Updated the `buildSearchHeadlessService` function to include a new
service port for health checks, using the `GetMongotHealthCheckPort`
method.
* Modified the `createMongotConfig` function to dynamically set the
health check address using the new health check port.

### Readiness and liveness probe improvements:

* Introduced constants for probe paths (`SearchLivenessProbePath` and
`SearchReadinessProbePath`) in `search_construction.go`.
* Replaced the existing readiness probe configuration with new methods
`mongotLivenessProbe` and `mongotReadinessProbe`. These methods
configure HTTP-based probes using the health check port and paths.
### Test suite simplifications:

* Removed the `@skip_if_local` decorator from several
`test_om_connectivity` test cases in `om_appdb_scale_up_down.py`,
enabling them to run in all environments.

Author: anandsyncs

api/v1/search/mongodbsearch_types.go

@@ -18,6 +18,7 @@ import (
 const (
 	MongotDefaultPort               = 27027
 	MongotDefaultMetricsPort        = 9946
+	MongotDefautHealthCheckPort = 8080
 	MongotDefaultSyncSourceUsername = "mongot-user"
 )
 
@@ -189,3 +190,7 @@ func (s *MongoDBSearch) TLSSecretNamespacedName() types.NamespacedName {
 func (s *MongoDBSearch) TLSOperatorSecretNamespacedName() types.NamespacedName {
 	return types.NamespacedName{Name: s.Name + "-search-certificate-key", Namespace: s.Namespace}
 }
+
+func (s *MongoDBSearch) GetMongotHealthCheckPort() int32 {
+	return MongotDefautHealthCheckPort
+}

controllers/search_controller/mongodbsearch_reconcile_helper.go

@@ -313,6 +313,13 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service {
 		TargetPort: intstr.FromInt32(search.GetMongotMetricsPort()),
 	})
 
+	serviceBuilder.AddPort(&corev1.ServicePort{
+		Name:       "healthcheck",
+		Protocol:   corev1.ProtocolTCP,
+		Port:       search.GetMongotHealthCheckPort(),
+		TargetPort: intstr.FromInt32(search.GetMongotHealthCheckPort()),
+	})
+
 	return serviceBuilder.Build()
 }
 
@@ -350,7 +357,7 @@ func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResourc
 			Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()),
 		}
 		config.HealthCheck = mongot.ConfigHealthCheck{
-			Address: "0.0.0.0:8080",
+			Address: fmt.Sprintf("localhost:%d", search.GetMongotHealthCheckPort()),
 		}
 		config.Logging = mongot.ConfigLogging{
 			Verbosity: "TRACE",

controllers/search_controller/search_construction.go

@@ -19,7 +19,9 @@ import (
 )
 
 const (
-	MongotContainerName = "mongot"
+	MongotContainerName      = "mongot"
+	SearchLivenessProbePath  = "/health"
+	SearchReadinessProbePath = "/health" // Todo: Update this when search GA is available
 )
 
 // SearchSourceDBResource is an object wrapping a MongoDBCommunity object
@@ -183,11 +185,8 @@ func mongodbSearchContainer(mdbSearch *searchv1.MongoDBSearch, volumeMounts []co
 		container.WithName(MongotContainerName),
 		container.WithImage(searchImage),
 		container.WithImagePullPolicy(corev1.PullAlways),
-		container.WithReadinessProbe(probes.Apply(
-			probes.WithTCPSocket("", intstr.FromInt32(mdbSearch.GetMongotPort())),
-			probes.WithInitialDelaySeconds(20),
-			probes.WithPeriodSeconds(10),
-		)),
+		container.WithLivenessProbe(mongotLivenessProbe(mdbSearch)),
+		container.WithReadinessProbe(mongotReadinessProbe(mdbSearch)),
 		container.WithResourceRequirements(createSearchResourceRequirements(mdbSearch.Spec.ResourceRequirements)),
 		container.WithVolumeMounts(volumeMounts),
 		container.WithCommand([]string{"sh"}),
@@ -209,6 +208,42 @@ chmod 0600 /tmp/sourceUserPassword
 	)
 }
 
+func mongotLivenessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) {
+	return probes.Apply(
+		probes.WithHandler(corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Scheme: corev1.URISchemeHTTP,
+				Port:   intstr.FromInt32(search.GetMongotHealthCheckPort()),
+				Path:   SearchLivenessProbePath,
+			},
+		}),
+		probes.WithInitialDelaySeconds(10),
+		probes.WithPeriodSeconds(10),
+		probes.WithTimeoutSeconds(5),
+		probes.WithSuccessThreshold(1),
+		probes.WithFailureThreshold(10),
+	)
+}
+
+// mongotReadinessProbe just uses the endpoint intended for liveness checks;
+// readiness check endpoint may be available in search GA.
+func mongotReadinessProbe(search *searchv1.MongoDBSearch) func(*corev1.Probe) {
+	return probes.Apply(
+		probes.WithHandler(corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Scheme: corev1.URISchemeHTTP,
+				Port:   intstr.FromInt32(search.GetMongotHealthCheckPort()),
+				Path:   SearchReadinessProbePath,
+			},
+		}),
+		probes.WithInitialDelaySeconds(20),
+		probes.WithPeriodSeconds(10),
+		probes.WithTimeoutSeconds(5),
+		probes.WithSuccessThreshold(1),
+		probes.WithFailureThreshold(3),
+	)
+}
+
 func createSearchResourceRequirements(requirements *corev1.ResourceRequirements) corev1.ResourceRequirements {
 	if requirements != nil {
 		return *requirements

docker/mongodb-kubernetes-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_scale_up_down.py

@@ -8,6 +8,7 @@
 from pytest import fixture
 from tests.conftest import is_multi_cluster
 
+
 # Important - you need to ensure that OM and Appdb images are build and pushed into your current docker registry before
 # running tests locally - use "make om-image" and "make appdb" to do this