Merge pull request #109 from cloudogu/feature/108_fix_healthcheck_in_multinode

#108 fix ready health check in multinode

Author: meiserloh

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Dogu state did not become `ready` in multinode [#108]
+  - This is because Kubernetes ignores Dockerfile HEALTHCHECK statements
 
 ## [3.7.4-2] 2025-04-02
 ### Added

Dockerfile

@@ -68,7 +68,7 @@ EXPOSE 8080 2222
 
 USER scm
 
-HEALTHCHECK --start-period=30s --interval=20s CMD /healthcheck.sh
+HEALTHCHECK --interval=5s CMD doguctl healthy scm || exit 1
 
 # start scm
 CMD ["/startup.sh"]

batsTests/healthcheck.bats

@@ -7,7 +7,6 @@ load '/workspace/target/bats_libs/bats-assert/load.bash'
 load '/workspace/target/bats_libs/bats-mock/load.bash'
 
 setup() {
-  export CES_TOKEN_CONFIGURATION_KEY=TestConfig
   export CES_TOKEN_HEADER=TestHeader
 
   doguctl="$(mock_create)"
@@ -21,7 +20,7 @@ setup() {
 }
 
 teardown() {
-  unset CES_TOKEN_CONFIGURATION_KEY
+  unset CES_TOKEN_HEADER
 
   rm "${BATS_TMPDIR}/doguctl"
   rm "${BATS_TMPDIR}/curl"
@@ -30,66 +29,29 @@ teardown() {
 @test "health_check passes when everything is healthy" {
   source /workspace/resources/healthcheck.sh
 
-  # config --encrypted
-  mock_set_status "${doguctl}" 0 1
-  mock_set_output "${doguctl}" "fake_token" 1
-
   # curl
   mock_set_status "${curl}" 0
   mock_set_output "${curl}" "200"
 
-  mock_set_status "${doguctl}" 0 2            # doguctl state ready
-  mock_set_status "${doguctl}" 0 3            # doguctl healthy
-  mock_set_status "${doguctl}" 0 4            # doguctl state ready
-
-  run runHealthCheck
+  run runHealthCheck myApiToken
 
   assert_success
   assert_equal "$(mock_get_call_num "${curl}")" "1"
-  assert_equal "$(mock_get_call_num "${doguctl}")" "4"
-  assert_equal "$(mock_get_call_args "${doguctl}" "4")" "state ready"
+  assert_equal "$(mock_get_call_num "${doguctl}")" "0"
 }
 
 @test "health_check fails when curl returns != 200" {
   source /workspace/resources/healthcheck.sh
 
-  # config --encrypted
-  mock_set_status "${doguctl}" 0 1
-  mock_set_output "${doguctl}" "fake_token" 1
-
   # curl
   mock_set_status "${curl}" 0
   mock_set_output "${curl}" "401"
 
-  run runHealthCheck
+  run runHealthCheck myApiToken
 
   assert_failure
   assert_equal "$(mock_get_call_num "${curl}")" "1"
-  assert_equal "$(mock_get_call_num "${doguctl}")" "2"
-  assert_equal "$(mock_get_call_args "${doguctl}" "2")" "state unhealthy"
-}
-
-@test "health_check fails when doguctl healthy returns failure (exit 1)" {
-   source /workspace/resources/healthcheck.sh
-
-    # config --encrypted
-    mock_set_status "${doguctl}" 0 1
-    mock_set_output "${doguctl}" "fake_token" 1
-
-    # curl
-    mock_set_status "${curl}" 0
-    mock_set_output "${curl}" "200"
-
-    mock_set_status "${doguctl}" 0 2            # doguctl state ready
-    mock_set_status "${doguctl}" 1 3            # doguctl healthy
-    mock_set_status "${doguctl}" 0 4            # doguctl state ready
-
-    run runHealthCheck
-
-    assert_failure
-    assert_equal "$(mock_get_call_num "${curl}")" "1"
-    assert_equal "$(mock_get_call_num "${doguctl}")" "4"
-    assert_equal "$(mock_get_call_args "${doguctl}" "4")" "state unhealthy"
+  assert_equal "$(mock_get_call_num "${doguctl}")" "0"
 }
 
 

resources/healthcheck.sh

@@ -5,47 +5,54 @@ set -o pipefail
 
 # Script to perform a health check
 #
-# This script monitors the health of the SCM-Dogu. It uses `doguctl`
-# and a HTTP endpoint to perform the health checks. This is necessary as SCM-Dogu provides no dedicated health
-# endpoint at the moment.
+# This script monitors the health of the SCM-Dogu on startup.
 #
 # Notes:
-# - Ensure that the script has execute permissions (`chmod +x health_check.sh`).
-# - Configure the script in the Docker container as a health check using Docker's `HEALTHCHECK` instruction.
+# - Ensure that the script has execute permissions (`chmod +x healthcheck.sh`).
 #
 # Example:
-#   ./health_check.sh
+#   ./healthcheck.sh
 
 function runHealthCheck() {
-  # connection token
-  API_TOKEN=$(doguctl config --encrypted "${CES_TOKEN_CONFIGURATION_KEY}")
+  local api_token=${1}
 
   # Perform health check against scm endpoint
-  HTTP_STATUS=$(curl --write-out "%{http_code}" --silent --output /dev/null --max-time 10 http://localhost:8080/scm/api/v2 -H "${CES_TOKEN_HEADER}: ${API_TOKEN}") || HTTP_STATUS=0
+  HTTP_STATUS=$(curl --write-out "%{http_code}" --silent --output /dev/null --max-time 10 http://localhost:8080/scm/api/v2 -H "${CES_TOKEN_HEADER}: ${api_token}") || HTTP_STATUS=0
   if [[ "$HTTP_STATUS" -ne 200 ]]; then
     echo "scm is unhealthy, received http status code ${HTTP_STATUS}"
-    doguctl state "unhealthy"
     return 1
   fi
 
-
-  # Successful http check means tcp port is also available
-  doguctl state "ready"
-
-  # Perform health check using doguctl
-  if ! doguctl healthy -t 10 scm; then
-    echo "doguctl is unhealthy"
-    doguctl state "unhealthy"
-    return 1
-  fi
-
-  doguctl state "ready"
-
   return 0
+}
 
+waitForHealth () {
+  doguctl state "starting"
+
+  local wait_timeout api_token
+  wait_timeout=${1}
+  api_token=${2}
+  for i in $(seq 1 "${wait_timeout}"); do
+    set +e
+    local requestExitCode
+    runHealthCheck "${api_token}"
+    requestExitCode=$?
+    set -e
+    if [[ "${requestExitCode}" -eq 0 ]]; then
+      echo "Reached SCM-Manager health endpoint successfully"
+      doguctl state "ready"
+      return 0
+    fi
+    if [[ "${i}" -eq ${wait_timeout} ]] ; then
+      echo "SCM-Manager did not get healthy within ${wait_timeout} seconds. Dogu exits now"
+      doguctl state "unhealthy"
+      exit 1
+    fi
+    sleep 1
+  done
 }
 
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  runHealthCheck
+  waitForHealth "${1}" "${2}"
   exit $?
 fi

resources/startup.sh

@@ -60,13 +60,17 @@ start_scm_server () {
       cp "${SCM_REQUIRED_PLUGINS_FOLDER}/${plugin}.smp" "${SCM_DATA}/plugins"
     fi
   done
+
+  /healthcheck.sh 600 "${API_TOKEN}" &
+
   /opt/scm-server/bin/scm-server
 }
 
 # Final startup and restart on exit code 42 (restart event)
 
 while start_scm_server ; scm_exit_code=$? ; [ $scm_exit_code -eq 42 ] ; do
   echo Got exit code $scm_exit_code -- restarting SCM-Manager
+  doguctl state restarting
 done
 
 exit $scm_exit_code