add optional verification jobs after deploys

.github/workflows/app-build-and-deploy.yml

@@ -44,10 +44,18 @@ on:
         description: 'postfix to apply to the base name for the primary deploy site (e.g. -prod, -dev)'
         required: true
         type: string
+      primary-verification-url:
+        description: 'URL for primary deploy to an endpoint returning JSON that includes `version` and `sha`'
+        required: false
+        type: string
       secondary-azure-app-name-postfix:
         description: 'postfix to apply to the base name for a secondary deploy site (e.g. -prod-europe, do not specify if no secondary site)'
         type: string
         default: ''
+      secondary-verification-url:
+        description: 'URL for secondary deploy to an endpoint returning JSON that includes `version` and `sha`'
+        required: false
+        type: string
       docker-build-args:
         description: 'optionally pass in build args to the Docker build command (e.g. "MY_VAR=my_value")'
         required: false
@@ -56,7 +64,7 @@ on:
         description: 'optionally pass to publish image to docker-hub'
         required: false
         type: string
-
+      
 jobs:
   determine-trigger:
     name: Determine if this was triggered by a release or workflow_dispatch
@@ -129,6 +137,44 @@ jobs:
       azure-webapp-name: ${{ inputs.azure-app-base-name }}${{ inputs.azure-app-name-postfix }}
       image-name-with-tag: ${{ needs.build-and-publish-image.outputs.docker-image-name-with-tag }}
 
+  pause-between-primary-deploy-and-verification:
+    needs: deploy-primary-app-to-azure
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pause for 3 minutes
+        run: sleep 180  # Sleep for 180 seconds (3 minutes)
+
+  verify-primary-app:
+    name: Verify primary health endpoint
+    if: ${{inputs.primary-verification-url != '' }}
+    needs: [get-version, pause-between-primary-deploy-and-verification]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout this repo
+        uses: actions/[email protected]
+        with:
+          repository: 'clearlydefined/operations'
+          ref: 'elr/verify-health'
+          path: 'operations'
+      - name: Validate primary deploy
+        id: validate-primary
+        shell: bash
+        run: |
+          script_log=$(./operations/scripts/app-workflows/fetch-deploy-info.sh \
+            "${{ inputs.primary-verification-url }}") || (echo "$script_log" && exit 1)
+          echo -e "---- script log\n$script_log\n----"; \
+          response=$(echo "$script_log" | tail -n 1)
+          script_log=$(./operations/scripts/app-workflows/validate-deploy.sh \
+            "$response" \
+            "${{ needs.get-version.outputs.version }}" \
+            "${{ github.sha }}") || (echo "$script_log" && exit 1)
+          echo -e "---- script log\n$script_log\n----"; \
+          valid=$(echo "$script_log" | tail -n 1)
+          if [ "$valid" != "true" ]; then
+            echo "Validation of primary deploy failed"
+            exit 1
+          fi
+
   deploy-secondary-app-to-azure:
     name: Deploy to secondary Azure app
     if: ${{ inputs.secondary-azure-app-name-postfix != '' }}
@@ -143,3 +189,41 @@ jobs:
       deploy-env: ${{ inputs.deploy-env }}
       azure-webapp-name: ${{ inputs.azure-app-base-name }}${{ inputs.secondary-azure-app-name-postfix }}
       image-name-with-tag: ${{ needs.build-and-publish-image.outputs.docker-image-name-with-tag }}
+
+  pause-between-secondary-deploy-and-verification:
+    needs: deploy-secondary-app-to-azure
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pause for 3 minutes
+        run: sleep 180  # Sleep for 180 seconds (3 minutes)
+
+  verify-secondary-app:
+    name: Verify secondary health endpoint
+    if: ${{ inputs.secondary-azure-app-name-postfix != '' && inputs.secondary-verification-url != '' }}
+    needs: [get-version, pause-between-secondary-deploy-and-verification]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout this repo
+        uses: actions/[email protected]
+        with:
+          repository: 'clearlydefined/operations'
+          ref: 'elr/verify-health'
+          path: 'operations'
+      - name: Validate secondary deploy
+        id: validate-secondary
+        shell: bash
+        run: |
+          script_log=$(./operations/scripts/app-workflows/fetch-deploy-info.sh \
+            "${{ inputs.secondary-verification-url }}") || (echo "$script_log" && exit 1)
+          echo -e "---- script log\n$script_log\n----"; \
+          response=$(echo "$script_log" | tail -n 1)
+          script_log=$(./operations/scripts/app-workflows/validate-deploy.sh \
+            "$response" \
+            "${{ needs.get-version.outputs.version }}" \
+            "${{ github.sha }}") || (echo "$script_log" && exit 1)
+          echo -e "---- script log\n$script_log\n----"; \
+          valid=$(echo "$script_log" | tail -n 1)
+          if [ "$valid" != "true" ]; then
+            echo "Validation of secondary deploy failed"
+            exit 1
+          fi

scripts/app-workflows/fetch-deploy-info.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Inputs
+#   $1 - verification_url: url to the health endpoint that returns json with version and sha
+
+# Check if the correct number of arguments are provided
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <verification-url> <expected-version> <expected-sha>"
+  exit 1
+fi
+verification_url=$1
+
+response=$(curl -s "$verification_url")
+
+# Check if the curl command was successful
+if [ $? -ne 0 ]; then
+  echo "Failed to fetch the verification URL: $verification_url"
+  exit 1
+fi
+
+echo $response

scripts/app-workflows/validate-deploy.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# Inputs
+#   $1 - response: valid json that holds status, version, and sha
+#   $2 - expected_version: the version that was deployed
+#   $3 - expected_sha: the sha of the code being deployed
+
+# Check if the correct number of arguments are provided
+if [ "$#" -ne 3 ]; then
+  echo "Usage: $0 <response> <expected-version> <expected-sha>; $# parameters received, but 3 are expected"
+  exit 1
+fi
+
+response=$1
+expected_version=$2
+expected_sha=$3
+
+# Validate and reformat the response as a JSON string.
+# This is needed because shell scripts pass parameters
+# as simple strings.
+if [ -z "$response" ] || ! echo "$response" | jq empty > /dev/null 2>&1; then
+  echo "Error: Invalid JSON string"
+  exit 1
+else
+  # If valid, reformat the JSON string
+  response_json=$(echo "$response" | jq -c .)
+fi
+
+# Parse the JSON response
+status=$(echo "$response_json" | jq -r '.status')
+version=$(echo "$response_json" | jq -r '.version')
+sha=$(echo "$response_json" | jq -r '.sha')
+
+# Validate the response
+if [ "$status" != "OK" ]; then
+  echo "Validation failed: status is not OK"
+  echo "Expected: OK, Actual: $status"
+  exit 1
+fi
+
+if [ "$version" != "$expected_version" ]; then
+  echo "Validation failed: version mismatch"
+  echo "Expected: $expected_version, Actual: $version"
+  exit 1
+fi
+
+if [ "$sha" != "$expected_sha" ]; then
+  echo "Validation failed: sha mismatch"
+  echo "Expected: $expected_sha, Actual: $sha"
+  exit 1
+fi
+
+# If all validations pass
+echo "Validation successful"
+echo "true"

tests/scripts/app-workflows/fixtures/validate-good-response.json

@@ -0,0 +1 @@
+{ "status": "OK", "version": "v2.999.0", "sha": "1234567890ABCDEF" }

tests/scripts/app-workflows/fixtures/validate-invalid-json-response.json

@@ -0,0 +1 @@
+{ "status": "OK", "version": "undefined", "sha": "undefined"

tests/scripts/app-workflows/fixtures/validate-non-json-response.json

@@ -0,0 +1 @@
+"Invalid Response Format"

tests/scripts/app-workflows/fixtures/validate-sha-error.json

@@ -0,0 +1 @@
+{ "status": "OK", "version": "v2.999.0", "sha": "BAD_SHA" }

tests/scripts/app-workflows/fixtures/validate-status-error.json

@@ -0,0 +1 @@
+{ "status": "BAD_STATUS", "version": "v2.999.0", "sha": "1234567890ABCDEF" }

tests/scripts/app-workflows/fixtures/validate-version-error.json

@@ -0,0 +1 @@
+{ "status": "OK", "version": "v1.0.0-BAD", "sha": "1234567890ABCDEF" }

tests/scripts/app-workflows/test-fetch-deploy-info.bats

@@ -0,0 +1,9 @@
+#!/usr/bin/env bats
+
+load 'test_helpers'
+
+@test "Test bogus URL" {
+  run ./scripts/app-workflows/fetch-deploy-info.sh "http://localhost/bogus-url"
+  test_value 1 "$status"
+  test_value "Failed to fetch the verification URL: http://localhost/bogus-url" "${lines[0]}"
+}

tests/scripts/app-workflows/test-validate-deploy.bats

@@ -0,0 +1,53 @@
+#!/usr/bin/env bats
+
+load 'test_helpers'
+
+@test "Test bad status" {
+  fixture=$(load_json_fixture "validate-status-error.json")
+  run ./scripts/app-workflows/validate-deploy.sh $fixture "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Validation failed: status is not OK" "${lines[0]}"
+  test_value "Expected: OK, Actual: BAD_STATUS" "${lines[1]}"
+}
+
+@test "Test incorrect version" {
+  fixture=$(load_json_fixture "validate-version-error.json")
+  run ./scripts/app-workflows/validate-deploy.sh $fixture "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Validation failed: version mismatch" "${lines[0]}"
+  test_value "Expected: v2.999.0, Actual: v1.0.0-BAD" "${lines[1]}"
+}
+
+@test "Test incorrect sha" {
+  fixture=$(load_json_fixture "validate-sha-error.json")
+  run ./scripts/app-workflows/validate-deploy.sh $fixture "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Validation failed: sha mismatch" "${lines[0]}"
+  test_value "Expected: 1234567890ABCDEF, Actual: BAD_SHA" "${lines[1]}"
+}
+
+@test "Test valid response" {
+  fixture=$(load_json_fixture "validate-good-response.json")
+  run ./scripts/app-workflows/validate-deploy.sh $fixture "v2.999.0" "1234567890ABCDEF"
+  test_value 0 "$status"
+}
+
+@test "Test non-JSON response" {
+  fixture=$(load_json_fixture "validate-non-json-response.json")
+  run ./scripts/app-workflows/validate-deploy.sh '$fixture' "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Error: Invalid JSON string" "${lines[0]}"
+}
+
+@test "Test invalid JSON response" {
+  fixture=$(load_json_fixture "validate-invalid-json-response.json")
+  run ./scripts/app-workflows/validate-deploy.sh '$fixture' "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Error: Invalid JSON string" "${lines[0]}"
+}
+
+@test "Test empty response" {
+  run ./scripts/app-workflows/validate-deploy.sh "" "v2.999.0" "1234567890ABCDEF"
+  test_value 1 "$status"
+  test_value "Error: Invalid JSON string" "${lines[0]}"
+}

tests/scripts/app-workflows/test_helpers.bash

@@ -3,5 +3,34 @@
 test_value() {
   local expected="$1"
   local actual="$2"
-  diff <(echo "$actual") <(echo "$expected") || { echo -e "expected: $expected\nactual: '$actual'"; return 1; }
+  diff <(echo "$actual") <(echo "$expected") || { echo -e "expected: '$expected'\nactual:   '$actual'"; return 1; }
+}
+
+load_json_fixture() {
+  local fixture_filename="$1"
+  fixtures_dir="./tests/scripts/app-workflows/fixtures"
+  fixture_fullpath="$fixtures_dir/$fixture_filename"
+
+  run cat "$fixture_fullpath"
+  fixture=$output
+
+  # Validate and reformat the fixture as a JSON string
+  if echo "$fixture" | jq empty > /dev/null 2>&1; then
+    # If valid, reformat the JSON string
+    fixture_json=$(echo "$fixture" | jq -c .)
+  else
+    # Let bad data through for testing
+    fixture_json=$fixture
+    # exit 1
+  fi
+
+  echo "$fixture_json"
+}
+
+script_fullpath() {
+  local script_filename="$1"
+  scripts_dir="./scripts/app-workflows"
+  fullpath="$scripts_dir/$script_filename"
+
+  echo "$fullpath"
 }