diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml index a8e4a9187d0b1..6d27fe08a075b 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml @@ -57,7 +57,7 @@ tests: OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.16", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, - {"name": "rhacs-operator", "source": "redhat-operators", "channel": "stable", "install_namespace": "rhacs-operator", "target_namespaces": "rhacs-operator"}, + {"name": "rhacs-operator", "source": "redhat-operators", "channel": "stable", "install_namespace": "openshift-operators"}, {"name": "odf-operator", "source": "redhat-operators", "channel": "stable-4.21", "install_namespace": "openshift-storage", "target_namespaces": "openshift-storage"}, {"name": "quay-operator", "source": "redhat-operators", "channel": "stable-3.17", "install_namespace": "openshift-operators"} ] diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml index 6573ee6e9e007..54714608ca786 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml @@ -57,8 +57,10 @@ tests: OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, + {"name": "rhacs-operator", "source": "redhat-operators", "channel": "stable", "install_namespace": "openshift-operators"}, {"name": "quay-operator", "source": "redhat-operators", "channel": "stable-3.17", "install_namespace": "openshift-operators"} ] + OPP_OPERATORS: advanced-cluster-management,rhacs-operator,quay-operator ZONES_COUNT: "3" post: - ref: gather-aws-console diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/OWNERS b/ci-operator/step-registry/interop-tests/opp-quay-smoke/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh new file mode 100755 index 0000000000000..0fb6bc579dd09 --- /dev/null +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -0,0 +1,330 @@ +#!/bin/bash +set -euo pipefail +shopt -s inherit_errexit + +ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" +mkdir -p "${ARTIFACT_DIR}" +typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" +typeset imageTag="${BUILD_ID:-$(date +%s)}" + +typeset -A testStatus +typeset -A testDuration +typeset -A testFailureMsg +typeset -a allTests=( + "[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route" + "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" + "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" +) + +for t in "${allTests[@]}"; do + testStatus["${t}"]="skipped" + testDuration["${t}"]=0 + testFailureMsg["${t}"]="Test did not run" +done + +typeset -i suiteStart=0 +suiteStart=$(date +%s) + +function RecordResult () { + typeset name="${1}"; shift + typeset status="${1}"; shift + typeset msg="${1:-}"; shift || true + typeset dur="${1:-0}"; shift || true + testStatus["${name}"]="${status}" + testDuration["${name}"]="${dur}" + testFailureMsg["${name}"]="${msg}" +} + +# shellcheck disable=SC2329 +function GenerateJunit () { + typeset -i total=${#allTests[@]} + typeset -i failures=0 skipped=0 + typeset -i elapsed=$(( $(date +%s) - suiteStart )) + + for t in "${allTests[@]}"; do + [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) + [[ "${testStatus[${t}]}" == "skipped" ]] && skipped=$((skipped + 1)) + done + + cat > "${junitFile}" < + + +EOF + + for t in "${allTests[@]}"; do + typeset escaped_name + escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escaped_msg + escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + + if [[ "${testStatus[${t}]}" == "failed" ]]; then + echo " " >> "${junitFile}" + elif [[ "${testStatus[${t}]}" == "skipped" ]]; then + echo " " >> "${junitFile}" + else + echo " " >> "${junitFile}" + fi + done + + cat >> "${junitFile}" < + +EOF + cat "${junitFile}" +} + +trap GenerateJunit EXIT + +function DiscoverQuay () { + QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') + QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}') + QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') + QUAY_HOST="${QUAY_HOST#https://}" + export QUAY_NS QUAY_REGISTRY QUAY_HOST +} + +function GetQuayAuth () { + typeset configSecret + configSecret=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.spec.configBundleSecret}') + if [[ -z "${configSecret}" ]]; then + configSecret="${QUAY_REGISTRY}-config-bundle" + fi + + QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "") + if [[ -z "${QUAY_USER}" ]]; then + QUAY_USER="quayadmin" + fi + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "") + + if [[ -z "${QUAY_PASSWORD}" ]]; then + typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "") + fi + + if [[ -z "${QUAY_PASSWORD}" ]]; then + for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "") + [[ -n "${QUAY_PASSWORD}" ]] && break + done + fi + + export QUAY_USER QUAY_PASSWORD +} + +function PreflightCheck () { + if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then + echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 + return 1 + fi +} + +function CreateTestOrg () { + typeset token + token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ + -H "Content-Type: application/json" \ + -d "{\"user\":\"${QUAY_USER}\",\"pass\":\"${QUAY_PASSWORD}\"}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") + + if [[ -z "${token}" ]]; then + token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ + "https://${QUAY_HOST}/api/v1/user/" | \ + python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "") + fi + + QUAY_TOKEN="${token}" + export QUAY_TOKEN + + curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ + -H "Authorization: Bearer ${QUAY_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true +} + +################################################################################ +# Test Case 1: Push and pull image via Quay route +################################################################################ +function RunPushPull () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route" + typeset -i start elapsed + start=$(date +%s) + + DiscoverQuay + GetQuayAuth + PreflightCheck || { elapsed=$(( $(date +%s) - start )); RecordResult "${testName}" "failed" "Quay route not reachable" "${elapsed}"; return 1; } + CreateTestOrg + + typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" + typeset authFile="/tmp/quay-auth.json" + + cat > "${authFile}" <&1; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "skopeo push to Quay failed" "${elapsed}" + return 1 + fi + + if ! skopeo inspect --tls-verify=false \ + --authfile="${authFile}" \ + "docker://${pushTarget}" >/dev/null 2>&1; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" + return 1 + fi + + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "passed" "" "${elapsed}" + return 0 +} + +################################################################################ +# Test Case 2: Verify ODF PVC backing Quay storage +################################################################################ +function RunOdfPvcCheck () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" + typeset -i start elapsed + start=$(date +%s) + + typeset pvcCount + pvcCount=$(oc get pvc -n "${QUAY_NS}" -l app=quay -o json 2>/dev/null | python3 -c " +import sys, json +data = json.load(sys.stdin) +items = data.get('items', []) +print(len(items)) +" 2>/dev/null || echo "0") + + if [[ "${pvcCount}" == "0" ]]; then + pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " +import sys, json +data = json.load(sys.stdin) +items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] +print(len(items)) +" 2>/dev/null || echo "0") + fi + + if [[ "${pvcCount}" == "0" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "No Quay-related PVCs found in ${QUAY_NS}" "${elapsed}" + return 1 + fi + + typeset unboundPvcs + unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " +import sys, json +data = json.load(sys.stdin) +items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] +unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] +print(' '.join(unbound)) +" 2>/dev/null || echo "") + + if [[ -n "${unboundPvcs}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "Unbound PVCs: ${unboundPvcs}" "${elapsed}" + return 1 + fi + + typeset odfBacked + odfBacked=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " +import sys, json +data = json.load(sys.stdin) +items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] +sc_names = set(i['spec'].get('storageClassName','') for i in items) +odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) +print('true' if odf else 'false') +" 2>/dev/null || echo "false") + + if [[ "${odfBacked}" != "true" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "Quay PVCs not using ODF/Ceph storage class" "${elapsed}" + return 1 + fi + + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "passed" "" "${elapsed}" + return 0 +} + +################################################################################ +# Test Case 3: ACS scan of pushed Quay image +################################################################################ +function RunAcsScan () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" + typeset -i start elapsed + start=$(date +%s) + + typeset acsHost acsPassword + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + if [[ -z "${acsHost}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" + return 1 + fi + + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + if [[ -z "${acsPassword}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" + return 1 + fi + + typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" + typeset -i attempts=0 maxAttempts=20 + + while (( attempts < maxAttempts )); do + typeset scanResult + scanResult=$(curl -sk -u "admin:${acsPassword}" \ + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + + if echo "${scanResult}" | python3 -c " +import sys, json +data = json.load(sys.stdin) +images = data.get('images', []) +sys.exit(0 if len(images) > 0 else 1) +" 2>/dev/null; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "passed" "" "${elapsed}" + return 0 + fi + + attempts=$((attempts + 1)) + sleep 15 + done + + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "ACS did not detect pushed image within 5 minutes" "${elapsed}" + return 1 +} + +################################################################################ +# Main execution +################################################################################ + +function Main () { + typeset -i status=0 + RunPushPull || status=1 + RunOdfPvcCheck || status=1 + RunAcsScan || status=1 + + if [[ "${MAP_TESTS}" == "true" ]]; then + eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget --timeout=30 -qO-) || _fURL=(curl --connect-timeout 10 --max-time 30 -fsSL) + "${_fURL[@]}" \ + https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh + )" || true + if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ + ExitTrap--PostProcessPrep || true + fi + fi + + exit "${status}" +} + +Main "$@" diff --git a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh index f11b8d892696c..8c89d1e0cd153 100755 --- a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh +++ b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh @@ -1,8 +1,7 @@ #!/bin/bash -set -o nounset -set -o errexit -set -o pipefail +set -eux -o pipefail +shopt -s inherit_errexit OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" @@ -11,36 +10,42 @@ export XDG_RUNTIME_DIR="${HOME}/run" export REGISTRY_AUTH_PREFERENCE=podman mkdir -p "${XDG_RUNTIME_DIR}" -if ! command -v jq &>/dev/null; then - echo "jq not found; installing..." - dnf install -y -q jq 2>/dev/null || yum install -y -q jq 2>/dev/null || { - echo >&2 "ERROR: failed to install jq" - exit 1 - } +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + set +x + source "${SHARED_DIR}/proxy-conf.sh" + set -x fi REPORT_DIR="${ARTIFACT_DIR}/preflight" REPORT_FILE="${REPORT_DIR}/preflight-report.json" mkdir -p "${REPORT_DIR}" -CHECKS_FAILED=0 +typeset -i CHECKS_FAILED=0 +typeset -i EXIT_CODE=0 -DebugOnExit() { +function DebugOnExit () { if (( EXIT_CODE != 0 )); then - echo -e "\n### DEBUG: Pre-flight failure diagnostics ###\n" - echo -e "\n# ClusterVersion\n$(oc get clusterversion 2>/dev/null || echo 'unavailable')" - echo -e "\n# ClusterOperators\n$(oc get co 2>/dev/null || echo 'unavailable')" - echo -e "\n# MachineConfigPools\n$(oc get machineconfigpools 2>/dev/null || echo 'unavailable')" - echo -e "\n# Nodes\n$(oc get nodes 2>/dev/null || echo 'unavailable')" - echo -e "\n# OPP Operator CSVs\n$(oc get csv -A 2>/dev/null || echo 'unavailable')" + : "### DEBUG: Pre-flight failure diagnostics ###" + : "# ClusterVersion" + oc get clusterversion 2>/dev/null || : "unavailable" + : "# ClusterOperators" + oc get co 2>/dev/null || : "unavailable" + : "# MachineConfigPools" + oc get machineconfigpools 2>/dev/null || : "unavailable" + : "# Nodes" + oc get nodes 2>/dev/null || : "unavailable" + : "# OPP Operator CSVs" + oc get csv -A 2>/dev/null || : "unavailable" if [[ -f "${REPORT_FILE}" ]]; then - echo -e "\n# Pre-flight report:\n$(cat "${REPORT_FILE}")" + : "# Pre-flight report:" + cat "${REPORT_FILE}" fi fi true } -trap 'EXIT_CODE=$?; DebugOnExit' EXIT TERM +trap '{ EXIT_CODE=$?; DebugOnExit; true; }' EXIT +trap '{ EXIT_CODE=143; DebugOnExit; trap - EXIT; exit 143; }' TERM # ────────────────────────────────────────────────────────────────────── # Known removed / deprecated APIs per OCP minor version. @@ -48,14 +53,10 @@ trap 'EXIT_CODE=$?; DebugOnExit' EXIT TERM # was removed IN that minor version (i.e. no longer available). # Source: Kubernetes deprecation guide + OCP release notes. # ────────────────────────────────────────────────────────────────────── -declare -A REMOVED_APIS -# APIs removed in 4.12 (Kubernetes 1.25) +typeset -A REMOVED_APIS REMOVED_APIS["12"]="batch/v1beta1/CronJob policy/v1beta1/PodDisruptionBudget policy/v1beta1/PodSecurityPolicy discovery.k8s.io/v1beta1/EndpointSlice events.k8s.io/v1beta1/Event autoscaling/v2beta1/HorizontalPodAutoscaler" -# APIs removed in 4.14 (Kubernetes 1.27) REMOVED_APIS["14"]="storage.k8s.io/v1beta1/CSIStorageCapacity" -# APIs removed in 4.17 (Kubernetes 1.30) REMOVED_APIS["17"]="flowcontrol.apiserver.k8s.io/v1beta2/FlowSchema flowcontrol.apiserver.k8s.io/v1beta2/PriorityLevelConfiguration" -# APIs removed in 4.18 (Kubernetes 1.31) REMOVED_APIS["18"]="flowcontrol.apiserver.k8s.io/v1beta3/FlowSchema flowcontrol.apiserver.k8s.io/v1beta3/PriorityLevelConfiguration" # ────────────────────────────────────────────────────────────────────── @@ -63,7 +64,7 @@ REMOVED_APIS["18"]="flowcontrol.apiserver.k8s.io/v1beta3/FlowSchema flowcontrol. # Maps OCP minor version to minimum required operator major.minor. # Format: "operator_csv_prefix:min_major.min_minor" # ────────────────────────────────────────────────────────────────────── -declare -A OPP_COMPAT +typeset -A OPP_COMPAT OPP_COMPAT["4.14"]="advanced-cluster-management:2.9 rhacs-operator:4.3 odf-operator:4.14 quay-operator:3.10" OPP_COMPAT["4.15"]="advanced-cluster-management:2.10 rhacs-operator:4.4 odf-operator:4.15 quay-operator:3.11" OPP_COMPAT["4.16"]="advanced-cluster-management:2.11 rhacs-operator:4.5 odf-operator:4.16 quay-operator:3.12" @@ -72,13 +73,10 @@ OPP_COMPAT["4.18"]="advanced-cluster-management:2.13 rhacs-operator:4.7 odf-oper OPP_COMPAT["4.19"]="advanced-cluster-management:2.13 rhacs-operator:4.8 odf-operator:4.19 quay-operator:3.14" OPP_COMPAT["4.20"]="advanced-cluster-management:2.14 rhacs-operator:4.9 odf-operator:4.20 quay-operator:3.15" OPP_COMPAT["4.21"]="advanced-cluster-management:2.15 rhacs-operator:4.10 odf-operator:4.21 quay-operator:3.15" -OPP_COMPAT["4.22"]="advanced-cluster-management:2.16 rhacs-operator:4.11 odf-operator:4.22 quay-operator:3.16" +OPP_COMPAT["4.22"]="advanced-cluster-management:2.17 rhacs-operator:4.11 odf-operator:4.22 quay-operator:3.16" OPP_COMPAT["5.0"]="advanced-cluster-management:2.17 quay-operator:3.17" -# ────────────────────────────────────────────────────────────────────── -# Utility: append a check result to the JSON report -# ────────────────────────────────────────────────────────────────────── -InitReport() { +function InitReport () { cat > "${REPORT_FILE}" <<'EOFJSON' { "preflight_checks": [] @@ -87,21 +85,21 @@ EOFJSON true } -AppendCheck() { +function AppendCheck () { typeset checkName="${1}" checkStatus="${2}" checkDetails="${3}" - typeset tmpFile - tmpFile="$(mktemp)" - jq --arg n "${checkName}" --arg s "${checkStatus}" --arg d "${checkDetails}" \ - '.preflight_checks += [{"check": $n, "status": $s, "details": $d}]' \ - "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" + python3 -c " +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +data['preflight_checks'].append({'check': sys.argv[2], 'status': sys.argv[3], 'details': sys.argv[4]}) +with open(sys.argv[1], 'w') as f: + json.dump(data, f, indent=2) +" "${REPORT_FILE}" "${checkName}" "${checkStatus}" "${checkDetails}" true } -# ────────────────────────────────────────────────────────────────────── -# Check 1: API deprecation scan -# ────────────────────────────────────────────────────────────────────── -CheckApiDeprecations() { - echo "=== Check 1: API deprecation scan ===" +function CheckApiDeprecations () { + : "=== Check 1: API deprecation scan ===" typeset targetMinor="${1}" typeset ocpDisplay="${2:-4.${targetMinor}}" @@ -109,8 +107,8 @@ CheckApiDeprecations() { typeset flagged="" foundCount=0 typeset clusterApis - clusterApis="$(oc api-resources --no-headers 2>/dev/null)" || { - echo "WARNING: Failed to list API resources" + clusterApis="$(oc api-resources --no-headers)" || { + : "WARNING: Failed to list API resources" AppendCheck "api_deprecation_scan" "warn" "Could not list cluster API resources" return 0 } @@ -136,35 +134,33 @@ CheckApiDeprecations() { done if (( foundCount > 0 )); then - echo -e "WARNING: Found ${foundCount} deprecated API(s) still in use:\n${flagged}" + : "WARNING: Found ${foundCount} deprecated API(s) still in use" + echo -e "${flagged}" AppendCheck "api_deprecation_scan" "warn" "Found ${foundCount} deprecated API(s) in use: ${flagged}" else - echo "No deprecated APIs detected for target version ${ocpDisplay}" + : "No deprecated APIs detected for target version ${ocpDisplay}" AppendCheck "api_deprecation_scan" "pass" "No deprecated APIs detected for ${ocpDisplay}" fi true } -# ────────────────────────────────────────────────────────────────────── -# Check 2: OPP compatibility matrix -# ────────────────────────────────────────────────────────────────────── -CheckOppCompatibility() { - echo -e "\n=== Check 2: OPP operator compatibility matrix ===" +function CheckOppCompatibility () { + : "=== Check 2: OPP operator compatibility matrix ===" typeset ocpKey="${1}" typeset compatSpec="${OPP_COMPAT[${ocpKey}]:-}" typeset allCsvs typeset failed=0 - allCsvs="$(oc get csv -A --no-headers 2>/dev/null)" || { - echo >&2 "Failed to retrieve CSVs" + allCsvs="$(oc get csv -A --no-headers)" || { + : "Failed to retrieve CSVs" AppendCheck "opp_compatibility_matrix" "fail" "Could not list CSVs" (( CHECKS_FAILED += 1 )) return 0 } if [[ -z "${compatSpec}" ]]; then - echo "No compatibility matrix entry for OCP ${ocpKey}; skipping version check" + : "No compatibility matrix entry for OCP ${ocpKey}; skipping version check" AppendCheck "opp_compatibility_matrix" "skip" "No matrix entry for OCP ${ocpKey}" return 0 fi @@ -180,7 +176,7 @@ CheckOppCompatibility() { typeset csvLine csvName installedVersion csvLine="$(echo "${allCsvs}" | grep "${opPrefix}" | head -1)" || true if [[ -z "${csvLine}" ]]; then - echo >&2 "Operator not found: ${opPrefix}" + : "Operator not found: ${opPrefix}" details="${details}${opPrefix}: NOT INSTALLED; " (( failed += 1 )) continue @@ -189,7 +185,7 @@ CheckOppCompatibility() { csvName="$(echo "${csvLine}" | awk '{print $2}')" installedVersion="$(echo "${csvName}" | grep -oE '[0-9]+\.[0-9]+' | head -1)" || true if [[ -z "${installedVersion}" ]]; then - echo >&2 "Operator ${opPrefix}: could not parse version from CSV ${csvName}" + : "Operator ${opPrefix}: could not parse version from CSV ${csvName}" details="${details}${opPrefix}: version unparseable from ${csvName}; " (( failed += 1 )) continue @@ -200,88 +196,104 @@ CheckOppCompatibility() { instMinor="${installedVersion##*.}" if (( instMajor < minMajor || (instMajor == minMajor && instMinor < minMinor) )); then - echo >&2 "Operator ${opPrefix} version ${installedVersion} is below minimum ${minVersion} for OCP ${ocpKey}" + : "Operator ${opPrefix} version ${installedVersion} is below minimum ${minVersion} for OCP ${ocpKey}" details="${details}${opPrefix}: ${installedVersion} < ${minVersion} (INCOMPATIBLE); " (( failed += 1 )) else - echo "Operator ${opPrefix}: version ${installedVersion} >= ${minVersion} (OK)" + : "Operator ${opPrefix}: version ${installedVersion} >= ${minVersion} (OK)" details="${details}${opPrefix}: ${installedVersion} >= ${minVersion} (OK); " fi done if (( failed > 0 )); then - echo >&2 "${failed} operator(s) failed compatibility check" + : "${failed} operator(s) failed compatibility check" AppendCheck "opp_compatibility_matrix" "fail" "${details}" (( CHECKS_FAILED += 1 )) else - echo "All OPP operators are compatible with OCP ${ocpKey}" + : "All OPP operators are compatible with OCP ${ocpKey}" AppendCheck "opp_compatibility_matrix" "pass" "${details}" fi true } -# ────────────────────────────────────────────────────────────────────── -# Check 3: Cluster health baseline -# ────────────────────────────────────────────────────────────────────── -CheckClusterHealth() { - echo -e "\n=== Check 3: Cluster health baseline ===" +function CheckClusterHealth () { + : "=== Check 3: Cluster health baseline ===" typeset failed=0 details="" - echo "Checking node health..." + : "Checking node health..." typeset unreadyNodes - unreadyNodes="$(oc get node --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1}')" || true - if [[ -n "${unreadyNodes}" ]]; then - echo >&2 "Not-Ready nodes: ${unreadyNodes}" + if ! unreadyNodes="$(oc get node --no-headers | awk '$2 != "Ready" {print $1}')"; then + : "Failed to query nodes" + details="${details}nodes: query failed; " + (( failed += 1 )) + elif [[ -n "${unreadyNodes}" ]]; then + : "Not-Ready nodes: ${unreadyNodes}" details="${details}unready_nodes: ${unreadyNodes}; " (( failed += 1 )) else typeset nodeCount - nodeCount="$(oc get node --no-headers 2>/dev/null | wc -l)" - echo "All ${nodeCount} nodes Ready" + nodeCount="$(oc get node --no-headers | wc -l)" + : "All ${nodeCount} nodes Ready" details="${details}nodes: all ${nodeCount} ready; " fi - echo "Checking ClusterOperator health..." + : "Checking ClusterOperator health..." typeset unhealthyCo - unhealthyCo="$(oc get co --no-headers 2>/dev/null | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - if [[ -n "${unhealthyCo}" ]]; then - echo >&2 "Unhealthy ClusterOperators: ${unhealthyCo}" + if ! unhealthyCo="$(oc get co --no-headers | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')"; then + : "Failed to query ClusterOperators" + details="${details}cluster_operators: query failed; " + (( failed += 1 )) + elif [[ -n "${unhealthyCo}" ]]; then + : "Unhealthy ClusterOperators: ${unhealthyCo}" details="${details}unhealthy_co: ${unhealthyCo}; " (( failed += 1 )) else - echo "All ClusterOperators healthy" + : "All ClusterOperators healthy" details="${details}cluster_operators: all healthy; " fi - echo "Checking ClusterVersion conditions..." + : "Checking ClusterVersion conditions..." typeset avail progressing degraded - avail="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null)" || true - progressing="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null)" || true - degraded="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Degraded")].status}' 2>/dev/null)" || true + avail="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Available")].status}')" || true + progressing="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}')" || true + degraded="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Degraded")].status}')" || true if [[ "${avail}" != "True" || "${progressing}" != "False" || "${degraded}" != "False" ]]; then - echo >&2 "CVO health check failed: Available=${avail} Progressing=${progressing} Degraded=${degraded}" + : "CVO health check failed: Available=${avail} Progressing=${progressing} Degraded=${degraded}" details="${details}cvo: Available=${avail} Progressing=${progressing} Degraded=${degraded}; " (( failed += 1 )) else - echo "CVO: Available=True, Progressing=False, Degraded=False" + : "CVO: Available=True, Progressing=False, Degraded=False" details="${details}cvo: healthy; " fi - echo "Checking for firing alerts..." + : "Checking for firing alerts..." typeset firingAlerts="" - firingAlerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ - curl -s 'http://localhost:9090/api/v1/alerts' 2>/dev/null | \ - jq -r '.data.alerts[]? | select(.state=="firing") | select(.labels.alertname != "Watchdog") | select(.labels.alertname != "AlertmanagerReceiversNotConfigured") | .labels.alertname' 2>/dev/null | \ - sort -u)" || true - - if [[ -n "${firingAlerts}" ]]; then + if ! firingAlerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ + curl -s 'http://localhost:9090/api/v1/alerts' | \ + python3 -c " +import json, sys +data = json.load(sys.stdin) +if data.get('status') != 'success': + print('query returned non-success status', file=sys.stderr) + sys.exit(1) +alerts = data.get('data', {}).get('alerts', []) +names = sorted(set( + a['labels']['alertname'] for a in alerts + if a.get('state') == 'firing' + and a.get('labels', {}).get('alertname') not in ('Watchdog', 'AlertmanagerReceiversNotConfigured') +)) +print('\n'.join(names)) +")"; then + : "Alert query failed or unavailable" + details="${details}alerts: query failed; " + elif [[ -n "${firingAlerts}" ]]; then typeset alertCount alertCount="$(echo "${firingAlerts}" | wc -l)" - echo "WARNING: ${alertCount} alert(s) firing: ${firingAlerts}" + : "WARNING: ${alertCount} alert(s) firing: ${firingAlerts}" details="${details}firing_alerts: ${alertCount} (${firingAlerts}); " else - echo "No critical alerts firing" + : "No critical alerts firing" details="${details}alerts: none firing; " fi @@ -289,57 +301,63 @@ CheckClusterHealth() { oc get co -o json > "${REPORT_DIR}/co-baseline.json" 2>/dev/null || true if (( failed > 0 )); then - echo >&2 "Cluster health baseline: ${failed} issue(s) found" + : "Cluster health baseline: ${failed} issue(s) found" AppendCheck "cluster_health_baseline" "fail" "${details}" (( CHECKS_FAILED += 1 )) else - echo "Cluster health baseline: all checks passed" + : "Cluster health baseline: all checks passed" AppendCheck "cluster_health_baseline" "pass" "${details}" fi true } -# ────────────────────────────────────────────────────────────────────── -# Check 4: MachineConfigPool readiness -# ────────────────────────────────────────────────────────────────────── -CheckMcpReadiness() { - echo -e "\n=== Check 4: MachineConfigPool readiness ===" +function CheckMcpReadiness () { + : "=== Check 4: MachineConfigPool readiness ===" typeset failed=0 details="" - typeset mcpIssues - mcpIssues="$(oc get machineconfigpools --no-headers 2>/dev/null | \ - awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - - if [[ -n "${mcpIssues}" ]]; then - echo >&2 "Unhealthy MachineConfigPools: ${mcpIssues}" - details="unhealthy_mcps: ${mcpIssues}; " + typeset mcpRaw="" + if ! mcpRaw="$(oc get machineconfigpools --no-headers 2>&1)"; then + : "Failed to query MachineConfigPools" + details="machineconfigpools: query failed; " (( failed += 1 )) - - for mcp in ${mcpIssues}; do - echo -e "\n### MCP ${mcp} ###" - oc describe machineconfigpool "${mcp}" 2>/dev/null || true - done else - typeset mcpCount - mcpCount="$(oc get machineconfigpools --no-headers 2>/dev/null | wc -l)" - echo "All ${mcpCount} MachineConfigPools are updated and not degraded" - details="all ${mcpCount} MCPs healthy (Updated=True, Updating=False, Degraded=False); " + typeset mcpIssues="" + mcpIssues="$(echo "${mcpRaw}" | \ + awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true + + if [[ -n "${mcpIssues}" ]]; then + : "Unhealthy MachineConfigPools: ${mcpIssues}" + details="unhealthy_mcps: ${mcpIssues}; " + (( failed += 1 )) + + for mcp in ${mcpIssues}; do + : "### MCP ${mcp} ###" + oc describe machineconfigpool "${mcp}" || true + done + else + typeset mcpCount + mcpCount="$(echo "${mcpRaw}" | wc -l)" + : "All ${mcpCount} MachineConfigPools are updated and not degraded" + details="all ${mcpCount} MCPs healthy (Updated=True, Updating=False, Degraded=False); " + fi fi typeset mismatch="" - while IFS= read -r line; do - typeset mcpName ready desired - mcpName="$(echo "${line}" | awk '{print $1}')" - ready="$(echo "${line}" | awk '{print $7}')" - desired="$(echo "${line}" | awk '{print $6}')" - if [[ -n "${ready}" && -n "${desired}" && "${ready}" != "${desired}" ]]; then - mismatch="${mismatch}${mcpName} (ready=${ready}, desired=${desired}); " - fi - done < <(oc get machineconfigpools --no-headers 2>/dev/null || true) + if [[ -n "${mcpRaw:-}" ]]; then + while IFS= read -r line; do + typeset mcpName ready desired + mcpName="$(echo "${line}" | awk '{print $1}')" + ready="$(echo "${line}" | awk '{print $7}')" + desired="$(echo "${line}" | awk '{print $6}')" + if [[ -n "${ready}" && -n "${desired}" && "${ready}" != "${desired}" ]]; then + mismatch="${mismatch}${mcpName} (ready=${ready}, desired=${desired}); " + fi + done <<< "${mcpRaw}" + fi if [[ -n "${mismatch}" ]]; then - echo >&2 "MCP machine count mismatch: ${mismatch}" + : "MCP machine count mismatch: ${mismatch}" details="${details}machine_count_mismatch: ${mismatch}" (( failed += 1 )) fi @@ -347,69 +365,74 @@ CheckMcpReadiness() { oc get machineconfigpools -o json > "${REPORT_DIR}/mcp-baseline.json" 2>/dev/null || true if (( failed > 0 )); then - echo >&2 "MachineConfigPool readiness: ${failed} issue(s) found" + : "MachineConfigPool readiness: ${failed} issue(s) found" AppendCheck "mcp_readiness" "fail" "${details}" (( CHECKS_FAILED += 1 )) else - echo "MachineConfigPool readiness: all checks passed" + : "MachineConfigPool readiness: all checks passed" AppendCheck "mcp_readiness" "pass" "${details}" fi true } -# ────────────────────────────────────────────────────────────────────── -# Main -# ────────────────────────────────────────────────────────────────────── -Main() { +function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi typeset target="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" if [[ -z "${target}" ]]; then - echo >&2 "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot determine upgrade target" + : "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot determine upgrade target" exit 3 fi - echo "Target release image: ${target}" + : "Target release image: ${target}" + set +x KUBECONFIG="" oc registry login + set -x typeset targetVersion targetMajor targetMinor ocpXy - targetVersion="$(oc adm release info "${target}" --output=json | jq -r '.metadata.version')" + targetVersion="$(oc adm release info "${target}" -o jsonpath='{.metadata.version}')" targetMajor="$(echo "${targetVersion}" | cut -f1 -d.)" targetMinor="$(echo "${targetVersion}" | cut -f2 -d.)" ocpXy="${targetMajor}.${targetMinor}" - echo "Target OCP version: ${targetVersion} (${ocpXy})" + : "Target OCP version: ${targetVersion} (${ocpXy})" typeset sourceVersion sourceVersion="$(oc get clusterversion --no-headers | awk '{print $2}')" - echo "Source OCP version: ${sourceVersion}" + : "Source OCP version: ${sourceVersion}" - echo -e "\n=== Starting OPP pre-flight validation ===\n" + : "=== Starting OPP pre-flight validation ===" InitReport - typeset tmpFile - tmpFile="$(mktemp)" - jq --arg tv "${targetVersion}" --arg sv "${sourceVersion}" --arg ti "${target}" \ - '. + {"target_version": $tv, "source_version": $sv, "target_image": $ti, "timestamp": now | tostring}' \ - "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" + python3 -c " +import json, sys, time +with open(sys.argv[1]) as f: + data = json.load(f) +data['target_version'] = sys.argv[2] +data['source_version'] = sys.argv[3] +data['target_image'] = sys.argv[4] +data['timestamp'] = str(time.time()) +with open(sys.argv[1], 'w') as f: + json.dump(data, f, indent=2) +" "${REPORT_FILE}" "${targetVersion}" "${sourceVersion}" "${target}" CheckApiDeprecations "${targetMinor}" "${ocpXy}" CheckOppCompatibility "${ocpXy}" CheckClusterHealth CheckMcpReadiness - echo -e "\n=== Pre-flight summary ===" - jq '.' "${REPORT_FILE}" + : "=== Pre-flight summary ===" + python3 -m json.tool "${REPORT_FILE}" if (( CHECKS_FAILED > 0 )); then - echo >&2 "Pre-flight validation FAILED: ${CHECKS_FAILED} check(s) did not pass" - echo >&2 "Review ${REPORT_FILE} for details" + : "Pre-flight validation FAILED: ${CHECKS_FAILED} check(s) did not pass" + : "Review ${REPORT_FILE} for details" exit 3 fi - echo "Pre-flight validation PASSED: all checks succeeded" + : "Pre-flight validation PASSED: all checks succeeded" true } diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS b/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS b/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh new file mode 100755 index 0000000000000..f1f28f1c71cef --- /dev/null +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh @@ -0,0 +1,427 @@ +#!/bin/bash +set -euxo pipefail +shopt -s inherit_errexit + +ACM_TARGET_CHANNEL="${ACM_TARGET_CHANNEL:-}" +ACM_UPGRADE_TIMEOUT="${ACM_UPGRADE_TIMEOUT:-30m}" +ACM_SUBSCRIPTION_NAME="${ACM_SUBSCRIPTION_NAME:-advanced-cluster-management}" +ACM_SUBSCRIPTION_NAMESPACE="${ACM_SUBSCRIPTION_NAMESPACE:-open-cluster-management}" + +ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/artifacts}" +mkdir -p "${ARTIFACT_DIR}" + +function CollectDiagnostics () { + typeset artifactFile="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt" + { + printf '=== ACM Operator Upgrade Diagnostics ===\n\n' + printf '=== Subscription ===\n' + oc get subscription "${ACM_SUBSCRIPTION_NAME}" -n "${ACM_SUBSCRIPTION_NAMESPACE}" -o yaml 2>&1 || true + printf '\n=== CSVs in %s ===\n' "${ACM_SUBSCRIPTION_NAMESPACE}" + oc get csv -n "${ACM_SUBSCRIPTION_NAMESPACE}" 2>&1 || true + printf '\n=== InstallPlan ===\n' + oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" 2>&1 || true + printf '\n=== MCE CSVs ===\n' + oc get csv -n multicluster-engine 2>&1 || true + printf '\n=== Pods not Ready ===\n' + oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true + oc get pods -n multicluster-engine --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true + } > "${artifactFile}" + true +} + +trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT + +function GetCurrentCsv () { + oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.status.currentCSV}' || true +} + +function GetCsvPhase () { + typeset csvName="$1" + oc get csv "${csvName}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.status.phase}' || true +} + +function GetInstalledVersion () { + typeset csvName + csvName="$(GetCurrentCsv)" + if [[ -z "${csvName}" ]]; then + return 1 + fi + oc get csv "${csvName}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.spec.version}' || true +} + +function GetCurrentChannel () { + oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.spec.channel}' || true +} + +function ResolveTargetChannel () { + if [[ -n "${ACM_TARGET_CHANNEL}" ]]; then + echo "${ACM_TARGET_CHANNEL}" + return 0 + fi + + typeset currentChannel + currentChannel="$(GetCurrentChannel)" + if [[ -z "${currentChannel}" ]]; then + echo >&2 "ERROR: Cannot determine current subscription channel" + return 3 + fi + + typeset catalogNamespace + catalogNamespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.spec.sourceNamespace}' || true)" + + typeset packageName + packageName="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.spec.name}' || true)" + + typeset channels + channels="$(oc get packagemanifest "${packageName}" \ + -n "${catalogNamespace}" \ + -o jsonpath='{.status.channels[*].name}' || true)" + + if [[ -z "${channels}" ]]; then + echo >&2 "ERROR: No channels found in packagemanifest for ${packageName}" + return 3 + fi + + typeset currentVersion nextChannel="" + currentVersion="$(echo "${currentChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)" + + typeset -a channelList + read -ra channelList <<< "${channels}" + for ch in "${channelList[@]}"; do + typeset chVersion + chVersion="$(echo "${ch}" | grep -oE '[0-9]+\.[0-9]+' || true)" + if [[ -z "${chVersion}" ]]; then + continue + fi + if [[ -z "${currentVersion}" ]]; then + nextChannel="${ch}" + break + fi + typeset currentMajor currentMinor chMajor chMinor + currentMajor="${currentVersion%%.*}" + currentMinor="${currentVersion##*.}" + chMajor="${chVersion%%.*}" + chMinor="${chVersion##*.}" + + if (( chMajor > currentMajor )) || \ + (( chMajor == currentMajor && chMinor > currentMinor )); then + if [[ -z "${nextChannel}" ]]; then + nextChannel="${ch}" + else + typeset nextVersion nextMajor nextMinor + nextVersion="$(echo "${nextChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)" + nextMajor="${nextVersion%%.*}" + nextMinor="${nextVersion##*.}" + if (( chMajor < nextMajor )) || \ + (( chMajor == nextMajor && chMinor < nextMinor )); then + nextChannel="${ch}" + fi + fi + fi + done + + if [[ -z "${nextChannel}" ]]; then + echo >&2 "ERROR: No upgrade channel found newer than ${currentChannel}" + return 3 + fi + + echo "${nextChannel}" + true +} + +function WaitForCsvSucceeded () { + typeset previousCsv="$1" + typeset timeoutSeconds + timeoutSeconds="$(ParseTimeout "${ACM_UPGRADE_TIMEOUT}")" + typeset startTime elapsed newCsv phase + startTime="$(date +%s)" + + while true; do + elapsed="$(( $(date +%s) - startTime ))" + if (( elapsed > timeoutSeconds )); then + echo >&2 "ERROR: Timeout (${ACM_UPGRADE_TIMEOUT}) waiting for CSV upgrade" + return 2 + fi + + newCsv="$(GetCurrentCsv)" + if [[ -z "${newCsv}" || "${newCsv}" == "${previousCsv}" ]]; then + sleep 10 + continue + fi + + phase="$(GetCsvPhase "${newCsv}")" + echo " CSV: ${newCsv} Phase: ${phase} (${elapsed}s elapsed)" + + case "${phase}" in + Succeeded) + return 0 + ;; + Failed) + echo >&2 "ERROR: CSV ${newCsv} entered Failed phase" + return 1 + ;; + *) + sleep 15 + ;; + esac + done +} + +function ParseTimeout () { + typeset input="$1" + typeset minutes=0 seconds=0 + if [[ "${input}" =~ ^([0-9]+)m$ ]]; then + minutes="${BASH_REMATCH[1]}" + elif [[ "${input}" =~ ^([0-9]+)s$ ]]; then + seconds="${BASH_REMATCH[1]}" + elif [[ "${input}" =~ ^([0-9]+)h$ ]]; then + minutes="$(( BASH_REMATCH[1] * 60 ))" + elif [[ "${input}" =~ ^([0-9]+)$ ]]; then + minutes="${input}" + else + echo >&2 "WARNING: Unrecognized timeout format '${input}'; defaulting to 30m" + minutes=30 + fi + echo "$(( minutes * 60 + seconds ))" + true +} + +function ValidateMceUpgrade () { + echo "Validating MCE (MultiCluster Engine) upgrade..." + typeset mceCsv + mceCsv="$(oc get csv -n multicluster-engine \ + -o jsonpath='{.items[?(@.spec.displayName=="multicluster engine for Kubernetes")].metadata.name}' \ + || true)" + + if [[ -z "${mceCsv}" ]]; then + mceCsv="$(oc get csv -n multicluster-engine \ + -l operators.coreos.com/multicluster-engine.multicluster-engine= \ + -o jsonpath='{.items[0].metadata.name}' || true)" + fi + + if [[ -z "${mceCsv}" ]]; then + echo "WARNING: MCE CSV not found; skipping MCE validation" + return 0 + fi + + typeset mcePhase + mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \ + -o jsonpath='{.status.phase}' || true)" + + echo " MCE CSV: ${mceCsv} Phase: ${mcePhase}" + if [[ "${mcePhase}" != "Succeeded" ]]; then + echo "WARNING: MCE CSV phase is ${mcePhase}, not Succeeded" + typeset timeoutEnd + timeoutEnd="$(( $(date +%s) + 300 ))" + while (( $(date +%s) < timeoutEnd )); do + mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \ + -o jsonpath='{.status.phase}' || true)" + if [[ "${mcePhase}" == "Succeeded" ]]; then + echo " MCE CSV reached Succeeded phase" + return 0 + fi + sleep 15 + done + echo >&2 "ERROR: MCE CSV did not reach Succeeded within 5 minutes" + return 1 + fi + return 0 +} + +function ValidateHubHealth () { + echo "Validating ACM hub health post-upgrade..." + + typeset mchStatus + mchStatus="$(oc get multiclusterhub -A \ + -o jsonpath='{.items[0].status.phase}' || true)" + echo " MultiClusterHub phase: ${mchStatus}" + + if [[ "${mchStatus}" != "Running" ]]; then + echo " Waiting for MCH to reach Running phase (timeout: 5m)..." + typeset timeoutEnd + timeoutEnd="$(( $(date +%s) + 300 ))" + while (( $(date +%s) < timeoutEnd )); do + mchStatus="$(oc get multiclusterhub -A \ + -o jsonpath='{.items[0].status.phase}' || true)" + if [[ "${mchStatus}" == "Running" ]]; then + break + fi + sleep 15 + done + if [[ "${mchStatus}" != "Running" ]]; then + echo >&2 "ERROR: MultiClusterHub did not reach Running phase" + return 1 + fi + fi + + echo " Checking policy propagator..." + typeset propagatorReady="" + typeset -i propTimeout=300 + typeset -i propStart + propStart="$(date +%s)" + while (( $(date +%s) - propStart < propTimeout )); do + propagatorReady="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -l name=governance-policy-propagator \ + -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \ + || true)" + if [[ "${propagatorReady}" == "True" ]]; then + break + fi + sleep 15 + done + echo " Policy propagator ready: ${propagatorReady}" + if [[ "${propagatorReady}" != "True" ]]; then + echo >&2 "ERROR: Policy propagator not ready after ${propTimeout}s" + return 1 + fi + + echo " Checking managed clusters..." + typeset clusterOutput="" + clusterOutput="$(oc get managedclusters --no-headers || true)" + typeset -i clusterCount=0 + clusterCount="$(echo "${clusterOutput}" | grep -c . || true)" + typeset availableOutput="" + availableOutput="$(oc get managedclusters \ + -o jsonpath='{.items[?(@.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status=="True")].metadata.name}' \ + || true)" + typeset -i availableCount=0 + availableCount="$(echo "${availableOutput}" | wc -w)" + echo " Managed clusters: ${availableCount}/${clusterCount} available" + + if (( clusterCount > 0 && availableCount == 0 )); then + echo >&2 "WARNING: All ${clusterCount} managed cluster(s) are unavailable after upgrade" + elif (( clusterCount > 0 && availableCount < clusterCount )); then + echo >&2 "WARNING: ${availableCount}/${clusterCount} managed cluster(s) available (some may be reconciling post-upgrade)" + fi + + echo "ACM hub health validation complete" + return 0 +} + +# === Main === + +function Main () { + typeset currentCsv currentVersion currentChannel targetChannel + typeset prePatchPlan planPhase installPlan localApproval + typeset newCsv newVersion + + echo "=== ACM Operator Upgrade Step ===" + echo "Namespace: ${ACM_SUBSCRIPTION_NAMESPACE}" + echo "Subscription: ${ACM_SUBSCRIPTION_NAME}" + echo "Timeout: ${ACM_UPGRADE_TIMEOUT}" + + currentCsv="$(GetCurrentCsv)" + if [[ -z "${currentCsv}" ]]; then + echo >&2 "ERROR: No ACM subscription found or no currentCSV set" + exit 3 + fi + + currentVersion="$(GetInstalledVersion)" + currentChannel="$(GetCurrentChannel)" + echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChannel}" + + targetChannel="$(ResolveTargetChannel)" + echo "Target channel: ${targetChannel}" + + prePatchPlan="" + if ! prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null)"; then + echo "WARNING: Could not query current installPlanRef; treating as empty" + prePatchPlan="" + fi + echo "Pre-patch InstallPlan: ${prePatchPlan:-none}" + + if [[ "${targetChannel}" == "${currentChannel}" ]]; then + echo "Already on target channel ${targetChannel}; checking if upgrade is available..." + if [[ -z "${prePatchPlan}" ]]; then + echo "No pending upgrade on current channel; nothing to do" + exit 0 + fi + planPhase="$(oc get installplan "${prePatchPlan}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.status.phase}' || true)" + if [[ "${planPhase}" == "Complete" ]]; then + echo "InstallPlan ${prePatchPlan} already complete; no pending upgrade" + exit 0 + fi + installPlan="${prePatchPlan}" + else + echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}" + oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + --type merge \ + -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}" + + echo "Waiting for new InstallPlan (pre-patch ref: ${prePatchPlan:-none})..." + sleep 10 + + installPlan="" + for _ in {1..18}; do + installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.status.installPlanRef.name}' || true)" + if [[ -n "${installPlan}" && "${installPlan}" != "${prePatchPlan}" ]]; then + break + fi + installPlan="" + sleep 10 + done + + if [[ -z "${installPlan}" ]]; then + echo >&2 "ERROR: No new InstallPlan appeared after channel change (waited 3m)" + exit 2 + fi + fi + + echo "InstallPlan: ${installPlan}" + localApproval="$(oc get installplan "${installPlan}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + -o jsonpath='{.spec.approval}' || true)" + if [[ "${localApproval}" == "Manual" ]]; then + echo "Approving manual InstallPlan..." + oc patch installplan "${installPlan}" \ + -n "${ACM_SUBSCRIPTION_NAMESPACE}" \ + --type merge \ + -p '{"spec":{"approved":true}}' + fi + + echo "Waiting for ACM CSV to reach Succeeded phase..." + WaitForCsvSucceeded "${currentCsv}" + newCsv="$(GetCurrentCsv)" + newVersion="$(GetInstalledVersion)" + echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})" + + ValidateMceUpgrade + ValidateHubHealth + + { + printf '=== ACM Operator Upgrade Summary ===\n' + printf 'Previous: %s (%s)\n' "${currentVersion}" "${currentChannel}" + printf 'Current: %s (%s)\n' "${newVersion}" "${targetChannel}" + printf 'CSV: %s\n' "${newCsv}" + printf 'Status: SUCCESS\n' + } > "${ARTIFACT_DIR}/acm-upgrade-summary.txt" + + if [[ -n "${SHARED_DIR:-}" ]]; then + echo "${newVersion}" > "${SHARED_DIR}/acm-upgraded-version" + echo "${targetChannel}" > "${SHARED_DIR}/acm-upgraded-channel" + fi + + echo "=== ACM Operator Upgrade: SUCCESS ===" + true +} + +Main "$@" diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 61d6219238528..79d8ae4f7d5c8 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail +set -eux -o pipefail shopt -s inherit_errexit # NOTE: UPGRADE_TIMEOUT, POLL_INTERVAL, STALL_WINDOW, OPP_OPERATORS are set via step config YAML @@ -9,15 +9,17 @@ POLL_INTERVAL="${POLL_INTERVAL:-60}" STALL_WINDOW="${STALL_WINDOW:-10}" OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" -if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then - source "${SHARED_DIR}/proxy-conf.sh" -fi - export HOME="${HOME:-/tmp/home}" export XDG_RUNTIME_DIR="${HOME}/run" export REGISTRY_AUTH_PREFERENCE=podman mkdir -p "${XDG_RUNTIME_DIR}" +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + set +x + source "${SHARED_DIR}/proxy-conf.sh" + set -x +fi + typeset -i exitCode=0 typeset upgradeTarget="" typeset targetVersion="" @@ -26,7 +28,7 @@ typeset sourceVersion="" typeset -i sourceMinorVersion=0 typeset isForceUpdate="false" -DebugOnExit() { +function DebugOnExit () { if (( exitCode != 0 )); then : "### DEBUG: Upgrade failure diagnostics ###" if [[ -n "${targetMinorVersion:-}" ]] && (( targetMinorVersion >= 16 )); then @@ -39,43 +41,40 @@ DebugOnExit() { oc get machineconfig || : "unavailable" : "# Abnormal nodes" - oc get node -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status!="True")) | .metadata.name' | while read -r node; do + typeset node="" + for node in $(oc get node -o go-template='{{range .items}}{{$ready := ""}}{{range .status.conditions}}{{if eq .type "Ready"}}{{$ready = .status}}{{end}}{{end}}{{if ne $ready "True"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe node ${node} ###" oc describe node "${node}" || true - done || true + done : "# Abnormal ClusterOperators" - oc get co -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Available")).status != "True" or - (.status.conditions[] | select(.type=="Progressing")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name' | while read -r co; do + typeset co="" + for co in $(oc get co -o go-template='{{range .items}}{{$avail := ""}}{{$prog := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Available"}}{{$avail = .status}}{{end}}{{if eq .type "Progressing"}}{{$prog = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $avail "True") (ne $prog "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe co ${co} ###" oc describe co "${co}" || true - done || true + done : "# Abnormal MachineConfigPools" - oc get machineconfigpools -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Updated")).status != "True" or - (.status.conditions[] | select(.type=="Updating")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name' | while read -r mcp; do + typeset mcp="" + for mcp in $(oc get machineconfigpools -o go-template='{{range .items}}{{$upd := ""}}{{$upting := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Updated"}}{{$upd = .status}}{{end}}{{if eq .type "Updating"}}{{$upting = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $upd "True") (ne $upting "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe mcp ${mcp} ###" oc describe mcp "${mcp}" || true - done || true + done : "# OPP Operator CSVs" oc get csv -A || : "unavailable" fi + true } -trap 'exitCode=$?; DebugOnExit' EXIT TERM +trap '{ exitCode=$?; DebugOnExit; true; }' EXIT +trap '{ exitCode=143; DebugOnExit; trap - EXIT; exit 143; }' TERM set +x KUBECONFIG="" oc registry login set -x -ResolveTargetImage() { +function ResolveTargetImage () { typeset image="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" if [[ -z "${image}" ]]; then : "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot resolve upgrade target" @@ -83,16 +82,17 @@ ResolveTargetImage() { fi : "Target image: ${image}" upgradeTarget="${image}" + true } -CheckSigned() { +function CheckSigned () { typeset payload="${1:-}"; (($#)) && shift typeset digest="" algorithm="" hashValue="" typeset -i response=0 try=0 maxRetries=3 if [[ "${payload}" =~ "@sha256:" ]]; then digest="$(echo "${payload}" | cut -f2 -d@)" else - digest="$(oc image info "${payload}" -o json | jq -r '.digest')" + digest="$(oc image info "${payload}" -o jsonpath='{.digest}')" fi : "Image digest: ${digest}" algorithm="$(echo "${digest}" | cut -f1 -d:)" @@ -117,7 +117,7 @@ CheckSigned() { fi } -AdminAck() { +function AdminAck () { typeset -i srcMinor="${1:-0}"; (($#)) && shift typeset -i tgtMinor="${1:-0}"; (($#)) && shift if (( srcMinor == tgtMinor )) || (( srcMinor < 8 )); then @@ -126,9 +126,16 @@ AdminAck() { fi typeset gates="" - gates="$(oc -n openshift-config-managed get configmap admin-gates -o json | jq -r '.data')" || true - if [[ -z "${gates}" || "${gates}" == "null" ]]; then - : "No admin gates found" + if ! gates="$(oc -n openshift-config-managed get configmap admin-gates -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' 2>&1)"; then + if [[ "${gates}" == *"NotFound"* ]]; then + : "No admin-gates configmap; no acks required" + return 0 + fi + : "Failed to query admin-gates configmap: ${gates}" + return 1 + fi + if [[ -z "${gates}" ]]; then + : "admin-gates configmap exists but has no data keys" return 0 fi : "Admin gates: ${gates}" @@ -139,10 +146,8 @@ AdminAck() { fi : "Patching admin acks for 4.${srcMinor} -> 4.${tgtMinor}" - typeset ackKeys="" - ackKeys="$(echo "${gates}" | jq -r 'keys[]')" typeset ack="" - for ack in ${ackKeys}; do + for ack in ${gates}; do if [[ "${ack}" == *"ack-4.${srcMinor}"* ]]; then : "Applying ack: ${ack}" oc -n openshift-config patch configmap admin-acks \ @@ -167,7 +172,7 @@ AdminAck() { return 1 } -UpdateCcoAnnotation() { +function UpdateCcoAnnotation () { typeset srcVersion="${1:-}"; (($#)) && shift typeset tgtVersion="${1:-}"; (($#)) && shift typeset -i srcMinor=0 tgtMinor=0 @@ -210,7 +215,7 @@ UpdateCcoAnnotation() { return 1 } -InitiateUpgrade() { +function InitiateUpgrade () { typeset isForce="${1:-}"; (($#)) && shift : "Initiating upgrade to ${upgradeTarget}" : "Force flag: ${isForce}" @@ -225,18 +230,14 @@ InitiateUpgrade() { else : "CVO confirmed Progressing=True" fi + true } -MonitorUpgrade() { +function MonitorUpgrade () { typeset -i pollCount=0 typeset -i lastProgressChange=0 lastProgressChange=$(date +%s) - typeset statCmd="oc adm upgrade 2>&1 | grep -vE 'Upstream is unset|Upstream: https|available channels|No updates available|^$'" - if (( targetMinorVersion >= 16 )); then - statCmd="env OC_ENABLE_CMD_UPGRADE_STATUS=true oc adm upgrade status 2>&1 | grep -vE 'no token is currently in use|for additional description and links'" - fi - typeset prevStatus="" typeset snapshotDir="${ARTIFACT_DIR:-/tmp}/upgrade-progress" mkdir -p "${snapshotDir}" @@ -252,7 +253,11 @@ MonitorUpgrade() { (( pollCount += 1 )) typeset currentStatus="" - currentStatus="$(eval "${statCmd}")" || true + if (( targetMinorVersion >= 16 )); then + currentStatus="$(env OC_ENABLE_CMD_UPGRADE_STATUS=true oc adm upgrade status 2>&1 | grep -vE 'no token is currently in use|for additional description and links')" || currentStatus="" + else + currentStatus="$(oc adm upgrade 2>&1 | grep -vE 'Upstream is unset|Upstream: https|available channels|No updates available|^$')" || currentStatus="" + fi if [[ -n "${currentStatus}" && "${currentStatus}" != "${prevStatus}" ]]; then : "=== Upgrade Status $(date '+%T') ===" echo "${currentStatus}" @@ -293,13 +298,20 @@ MonitorUpgrade() { exit 2 } -StabilizeCluster() { +function StabilizeCluster () { : "Waiting for cluster stability (minimum-stable-period=5m, timeout=30m)" - oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m + if ! oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m; then + : "Cluster stabilization failed; gathering diagnostics" + oc get co || true + oc get nodes || true + oc get machineconfigpools || true + exit 1 + fi : "Cluster is stable" + true } -ValidatePlatformHealth() { +function ValidatePlatformHealth () { : "Validating platform health" typeset avail="" progressing="" degraded="" @@ -313,11 +325,7 @@ ValidatePlatformHealth() { : "CVO: Available=True, Progressing=False, Degraded=False" typeset unhealthyCo="" - unhealthyCo="$(oc get co -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Available")).status != "True" or - (.status.conditions[] | select(.type=="Progressing")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name')" + unhealthyCo="$(oc get co -o go-template='{{range .items}}{{$avail := ""}}{{$prog := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Available"}}{{$avail = .status}}{{end}}{{if eq .type "Progressing"}}{{$prog = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $avail "True") (ne $prog "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${unhealthyCo}" ]]; then : "Unhealthy ClusterOperators: ${unhealthyCo}" return 1 @@ -325,7 +333,7 @@ ValidatePlatformHealth() { : "All ClusterOperators healthy" typeset unreadyNodes="" - unreadyNodes="$(oc get node -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status!="True")) | .metadata.name')" + unreadyNodes="$(oc get node -o go-template='{{range .items}}{{$ready := ""}}{{range .status.conditions}}{{if eq .type "Ready"}}{{$ready = .status}}{{end}}{{end}}{{if ne $ready "True"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${unreadyNodes}" ]]; then : "Not-Ready nodes: ${unreadyNodes}" return 1 @@ -333,11 +341,7 @@ ValidatePlatformHealth() { : "All nodes Ready" typeset mcpIssues="" - mcpIssues="$(oc get machineconfigpools -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Updated")).status != "True" or - (.status.conditions[] | select(.type=="Updating")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name')" + mcpIssues="$(oc get machineconfigpools -o go-template='{{range .items}}{{$upd := ""}}{{$upting := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Updated"}}{{$upd = .status}}{{end}}{{if eq .type "Updating"}}{{$upting = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $upd "True") (ne $upting "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${mcpIssues}" ]]; then : "Unhealthy MachineConfigPools: ${mcpIssues}" return 1 @@ -346,7 +350,7 @@ ValidatePlatformHealth() { true } -ValidateOppOperators() { +function ValidateOppOperators () { : "Validating OPP operator health" typeset -a operatorsArr=() IFS=',' read -ra operatorsArr <<< "${OPP_OPERATORS}" @@ -356,14 +360,14 @@ ValidateOppOperators() { typeset allCsvsJson="" typeset -i failCount=0 - allCsvsJson="$(oc get csv -A -o json)" || { + allCsvsJson="$(oc get csv -A -o go-template='{{range .items}}{{.metadata.namespace}}{{"\t"}}{{.metadata.name}}{{"\t"}}{{with .status}}{{.phase}}{{end}}{{"\n"}}{{end}}')" || { : "Failed to retrieve CSVs" return 1 } - typeset phase="" + typeset phase="" op="" for op in "${operatorsArr[@]}"; do - phase="$(echo "${allCsvsJson}" | jq -r --arg op "${op}" '[.items[] | select(.metadata.name | startswith($op))][0].status.phase // empty')" || true + phase="$(awk -F'\t' -v op="${op}" 'index($2, op) == 1 {print $3; exit}' <<< "${allCsvsJson}")" if [[ -z "${phase}" ]]; then : "CSV not found for operator: ${op}" (( failCount += 1 )) @@ -380,16 +384,19 @@ ValidateOppOperators() { if (( failCount > 0 )); then : "${failCount} OPP operator(s) not healthy after upgrade" : "Full CSV listing:" - echo "${allCsvsJson}" | jq -r '.items[] | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.phase)"' + echo "${allCsvsJson}" return 1 fi : "Checking pod readiness for OPP operator namespaces" - typeset oppNamespaces="" - oppNamespaces="$(echo "${allCsvsJson}" | jq -r --arg ops "${OPP_OPERATORS}" '($ops | split(",")) as $opArr | [.items[] | select(.metadata.name as $n | $opArr | any(. as $op | $n | startswith($op))) | .metadata.namespace] | unique | .[]')" - typeset notReady="" ns="" - for ns in ${oppNamespaces}; do - notReady="$(oc get pods -n "${ns}" --no-headers | grep -v 'Completed' | grep -v 'Running' | grep -v 'Succeeded')" || true + typeset notReady="" ns="" podList="" + for ns in $(awk -F'\t' -v ops="${OPP_OPERATORS}" 'BEGIN{n=split(ops,arr,",")} {for(i=1;i<=n;i++) if(index($2,arr[i])==1){ns[$1]=1;break}} END{for(k in ns) print k}' <<< "${allCsvsJson}"); do + if ! podList="$(oc get pods -n "${ns}" --no-headers)"; then + : "Failed to list pods in ${ns}" + (( failCount += 1 )) + continue + fi + notReady="$(awk '!/Completed/ && !/Running/ && !/Succeeded/' <<< "${podList}")" if [[ -n "${notReady}" ]]; then : "WARNING: Non-running pods in ${ns}:" echo "${notReady}" @@ -408,21 +415,19 @@ ValidateOppOperators() { true } -Main() { +function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi ResolveTargetImage - targetVersion="$(oc adm release info "${upgradeTarget}" --output=json | jq -r '.metadata.version')" + targetVersion="$(oc adm release info "${upgradeTarget}" -o jsonpath='{.metadata.version}')" targetMinorVersion="$(echo "${targetVersion}" | cut -f2 -d.)" - export targetVersion targetMinorVersion : "Target release: ${targetVersion} (minor: ${targetMinorVersion})" sourceVersion="$(oc get clusterversion version -o jsonpath='{.status.desired.version}')" sourceMinorVersion="$(echo "${sourceVersion}" | cut -f2 -d.)" - export sourceVersion sourceMinorVersion : "Source release: ${sourceVersion} (minor: ${sourceMinorVersion})" isForceUpdate="false" diff --git a/ci-operator/step-registry/stackrox/opp-readiness/OWNERS b/ci-operator/step-registry/stackrox/opp-readiness/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/stackrox/opp-readiness/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh new file mode 100755 index 0000000000000..5a78660adc402 --- /dev/null +++ b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh @@ -0,0 +1,209 @@ +#!/bin/bash +set -eux -o pipefail +shopt -s inherit_errexit + +# --------------------------------------------------------------------------- +# ACS OPP Readiness Gate +# +# Verifies that ACS Central and SecuredCluster are operational before +# running SMOKE tests. Discovers namespaces dynamically via CRs. +# Writes credentials and connection details to $SHARED_DIR for +# downstream steps. +# +# Dependencies: oc, curl, python3 (all present in the `cli` image). +# --------------------------------------------------------------------------- + +if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then + export KUBECONFIG="${SHARED_DIR}/kubeconfig" +fi + +typeset -i pollInterval=30 +typeset -i timeout=600 +typeset -i elapsed=0 + +function WaitFor () { + typeset description="$1" + shift + typeset checkFn="$1" + shift + + elapsed=0 + echo "[readiness] Waiting for: ${description}" + while true; do + if "${checkFn}" "$@"; then + echo "[readiness] OK: ${description}" + return 0 + fi + elapsed=$((elapsed + pollInterval)) + if [[ ${elapsed} -ge ${timeout} ]]; then + echo "[readiness] TIMEOUT after ${timeout}s waiting for: ${description}" + return 1 + fi + echo "[readiness] ...retrying in ${pollInterval}s (${elapsed}/${timeout}s)" + sleep "${pollInterval}" + done + true +} + +function JsonLength () { + python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d.get('$1',[])))" +} + +# --------------------------------------------------------------------------- +# Namespace discovery via CRs (never hardcode) +# --------------------------------------------------------------------------- +function DiscoverCentralNs () { + centralNs="$(oc get centrals.platform.stackrox.io --all-namespaces \ + -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null)" \ + && [[ -n "${centralNs}" ]] +} + +function DiscoverScNs () { + scNs="$(oc get securedclusters.platform.stackrox.io --all-namespaces \ + -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null)" \ + && [[ -n "${scNs}" ]] +} + +typeset centralNs="" +typeset scNs="" + +WaitFor "Central CR namespace discovery" DiscoverCentralNs +echo "[readiness] Central namespace: ${centralNs}" + +WaitFor "SecuredCluster CR namespace discovery" DiscoverScNs +echo "[readiness] SecuredCluster namespace: ${scNs}" + +# --------------------------------------------------------------------------- +# Check 1: Central route exists +# --------------------------------------------------------------------------- +typeset centralUrl="" + +function CheckCentralRoute () { + oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null +} + +WaitFor "Central route" CheckCentralRoute + +set +x +centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')" +set -x +echo "[readiness] Central route discovered" + +# --------------------------------------------------------------------------- +# Extract ROX_ADMIN_PASSWORD before API checks +# --------------------------------------------------------------------------- +typeset roxAdminPassword="" +echo "[readiness] Extracting roxAdminPassword..." +set +x +roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \ + -o jsonpath='{.data.password}' | base64 -d)" +set -x + +if [[ -z "${roxAdminPassword}" ]]; then + echo "[readiness] FATAL: could not extract roxAdminPassword" + exit 1 +fi +echo "[readiness] roxAdminPassword extracted successfully" + +# --------------------------------------------------------------------------- +# Check 2: Central API health (authenticated v1/metadata) +# --------------------------------------------------------------------------- +function CheckCentralApi () { + set +x + typeset httpCode="" + httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ + -u "admin:${roxAdminPassword}" \ + "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; } + set -x + [[ "${httpCode}" == "200" ]] +} + +WaitFor "Central API health (v1/metadata)" CheckCentralApi + +# --------------------------------------------------------------------------- +# Check 3: At least 1 secured cluster connected +# --------------------------------------------------------------------------- +function CheckClustersConnected () { + set +x + typeset clusterCount="" + clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \ + "https://${centralUrl}/v1/clusters" --max-time 10 \ + | JsonLength clusters)" || { set -x; return 1; } + set -x + [[ "${clusterCount}" -ge 1 ]] +} + +WaitFor "secured cluster connected (v1/clusters)" CheckClustersConnected + +# --------------------------------------------------------------------------- +# Check 4: Sensor pods Running (detect OOMKilled) +# --------------------------------------------------------------------------- +function CheckSensorPods () { + typeset podCount="" + podCount="$(oc get pods -n "${scNs}" -l app=sensor \ + -o json 2>/dev/null | JsonLength items)" || return 1 + if [[ "${podCount}" -eq 0 ]]; then + echo "[readiness] no sensor pods found yet" + return 1 + fi + + typeset sensorJson="" + sensorJson="$(oc get pods -n "${scNs}" -l app=sensor -o json 2>/dev/null)" || return 1 + typeset oomContainers="" + if [[ -n "${sensorJson}" ]]; then + oomContainers="$(echo "${sensorJson}" | python3 -c " +import json,sys +d=json.load(sys.stdin) +for pod in d.get('items',[]): + for cs in pod.get('status',{}).get('containerStatuses',[]): + ls=cs.get('lastState',{}).get('terminated',{}) + if ls.get('reason')=='OOMKilled': + print(cs['name']) +")" + fi + if [[ -n "${oomContainers}" ]]; then + echo "[readiness] WARNING: OOMKilled detected in sensor containers: ${oomContainers}" + fi + + typeset podConditions="" + podConditions="$(oc get pods -n "${scNs}" -l app=sensor \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}={.status}{" "}{end}{"\n"}{end}' 2>/dev/null)" || return 1 + typeset notReady="" + notReady="$(echo "${podConditions}" | while IFS= read -r line; do + [[ -z "${line}" ]] && continue + if ! echo "${line}" | grep -q 'Ready=True'; then + echo "${line%% *}:NotReady" + fi + done)" + [[ -z "${notReady}" ]] +} + +WaitFor "sensor pods Running in ${scNs}" CheckSensorPods + +# --------------------------------------------------------------------------- +# Check 5: Default policies loaded (count > 80) +# --------------------------------------------------------------------------- +function CheckPoliciesLoaded () { + set +x + typeset policyCount="" + policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \ + "https://${centralUrl}/v1/policies?query=" --max-time 10 \ + | JsonLength policies)" || { set -x; return 1; } + set -x + echo "[readiness] policy count: ${policyCount}" + [[ "${policyCount}" -gt 80 ]] +} + +WaitFor "default policies loaded (>80)" CheckPoliciesLoaded + +echo "[readiness] Writing connection details to SHARED_DIR..." + +set +x +echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD" +echo "${centralUrl}" > "${SHARED_DIR}/CENTRAL_URL" +set -x + +echo "${centralNs}" > "${SHARED_DIR}/CENTRAL_NS" +echo "${scNs}" > "${SHARED_DIR}/SC_NS" + +echo "[readiness] All checks passed. ACS is ready for SMOKE tests." diff --git a/ci-operator/step-registry/stackrox/opp-smoke/OWNERS b/ci-operator/step-registry/stackrox/opp-smoke/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/stackrox/opp-smoke/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh new file mode 100755 index 0000000000000..3907a7d723dab --- /dev/null +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh @@ -0,0 +1,115 @@ +#!/bin/bash +set -eux -o pipefail +shopt -s inherit_errexit + +if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then + export KUBECONFIG="${SHARED_DIR}/kubeconfig" +fi + +echo "[smoke] Reading connection details from SHARED_DIR..." + +set +x +CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")" +ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")" +set -x + +echo "[smoke] Connection details loaded from SHARED_DIR" + +STACKROX_REF="${STACKROX_REF:-master}" +SCANNER_REF="${SCANNER_REF:-master}" + +echo "[smoke] Sparse-cloning stackrox/stackrox..." +cd /tmp +rm -rf stackrox scanner +git clone --depth 1 --filter=blob:none --sparse --branch "${STACKROX_REF}" \ + https://github.com/stackrox/stackrox.git stackrox +cd stackrox +git sparse-checkout set qa-tests-backend/ proto/ + +echo "[smoke] Fetching scanner protos..." +git clone --depth 1 --filter=blob:none --sparse --branch "${SCANNER_REF}" \ + https://github.com/stackrox/scanner.git /tmp/scanner +cd /tmp/scanner +git sparse-checkout set proto/scanner +cp -r proto/scanner /tmp/stackrox/qa-tests-backend/src/main/proto/scanner +chmod -R u+w /tmp/stackrox/qa-tests-backend/src/main/proto/scanner + +echo "[smoke] Materializing proto sources (replace symlinks with copies)..." +cd /tmp/stackrox/qa-tests-backend/src/main/proto +for link in api internalapi storage test tools; do + if [[ -L "${link}" ]]; then + target="$(readlink -f "${link}")" + rm "${link}" + cp -r "${target}" "${link}" + fi +done + +echo "[smoke] Patching DEFAULT_CLUSTER_NAME to 'local-cluster'..." +sed -i 's/DEFAULT_CLUSTER_NAME = "remote"/DEFAULT_CLUSTER_NAME = "local-cluster"/' \ + /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy +grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \ + /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \ + || { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; } + +set +x +export API_HOSTNAME="${CENTRAL_URL}" +export API_PORT="443" +export ROX_USERNAME="admin" +export ROX_ADMIN_PASSWORD +export CLUSTER="OPENSHIFT" +export CI="true" +export POD_SECURITY_POLICIES="false" +export TEST_TARGET="smoke-test" +REGISTRY_USERNAME="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/QUAY_RHACS_ENG_RO_USERNAME)" +export REGISTRY_USERNAME +REGISTRY_PASSWORD="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/QUAY_RHACS_ENG_RO_PASSWORD)" +export REGISTRY_PASSWORD +if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_CREDENTIALS_GCR_SCANNER_V2 ]]; then + GOOGLE_CREDENTIALS_GCR_SCANNER_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_CREDENTIALS_GCR_SCANNER_V2)" + export GOOGLE_CREDENTIALS_GCR_SCANNER_V2 +fi +if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 ]]; then + GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)" + export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 +fi +set -x + +cd /tmp/stackrox/qa-tests-backend + +cat > /tmp/fix-proto-deps.gradle <<'INIT' +allprojects { + afterEvaluate { + tasks.matching { it.name == 'compileGroovy' }.configureEach { + dependsOn tasks.matching { it.name == 'generateProto' } + } + } +} +INIT + +echo "[smoke] Running testSMOKE..." +typeset -i testExit=0 +./gradlew testSMOKE --no-daemon --init-script /tmp/fix-proto-deps.gradle \ + -Dorg.gradle.jvmargs="-Xmx2g" || testExit=$? + +echo "[smoke] Copying JUnit results to ARTIFACT_DIR..." +if [[ -d build/test-results/testSMOKE ]]; then + find build/test-results/testSMOKE -name '*.xml' -exec cp -v {} "${ARTIFACT_DIR}/" \; +fi + +if [[ -d build/reports/tests/testSMOKE ]]; then + mkdir -p "${ARTIFACT_DIR}/smoke-report" + find build/reports/tests/testSMOKE -mindepth 1 -maxdepth 1 \ + -exec cp -r {} "${ARTIFACT_DIR}/smoke-report/" \; +fi + +echo "[smoke] Test run finished with exit code: ${testExit}" +if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then + typeset total="" + total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)" + if [[ "${total}" -gt 0 ]]; then + echo "[smoke] Tests executed and results captured; treating as informational (exit 0)." + echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures." + exit 0 + fi +fi +exit "${testExit}"