From 38968151784414caf906535630290e89cd0bf767 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 11 Aug 2026 18:41:14 +0000 Subject: [PATCH] mpiit-data-router-reporter: add best_effort and retry logic The mpiit-data-router-reporter step currently has no retry logic and no best_effort flag, causing a 1-second Data Router 503 to fail an entire multi-hour CI job. This has generated 21 LPINTEROP tickets since Jan 2026 (~monthly Data Router outages). Changes: - Add best_effort: true to the ref YAML, matching medik8s-polarion-reporter - Add retry logic (5 attempts, 2-min intervals) modeled on send-results-to-reportportal - Align shell options with repo convention (set -euo pipefail, no -x) Jira: LPINTEROP-6976 --- .../mpiit-data-router-reporter-commands.sh | 54 ++++++++++++------- .../mpiit-data-router-reporter-ref.yaml | 1 + 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-commands.sh b/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-commands.sh index eed75bd2638cb..9735e91e81cf8 100644 --- a/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-commands.sh +++ b/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail # Legacy backward compatibility. TODO: To be removed once all caller are migrated. : "${DR__RP__CR_COMP_NAME:=${REPORTPORTAL_CMP}}" @@ -15,23 +15,37 @@ if [ -z "${OCP_VERSION}" ]; then fi fi -DATAROUTER_RESULTS="${SHARED_DIR}/*.xml" \ - REPORTPORTAL_LAUNCH_NAME="${DR__RP__CR_COMP_NAME}" \ - REPORTPORTAL_LAUNCH_ATTRIBUTES="$( - jq -nc \ - --arg jobName "${JOB_NAME}" \ - --arg buildID "${BUILD_ID}" \ - --arg ocpVer "${OCP_VERSION}" \ - --arg crCompName "${DR__RP__CR_COMP_NAME}" \ - --arg fipsEnabled "${FIPS_ENABLED}" \ - '[ - {key: "job_name", value: $jobName}, - {key: "build_id", value: $buildID}, - {key: "ocp_release", value: $ocpVer}, - {key: "ComponentReadiness_ComponentName", value: $crCompName}, - {key: "fips_enabled", value: $fipsEnabled} - ]' - )" \ - datarouter-openshift-ci +launch_attrs="$( + jq -nc \ + --arg jobName "${JOB_NAME}" \ + --arg buildID "${BUILD_ID}" \ + --arg ocpVer "${OCP_VERSION}" \ + --arg crCompName "${DR__RP__CR_COMP_NAME}" \ + --arg fipsEnabled "${FIPS_ENABLED}" \ + '[ + {key: "job_name", value: $jobName}, + {key: "build_id", value: $buildID}, + {key: "ocp_release", value: $ocpVer}, + {key: "ComponentReadiness_ComponentName", value: $crCompName}, + {key: "fips_enabled", value: $fipsEnabled} + ]' +)" -true \ No newline at end of file +MAX_RETRIES=5 +RETRY_INTERVAL=120 + +for (( attempt=1; attempt<=MAX_RETRIES; attempt++ )); do + if DATAROUTER_RESULTS="${SHARED_DIR}/*.xml" \ + REPORTPORTAL_LAUNCH_NAME="${DR__RP__CR_COMP_NAME}" \ + REPORTPORTAL_LAUNCH_ATTRIBUTES="${launch_attrs}" \ + datarouter-openshift-ci; then + echo "INFO: Data Router upload succeeded on attempt ${attempt}" + exit 0 + fi + if (( attempt < MAX_RETRIES )); then + echo "WARNING: Data Router upload failed (attempt ${attempt}/${MAX_RETRIES}), retrying in ${RETRY_INTERVAL}s..." + sleep "${RETRY_INTERVAL}" + fi +done +echo "ERROR: Data Router upload failed after ${MAX_RETRIES} attempts" +exit 1 diff --git a/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-ref.yaml b/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-ref.yaml index c083e56d0f336..508e1e57f3a0d 100644 --- a/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-ref.yaml +++ b/ci-operator/step-registry/mpiit/data-router-reporter/mpiit-data-router-reporter-ref.yaml @@ -7,6 +7,7 @@ ref: commands: mpiit-data-router-reporter-commands.sh grace_period: 10m timeout: 30m + best_effort: true resources: limits: cpu: 400m