diff --git a/plugins/edge-ocp-rc/README.md b/plugins/edge-ocp-rc/README.md index a47a4cad..b6014219 100644 --- a/plugins/edge-ocp-rc/README.md +++ b/plugins/edge-ocp-rc/README.md @@ -43,11 +43,14 @@ export MY_APPCI_TOKEN="sha256~..." ## Quick start ```bash -# List available TNF jobs +# Fetch all job lists from Sippy (TNF + TNA × 4.22, 4.23, 5.0) +scripts/collect.sh + +# List available TNF jobs (all releases) scripts/launch.sh tnf --list -# Refresh job lists from Sippy -scripts/launch.sh tnf --refresh +# Refresh a single topology from Sippy +scripts/launch.sh tnf 5.0.0 --refresh # Launch regular TNF jobs against an RC scripts/launch.sh tnf 4.22.0-rc.0 --job all @@ -83,16 +86,15 @@ Version tags are expanded automatically: `4.22.0-rc.0` becomes `quay.io/openshif ```text edge-ocp-rc/ ├── jobs/ -│ ├── tnf.txt # Regular TNF jobs -│ ├── tnf-z-stream.txt # TNF z-stream upgrade jobs -│ ├── tnf-y-stream.txt # TNF y-stream upgrade jobs -│ ├── tna.txt # Regular TNA jobs -│ ├── tna-z-stream.txt # TNA z-stream upgrade jobs -│ ├── tna-y-stream.txt # TNA y-stream upgrade jobs -│ ├── sno.txt # Regular SNO jobs -│ ├── sno-z-stream.txt # SNO z-stream upgrade jobs -│ └── sno-y-stream.txt # SNO y-stream upgrade jobs +│ └── / # e.g., 4.22/, 4.23/, 5.0/ +│ ├── tnf.txt # Regular TNF jobs +│ ├── tnf-z-stream.txt # TNF z-stream upgrade jobs +│ ├── tnf-y-stream.txt # TNF y-stream upgrade jobs +│ ├── tna.txt # Regular TNA jobs +│ ├── tna-z-stream.txt # TNA z-stream upgrade jobs +│ └── tna-y-stream.txt # TNA y-stream upgrade jobs ├── scripts/ +│ ├── collect.sh # Fetch all topologies × releases from Sippy │ ├── launch.sh # Unified launcher (wraps gangway-cli) │ └── status.sh # Status, logs, and Jira reporting ├── skills/ @@ -136,7 +138,7 @@ Before launching, the script verifies: ### Job files and Sippy refresh -Each topology has up to three job files — one per job type: +Job files are organized by release under `jobs//`. Each topology has up to three files: | File | Type | Description | |------|------|-------------| @@ -146,11 +148,17 @@ Each topology has up to three job files — one per job type: Each file is a plain list of Prow job names, one per line. No prefixes. -Use `--refresh` to update all three from Sippy: +Use `collect.sh` to fetch all topologies and releases at once: + +```bash +scripts/collect.sh # Fetches TNF + TNA for 4.22, 4.23, 5.0 +``` + +Or refresh a single topology with `--refresh`: ```bash -scripts/launch.sh tnf --refresh # Fetches nightly jobs matching "two-node-fencing" -scripts/launch.sh tna --refresh # Fetches nightly jobs matching "two-node-arbiter" +scripts/launch.sh tnf 5.0.0 --refresh # Fetches nightly jobs matching "two-node-fencing" for 5.0 +scripts/launch.sh tna 4.22.0 --refresh # Fetches nightly jobs matching "two-node-arbiter" for 4.22 ``` Jobs are sorted into files automatically: @@ -173,7 +181,7 @@ scripts/launch.sh tnf 4.22.0-rc.0 --job all scripts/launch.sh tna 4.22.0-rc.0 --initial 4.21.0 --job all ``` -Jobs are launched sequentially with a 10-second delay between each to avoid rate limiting. +Jobs are launched sequentially with a 10-second delay between each to avoid rate limiting. Override with `DELAY=30 scripts/launch.sh ...` if Gangway is throttling. ## status.sh diff --git a/plugins/edge-ocp-rc/scripts/collect.sh b/plugins/edge-ocp-rc/scripts/collect.sh new file mode 100755 index 00000000..7d3ae191 --- /dev/null +++ b/plugins/edge-ocp-rc/scripts/collect.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SIPPY_API="https://sippy.dptools.openshift.org/api/jobs" + +RELEASES=("4.22" "4.23" "5.0") +TOPOLOGIES=("tnf" "tna" "sno") + +sippy_filter_for() { + case "$1" in + tnf) echo "two-node-fencing" ;; + tna) echo "two-node-arbiter" ;; + sno) echo "-4vcpu" ;; + *) echo "" ;; + esac +} + +GRAND_TOTAL=0 + +for release in "${RELEASES[@]}"; do + for topo in "${TOPOLOGIES[@]}"; do + search=$(sippy_filter_for "$topo") + [[ -z "$search" ]] && continue + + filter_json=$(printf '{"items":[{"columnField":"name","operatorValue":"contains","value":"%s"}],"linkOperator":"and"}' "$search") + encoded=$(printf '%s' "$filter_json" | jq -sRr '@uri') + + echo "Fetching ${topo^^} jobs from Sippy (release $release, filter: $search)..." + response=$(curl --fail --silent --show-error --connect-timeout 5 --max-time 30 \ + "${SIPPY_API}?release=${release}&filter=${encoded}&period=default&sortField=name&sort=asc") + + out_dir="$SCRIPT_DIR/jobs/${release}" + mkdir -p "$out_dir" + + job_file="$out_dir/${topo}.txt" + job_file_z="$out_dir/${topo}-z-stream.txt" + job_file_y="$out_dir/${topo}-y-stream.txt" + + tmp_file=$(mktemp) tmp_z=$(mktemp) tmp_y=$(mktemp) + trap 'rm -f "$tmp_file" "$tmp_z" "$tmp_y"' EXIT + + echo "$response" \ + | jq -r '.[] | select(.current_runs > 0) | .name' 2>/dev/null \ + | { grep '^periodic-ci-openshift-release-main-nightly' || true; } \ + | sort \ + | while IFS= read -r name; do + if [[ "$name" == *"upgrade-from-stable"* ]]; then + echo "$name" >> "$tmp_y" + elif [[ "$name" == *"-upgrade"* ]]; then + echo "$name" >> "$tmp_z" + else + echo "$name" >> "$tmp_file" + fi + done + + mv "$tmp_file" "$job_file"; mv "$tmp_z" "$job_file_z"; mv "$tmp_y" "$job_file_y" + trap - EXIT + + regular=$([[ -f "$job_file" ]] && wc -l < "$job_file" || echo 0) + z_count=$([[ -f "$job_file_z" ]] && wc -l < "$job_file_z" || echo 0) + y_count=$([[ -f "$job_file_y" ]] && wc -l < "$job_file_y" || echo 0) + total=$((regular + z_count + y_count)) + GRAND_TOTAL=$((GRAND_TOTAL + total)) + + echo " ${topo^^} ${release}: $total jobs ($regular regular, $z_count z-stream, $y_count y-stream)" + echo "" + done +done + +echo "=== Collected $GRAND_TOTAL jobs total ===" +echo " Files written to: $SCRIPT_DIR/jobs/{$(IFS=,; echo "${RELEASES[*]}")}/" diff --git a/plugins/edge-ocp-rc/scripts/launch.sh b/plugins/edge-ocp-rc/scripts/launch.sh index ed6715a3..9a1ea874 100755 --- a/plugins/edge-ocp-rc/scripts/launch.sh +++ b/plugins/edge-ocp-rc/scripts/launch.sh @@ -8,7 +8,7 @@ GCSWEB_BASE="gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs" SIPPY_API="https://sippy.dptools.openshift.org/api/jobs" IMAGE_BASE="quay.io/openshift-release-dev/ocp-release" ARCH="x86_64" -DELAY=10 +DELAY="${DELAY:-10}" # Sippy search terms per topology sippy_filter_for() { @@ -37,17 +37,19 @@ detect_release() { return fi - for f in "$JOB_FILE" "$JOB_FILE_Z" "$JOB_FILE_Y"; do - if [[ -f "$f" ]]; then - local from_jobs - from_jobs=$(awk -F'nightly-' '/nightly-/{split($2,a,"[^0-9.]"); print a[1]; exit}' "$f") - if [[ -n "$from_jobs" ]]; then - echo "$from_jobs" - return - fi - fi + local -a releases=() + for d in "$SCRIPT_DIR"/jobs/*/; do + [[ -d "$d" ]] && releases+=("$(basename "$d")") done + if [[ ${#releases[@]} -eq 1 ]]; then + echo "${releases[0]}" + return + elif [[ ${#releases[@]} -gt 1 ]]; then + echo "Error: multiple releases found (${releases[*]}). Provide an explicit version." >&2 + return 1 + fi + echo "" } @@ -77,6 +79,9 @@ usage() { echo " $0 tna 4.22.0-rc.0 --initial 4.21.0 --job all # launch all jobs including upgrades" echo " $0 tnf 4.22.0-rc.1 --relaunch-failed # re-launch failures from latest run" echo "" + echo "Job files are stored under jobs// (e.g., jobs/4.22/tnf.txt)." + echo "Use scripts/collect.sh to fetch all topologies and releases at once." + echo "" echo "When --initial is provided, z-stream and y-stream upgrade jobs are included." echo "Without --initial, upgrade jobs are skipped." echo "" @@ -145,7 +150,7 @@ if ! $DRY_RUN && ! $LIST_ONLY && ! $REFRESH; then fi fi -if [[ -n "$RELEASE_IMAGE" ]] && ! $REFRESH; then +if [[ -n "$RELEASE_IMAGE" ]] && ! $REFRESH && ! $LIST_ONLY; then if [[ "$RELEASE_IMAGE" == "${IMAGE_BASE}:"* ]]; then RELEASE_TAG="${RELEASE_IMAGE#*:}" echo "Verifying image tag: $RELEASE_TAG" @@ -165,10 +170,14 @@ if [[ -n "$RELEASE_IMAGE" ]] && ! $REFRESH; then fi fi +# Detect release and set up job file paths under jobs/{release}/ +OCP_RELEASE=$(detect_release "${RELEASE_IMAGE#*:}") + # Three job files per topology: regular, z-stream upgrades, y-stream upgrades -JOB_FILE="$SCRIPT_DIR/jobs/${TOPOLOGY}.txt" -JOB_FILE_Z="$SCRIPT_DIR/jobs/${TOPOLOGY}-z-stream.txt" -JOB_FILE_Y="$SCRIPT_DIR/jobs/${TOPOLOGY}-y-stream.txt" +JOB_DIR="$SCRIPT_DIR/jobs/${OCP_RELEASE}" +JOB_FILE="$JOB_DIR/${TOPOLOGY}.txt" +JOB_FILE_Z="$JOB_DIR/${TOPOLOGY}-z-stream.txt" +JOB_FILE_Y="$JOB_DIR/${TOPOLOGY}-y-stream.txt" if $REFRESH; then SEARCH_TERM=$(sippy_filter_for "$TOPOLOGY") @@ -178,13 +187,14 @@ if $REFRESH; then exit 0 fi - OCP_RELEASE=$(detect_release "${RELEASE_IMAGE#*:}") if [[ -z "$OCP_RELEASE" ]]; then echo "Error: cannot detect OCP release version." echo "Provide a version (e.g., ./launch.sh $TOPOLOGY 4.22.0-rc.0 --refresh)" exit 1 fi + mkdir -p "$JOB_DIR" + SIPPY_FILTER_JSON=$(printf '{"items":[{"columnField":"name","operatorValue":"contains","value":"%s"}],"linkOperator":"and"}' "$SEARCH_TERM") ENCODED_FILTER=$(printf '%s' "$SIPPY_FILTER_JSON" | jq -sRr '@uri') @@ -192,8 +202,8 @@ if $REFRESH; then SIPPY_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 5 --max-time 30 \ "${SIPPY_API}?release=${OCP_RELEASE}&filter=${ENCODED_FILTER}&period=default&sortField=name&sort=asc") - # Clear existing files before writing - rm -f "$JOB_FILE" "$JOB_FILE_Z" "$JOB_FILE_Y" + tmp_file=$(mktemp) tmp_z=$(mktemp) tmp_y=$(mktemp) + trap 'rm -f "$tmp_file" "$tmp_z" "$tmp_y"' EXIT # Sort jobs into separate files by type echo "$SIPPY_RESPONSE" \ @@ -202,14 +212,17 @@ if $REFRESH; then | sort \ | while IFS= read -r name; do if [[ "$name" == *"upgrade-from-stable"* ]]; then - echo "$name" >> "$JOB_FILE_Y" + echo "$name" >> "$tmp_y" elif [[ "$name" == *"-upgrade"* ]]; then - echo "$name" >> "$JOB_FILE_Z" + echo "$name" >> "$tmp_z" else - echo "$name" >> "$JOB_FILE" + echo "$name" >> "$tmp_file" fi done + mv "$tmp_file" "$JOB_FILE"; mv "$tmp_z" "$JOB_FILE_Z"; mv "$tmp_y" "$JOB_FILE_Y" + trap - EXIT + JOB_COUNT=$(wc -l < "$JOB_FILE" 2>/dev/null || echo 0) Z_COUNT=$(wc -l < "$JOB_FILE_Z" 2>/dev/null || echo 0) Y_COUNT=$(wc -l < "$JOB_FILE_Y" 2>/dev/null || echo 0) @@ -242,11 +255,11 @@ has_job_files() { return 1 } -if ! has_job_files; then - echo "Error: no job files for topology '$TOPOLOGY'" - echo "Expected: ${TOPOLOGY}.txt, ${TOPOLOGY}-z-stream.txt, or ${TOPOLOGY}-y-stream.txt in jobs/" +if ! $LIST_ONLY && ! has_job_files; then + echo "Error: no job files for topology '$TOPOLOGY' (release ${OCP_RELEASE:-unknown})" + echo "Expected files in jobs/${OCP_RELEASE:-}/" echo "" - echo "Run './launch.sh $TOPOLOGY --refresh' to fetch from Sippy" + echo "Run './launch.sh $TOPOLOGY --refresh' or './scripts/collect.sh' to fetch from Sippy" exit 1 fi @@ -261,15 +274,6 @@ if ! $LIST_ONLY && ! $REFRESH && ! $RELAUNCH_FAILED && [[ -n "$RELEASE_IMAGE" ]] exit 1 fi -if ! $LIST_ONLY && [[ -n "$RELEASE_IMAGE" ]]; then - REQUESTED_RELEASE=$(echo "${RELEASE_IMAGE#*:}" | grep -oE '^[0-9]+\.[0-9]+' || true) - JOBS_RELEASE=$(detect_release "") - if [[ -n "$REQUESTED_RELEASE" && -n "$JOBS_RELEASE" && "$REQUESTED_RELEASE" != "$JOBS_RELEASE" ]]; then - echo "Error: job files are for $JOBS_RELEASE but you requested $REQUESTED_RELEASE" - echo "Run './launch.sh $TOPOLOGY $REQUESTED_RELEASE --refresh' to update job files." - exit 1 - fi -fi # List jobs from a file with continuous numbering # Args: file, section_label, counter_var_name (LINE_NUM is global) @@ -288,13 +292,30 @@ list_file() { } if $LIST_ONLY; then - LINE_NUM=0 - echo "=== $TOPOLOGY jobs ===" - list_file "$JOB_FILE" "$TOPOLOGY" - list_file "$JOB_FILE_Z" "$TOPOLOGY z-stream upgrades (--initial required)" - list_file "$JOB_FILE_Y" "$TOPOLOGY y-stream upgrades (--initial required)" - echo "" - echo "Use --job all, --job , --job , or --job to select" + if [[ -n "$RELEASE_IMAGE" ]]; then + LINE_NUM=0 + echo "=== $TOPOLOGY jobs (release $OCP_RELEASE) ===" + list_file "$JOB_FILE" "$TOPOLOGY" + list_file "$JOB_FILE_Z" "$TOPOLOGY z-stream upgrades (--initial required)" + list_file "$JOB_FILE_Y" "$TOPOLOGY y-stream upgrades (--initial required)" + echo "" + echo "Use --job all, --job , --job , or --job to select" + else + for release_dir in "$SCRIPT_DIR"/jobs/*/; do + [[ ! -d "$release_dir" ]] && continue + rel=$(basename "$release_dir") + rel_file="$release_dir/${TOPOLOGY}.txt" + rel_file_z="$release_dir/${TOPOLOGY}-z-stream.txt" + rel_file_y="$release_dir/${TOPOLOGY}-y-stream.txt" + [[ ! -f "$rel_file" && ! -f "$rel_file_z" && ! -f "$rel_file_y" ]] && continue + LINE_NUM=0 + echo "=== $TOPOLOGY jobs (release $rel) ===" + list_file "$rel_file" "$TOPOLOGY" + list_file "$rel_file_z" "$TOPOLOGY z-stream upgrades (--initial required)" + list_file "$rel_file_y" "$TOPOLOGY y-stream upgrades (--initial required)" + echo "" + done + fi exit 0 fi @@ -458,8 +479,8 @@ if [[ -n "${INITIAL_IMAGE:-}" ]]; then launch_from_file "$JOB_FILE_Z" "$INITIAL_IMAGE" launch_from_file "$JOB_FILE_Y" "$INITIAL_IMAGE" else - Z_COUNT=$(wc -l < "$JOB_FILE_Z" 2>/dev/null || echo 0) - Y_COUNT=$(wc -l < "$JOB_FILE_Y" 2>/dev/null || echo 0) + Z_COUNT=$([[ -f "$JOB_FILE_Z" ]] && wc -l < "$JOB_FILE_Z" || echo 0) + Y_COUNT=$([[ -f "$JOB_FILE_Y" ]] && wc -l < "$JOB_FILE_Y" || echo 0) SKIP_TOTAL=$((Z_COUNT + Y_COUNT)) if [[ "$SKIP_TOTAL" -gt 0 ]]; then echo "Skipped $SKIP_TOTAL upgrade jobs (no --initial provided)" diff --git a/plugins/edge-ocp-rc/scripts/status.sh b/plugins/edge-ocp-rc/scripts/status.sh index 7d89becc..e811e159 100755 --- a/plugins/edge-ocp-rc/scripts/status.sh +++ b/plugins/edge-ocp-rc/scripts/status.sh @@ -162,10 +162,12 @@ trap 'rm -rf "$RESULTS_DIR"' EXIT # Look up a job's number matching launch.sh --list numbering job_num_lookup() { local topo="$1" target="$2" + local release + release=$(echo "$target" | awk -F'nightly-' '/nightly-/{split($2,a,"[^0-9.]"); print a[1]}') local n=0 - for f in "$SCRIPT_DIR/jobs/${topo}.txt" \ - "$SCRIPT_DIR/jobs/${topo}-z-stream.txt" \ - "$SCRIPT_DIR/jobs/${topo}-y-stream.txt"; do + for f in "$SCRIPT_DIR/jobs/${release}/${topo}.txt" \ + "$SCRIPT_DIR/jobs/${release}/${topo}-z-stream.txt" \ + "$SCRIPT_DIR/jobs/${release}/${topo}-y-stream.txt"; do [[ ! -f "$f" ]] && continue while IFS= read -r line; do [[ -z "$line" ]] && continue