From 5287536f2b5de6242a43b57e86ad7f4215c8cec9 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 14:29:21 -0400 Subject: [PATCH 01/11] payload-analysis: add adversarial false-revert evals --- .../annotations.yaml | 66 +++++++++++++++++++ .../input.yaml | 1 + .../annotations.yaml | 62 +++++++++++++++++ .../input.yaml | 1 + .../annotations.yaml | 60 +++++++++++++++++ .../input.yaml | 1 + 6 files changed, 191 insertions(+) create mode 100644 plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml create mode 100644 plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml create mode 100644 plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml create mode 100644 plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml create mode 100644 plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml create mode 100644 plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml new file mode 100644 index 000000000..2e002a39b --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml @@ -0,0 +1,66 @@ +expected_phase: Rejected +expected_failed_job_count: 4 +has_revert_candidates: true +force_accept_expected: false + +expected_candidates: + - pr_url: "https://github.com/openshift/oc/pull/2279" + component: "cli" + min_confidence: 85 + expected_confidence: 100 + description: "oc adm upgrade recommend output changed while the TechPreview serial tests still required the removed conditions preamble" + expected_failing_jobs: + - "aws-ovn-techpreview-serial-1of3" + - "aws-ovn-techpreview-serial-2of3" + +must_not_conclude: + - "HyperShift PR #8871 caused the etcd leader-change failures by rolling or restarting the target HostedCluster etcd StatefulSet" + - "The first payload where the aggregated HyperShift test name failed proves that PR #8871 introduced the underlying leader-election signature" + - "A single nearby HyperShift PR that touches etcd configuration is sufficient evidence for a revert" + +distractors: + - "PR #8871 adds TLS security-profile arguments to etcd and is temporally near a six-payload job-level failure streak" + - "The aggregated test reports approximately one etcd leader change in 7 of 10 runs, which superficially resembles a rollout" + - "The PromQL result groups by pod and job but omits namespace, making another HostedCluster's etcd-0 look like the target etcd-0" + +key_evidence: + - fact: >- + The target HostedCluster and its etcd-0 pod were newly created with the + TLS arguments present from first startup. The target pod had + restartCount=0, so there was no target StatefulSet rollout or restart + caused by PR #8871 during the test. + artifacts: + - "failed child pod artifact: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-aws-ovn-conformance/2080077700080340992/artifacts/e2e-aws-ovn-conformance/dump/artifacts/namespaces/clusters-f59953faec06f68d99b5/core/pods/etcd-0.yaml" + - "failed child etcd log: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-aws-ovn-conformance/2080077700080340992/artifacts/e2e-aws-ovn-conformance/dump/artifacts/namespaces/clusters-f59953faec06f68d99b5/core/pods/logs/etcd-0-etcd.log" + - fact: >- + The external-topology etcd monitor query selects every management-cluster + namespace matching clusters-.* and then aggregates away namespace. A + leader election from another concurrent HostedCluster can therefore be + attributed to the target pod name etcd-0. + artifacts: + - "openshift/origin test/extended/etcd/leader_changes.go" + - "5.0/nightly/5.0.0-0.nightly-2026-07-22-233611/jobs/blocking/aggregated-hypershift-ovn-conformance-5.0/junit/results.json" + - fact: >- + PR #2279 has direct code-to-assertion evidence in two independent + TechPreview serial shards, and its revert PR #2320 merged before this + payload completed. + artifacts: + - "5.0/nightly/5.0.0-0.nightly-2026-07-22-072716/cli, cli-artifacts, deployer, tools/prs/2279/code.diff" + - "5.0/nightly/5.0.0-0.nightly-2026-07-22-233611/jobs/blocking/aws-ovn-techpreview-serial-1of3/job.json" + - "5.0/nightly/5.0.0-0.nightly-2026-07-22-233611/jobs/blocking/aws-ovn-techpreview-serial-2of3/job.json" + +discriminating_signal: >- + A causal rollout requires a target-pod lifecycle change. The target etcd-0 + has no restart and was born with the new arguments, while the metric query + crosses HostedCluster namespaces and discards namespace identity. Keep the + independently proven oc #2279 recommendation, but reject HyperShift #8871. + +notes: > + Mixed true-positive and false-positive case. The original Payload Agent + correctly recommended openshift/oc#2279 at 100 for two TechPreview serial + failures, but also recommended openshift/hypershift#8871 at 95 for the + aggregated HyperShift failure. The latter attribution is false: it substitutes + job-level temporal onset for the underlying signature onset and never proves + the claimed target etcd rollout. The true defect is cross-tenant evidence + contamination in the Origin etcd monitor query. Source Payload Agent analysis: + https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-main-claude-payload-agent/2080077739515187200/artifacts/claude-payload-agent/openshift-claude-payload-agent/artifacts/payload-analysis-5.0.0-0.nightly-2026-07-22-233611-summary.html diff --git a/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml new file mode 100644 index 000000000..76309d8c3 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.nightly-2026-07-22-233611" diff --git a/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml new file mode 100644 index 000000000..7cb77b7e0 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml @@ -0,0 +1,62 @@ +expected_phase: Rejected +expected_failed_job_count: 3 +has_revert_candidates: false +force_accept_expected: false + +expected_candidates: [] + +must_not_conclude: + - "openshift/api PR #2920 caused ClusterOperator availability regressions and should be reverted" + - "openshift/api PR #2923 caused ClusterOperator availability regressions and should be reverted" + - "A job-level F F F or F F F F streak establishes that the same operator degradation signature began in the earliest failed payload" + - "A test failure should be classified as infrastructure merely because the operator under observation was unhealthy" + +distractors: + - "Both API PRs merged near the job-level failure streak and change API declarations that appear in payload metadata" + - "The monitor report prints 3-to-59-minute waiting spans, apparently exceeding its three-minute failure threshold" + - "The failing AWS and Azure upgrade jobs mention multiple ClusterOperators, encouraging broad component attribution" + - "The same test name fails across consecutive payloads even though the atomic interval signature and affected operator differ" + +key_evidence: + - fact: >- + The Origin monitor test computes duration from the earliest interval start + to the latest interval end. It counts healthy gaps between disjoint waiting + episodes as waiting time instead of summing the union of actual intervals. + artifacts: + - "openshift/origin pkg/monitortests/clusterversionoperator/legacycvomonitests/operators.go" + - "aggregated AWS and Azure e2e-timelines_spyglass JSON artifacts referenced by the snapshot" + - fact: >- + Across 13 raw AWS and Azure failures, actual summed waiting time is only + 13-150 seconds and no individual episode exceeds 150 seconds, but the test + reports false spans of 3m27s-58m53s. One representative AWS result combines + a 15-second interval at 11:48:50-11:49:05 with a later point at 12:34:04 + and reports 45m14s. + artifacts: + - "periodic-ci-openshift-release-main-ci-5.0-upgrade-from-stable-4.22-e2e-aws-ovn-upgrade/2080964774497095680/artifacts/e2e-aws-ovn-upgrade/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_20260725-113839.json" + - "periodic-ci-openshift-release-main-ci-5.0-upgrade-from-stable-4.22-e2e-azure-ovn-upgrade/2080964703600775168/artifacts/e2e-azure-ovn-upgrade/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_20260725-113832.json" + - fact: >- + PR #2920 is a feature-gate promotion and PR #2923 adds a DevPreview BGP + API for bare metal; neither explains default AWS and Azure operator + intervals. Experimental revert jobs continued to fail frequently for both + PRs, directly falsifying the recommendations. + artifacts: + - "5.0/ci/5.0.0-0.ci-2026-07-24-135543/cluster-config-api/prs/2920/code.diff" + - "5.0/ci/5.0.0-0.ci-2026-07-24-075533/cluster-config-api/prs/2923/code.diff" + +discriminating_signal: >- + Reconstruct and merge the actual intervals. Healthy gaps must not contribute + to waiting duration. Once the interval union is used, the alleged three-minute + operator regression disappears, and neither API PR has an executed causal path + in the failing default AWS/Azure configurations. + +notes: > + False-revert case targeting the difference between job onset, test-name onset, + and minimal failure-signature onset. The true cause is a deterministic Origin + test-framework arithmetic defect, not an infrastructure incident and not a + regression from either payload API PR. The Payload Agent recommended + openshift/api#2920 and #2923 at 90. Experimental revert PRs #2953 and #2954 did + not restore the jobs: #2953 yielded AWS 0/5 and Azure 1/5 passes; #2954 yielded + AWS 3/5 and Azure 2/5 passes. Correct output has no payload revert candidate, + explains the interval-union defect, and separates unrelated HyperShift noise. + Source Payload Agent analysis: + https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-main-claude-payload-agent/2080964771632386048/artifacts/claude-payload-agent/openshift-claude-payload-agent/artifacts/payload-analysis-5.0.0-0.ci-2026-07-25-102741-summary.html diff --git a/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml new file mode 100644 index 000000000..a16311ada --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.ci-2026-07-25-102741" diff --git a/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml new file mode 100644 index 000000000..45ef9f7c7 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml @@ -0,0 +1,60 @@ +expected_phase: Rejected +expected_failed_job_count: 2 +has_revert_candidates: false +force_accept_expected: false + +expected_candidates: [] + +must_not_conclude: + - "openshift/ovn-kubernetes PR #3298 caused the GCP external-container panic and should be reverted" + - "openshift/cloud-provider-azure PR #164 introduced the empty VirtualMachine update and should be reverted" + - "The informing 0/10 OVN helper test is the blocking cause of the aggregated GCP job" + - "Every error in a compound failure must be folded into one broad signature and attributed to a nearby PR" + +distractors: + - "OVN PR #3298 is the only nearby ovn-kubernetes change and the artifacts include a panic in an OVN external-container helper" + - "Azure PR #164 migrated SDK call sites and the terminal disk-attach trace contains an Azure LocationRequired error" + - "The GCP job contains both pre-upgrade network reachability loss and cleanup invariant failures, which can look like independent product regressions" + - "The AKS job has a four-payload job streak even though the causal chain and terminal symptom vary across constituent runs" + +key_evidence: + - fact: >- + PR #3298 changed neither the panicking external-container adapter, the test + call site, nor the annotation controlling lifecycle. The helper failure is + informing and therefore cannot reject the payload. + artifacts: + - "5.0/nightly/5.0.0-0.nightly-2026-07-28-081944/ovn-kubernetes, ovn-kubernetes-microshift/prs/3298/code.diff" + - "5.0/nightly/5.0.0-0.nightly-2026-07-29-055948/jobs/blocking/aggregated-gcp-ovn-rt-upgrade-5.0-major/junit/results.json" + - fact: >- + Five distinct GCP VIPs were externally unreachable for roughly 701-713 + seconds before upgrade. Setup then panicked, cleanup was skipped, and leaked + resources triggered downstream invariants. These are ordered roles in a + causal chain, not proof that the informing test or PR #3298 was the trigger. + artifacts: + - "5.0/nightly/5.0.0-0.nightly-2026-07-29-055948/jobs/blocking/aggregated-gcp-ovn-rt-upgrade-5.0-major/junit/junit-aggregated.xml" + - fact: >- + PR #164 did not change the empty VirtualMachine update that produced + LocationRequired; base and head contain the same call. The AKS chain begins + with Azure DNS 429 throttling, delayed ignition DNS, and + OSProvisioningTimedOut before the CSI attach path amplifies the failure. + artifacts: + - "5.0/nightly/5.0.0-0.nightly-2026-07-28-081944/azure-cloud-controller-manager, azure-cloud-node-manager/prs/164/code.diff" + - "5.0/nightly/5.0.0-0.nightly-2026-07-29-055948/jobs/blocking/aggregated-hypershift-aks-conformance-5.0/junit/junit-aggregated.xml" + +discriminating_signal: >- + Decompose each job into minimal atomic signatures and preserve causal order. + For GCP, external VIP reachability precedes setup panic and cleanup fallout. + For AKS, DNS throttling precedes provisioning timeout and the CSI empty-VM PUT. + A candidate must change the executed trigger path; neither #3298 nor #164 does. + +notes: > + Compound infrastructure and resilience case. The Payload Agent recommended + openshift/ovn-kubernetes#3298 and openshift/cloud-provider-azure#164 at 90. + Both are false causal attributions. The correct analysis can still identify + product resilience and diagnostic defects: the external-container adapter + should return a capability error rather than panic, cleanup must survive setup + failure, HyperShift should gate on DNS readiness, and the Azure cloud provider + must not send an empty VM update after a provisioning failure. Those follow-up + fixes do not make the nearby payload PRs valid reverts. Source Payload Agent + analysis: + https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-main-claude-payload-agent/2082347184044380160/artifacts/claude-payload-agent/openshift-claude-payload-agent/artifacts/payload-analysis-5.0.0-0.nightly-2026-07-29-055948-summary.html diff --git a/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml new file mode 100644 index 000000000..3e243eb4d --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml @@ -0,0 +1 @@ +payload_tag: "5.0.0-0.nightly-2026-07-29-055948" From 311f48113d7fbfaf547ee2d32cf4e7e18e681193 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 14:30:34 -0400 Subject: [PATCH 02/11] evals: hide scenario details from case names --- plugins/ci/evals/cases/payload-analysis/README.md | 11 +++++++++++ .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 7 files changed, 11 insertions(+) create mode 100644 plugins/ci/evals/cases/payload-analysis/README.md rename plugins/ci/evals/cases/payload-analysis/{case-018-5.0-nightly-cross-tenant-etcd-evidence => case-018}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-018-5.0-nightly-cross-tenant-etcd-evidence => case-018}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-019-5.0-ci-interval-union-false-streak => case-019}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-019-5.0-ci-interval-union-false-streak => case-019}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-020-5.0-nightly-compound-infra-causal-chain => case-020}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-020-5.0-nightly-compound-infra-causal-chain => case-020}/input.yaml (100%) diff --git a/plugins/ci/evals/cases/payload-analysis/README.md b/plugins/ci/evals/cases/payload-analysis/README.md new file mode 100644 index 000000000..ea40db5b5 --- /dev/null +++ b/plugins/ci/evals/cases/payload-analysis/README.md @@ -0,0 +1,11 @@ +# Payload-analysis eval case index + +Case directory names are intentionally opaque because the eval harness exposes +them to the model under test. Keep scenario descriptions here rather than in +directory names. + +| Case | Scenario | +| --- | --- | +| `case-018` | Mixed true/false attribution: valid `oc#2279` recommendation alongside cross-tenant etcd evidence contamination falsely attributed to `hypershift#8871` | +| `case-019` | Disjoint ClusterOperator intervals counted as one continuous streak, producing false `api#2920` and `api#2923` revert recommendations | +| `case-020` | Compound GCP and Azure infrastructure causal chains falsely attributed to nearby `ovn-kubernetes#3298` and `cloud-provider-azure#164` changes | diff --git a/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-018/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-018-5.0-nightly-cross-tenant-etcd-evidence/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-018/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-019/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-019-5.0-ci-interval-union-false-streak/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-019/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-020/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-020-5.0-nightly-compound-infra-causal-chain/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-020/input.yaml From b3970b6f10e67628da1113264bad9304f2bf61ed Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 14:37:52 -0400 Subject: [PATCH 03/11] evals: make all payload case names opaque --- .../ci/evals/cases/payload-analysis/README.md | 18 ++++++++++++++++++ .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 .../annotations.yaml | 0 .../input.yaml | 0 29 files changed, 18 insertions(+) rename plugins/ci/evals/cases/payload-analysis/{case-001-4.22-nightly-ci-config-and-product-regression => case-001}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-001-4.22-nightly-ci-config-and-product-regression => case-001}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-002-5.0-nightly-all-new-failures => case-002}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-002-5.0-nightly-all-new-failures => case-002}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-003-5.0-ci-cno-networkpolicy-revert => case-003}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-003-5.0-ci-cno-networkpolicy-revert => case-003}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-004-4.22-two-reverts => case-004}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-004-4.22-two-reverts => case-004}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-005-4.22-ci-cco-revert => case-005}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-005-4.22-ci-cco-revert => case-005}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-006-4.22-ci-hypershift-revert => case-006}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-006-4.22-ci-hypershift-revert => case-006}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-007-4.22-olm-revert => case-007}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-007-4.22-olm-revert => case-007}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-008-5.0-nightly-cmo-monitoring-revert => case-008}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-008-5.0-nightly-cmo-monitoring-revert => case-008}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-009-5.0-ci-hypershift-builder-fp => case-009}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-009-5.0-ci-hypershift-builder-fp => case-009}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-010-4.18-rejected-multiple-failures => case-010}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-010-4.18-rejected-multiple-failures => case-010}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-011-5.0-ci-infra-only-no-candidates => case-011}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-011-5.0-ci-infra-only-no-candidates => case-011}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-012-4.20-rejected-streak => case-012}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-012-4.20-rejected-streak => case-012}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-013-4.20-accepted-with-failures => case-013}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-013-4.20-accepted-with-failures => case-013}/input.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-014-5.0-nightly-step-registry-infra-change => case-014}/annotations.yaml (100%) rename plugins/ci/evals/cases/payload-analysis/{case-014-5.0-nightly-step-registry-infra-change => case-014}/input.yaml (100%) diff --git a/plugins/ci/evals/cases/payload-analysis/README.md b/plugins/ci/evals/cases/payload-analysis/README.md index ea40db5b5..aff556e39 100644 --- a/plugins/ci/evals/cases/payload-analysis/README.md +++ b/plugins/ci/evals/cases/payload-analysis/README.md @@ -6,6 +6,24 @@ directory names. | Case | Scenario | | --- | --- | +| `case-001` | OCP 4.22 nightly with both a CI-configuration regression and a product regression | +| `case-002` | OCP 5.0 nightly whose blocking failures are all new | +| `case-003` | OCP 5.0 CI regression requiring a cluster-network-operator NetworkPolicy revert | +| `case-004` | OCP 4.22 payload with two independent valid revert candidates | +| `case-005` | OCP 4.22 CI cloud-credential-operator revert | +| `case-006` | OCP 4.22 CI HyperShift revert | +| `case-007` | OCP 4.22 OLM revert | +| `case-008` | OCP 5.0 nightly cluster-monitoring-operator monitoring regression | +| `case-009` | OCP 5.0 CI HyperShift builder false-positive trap | +| `case-010` | OCP 4.18 rejected payload with multiple independent failures | +| `case-011` | OCP 5.0 CI infrastructure-only rejection with no valid payload candidate | +| `case-012` | OCP 4.20 rejected payload with a consecutive failure streak | +| `case-013` | OCP 4.20 accepted payload that still contains failures | +| `case-014` | OCP 5.0 nightly failure caused by a step-registry infrastructure change | | `case-018` | Mixed true/false attribution: valid `oc#2279` recommendation alongside cross-tenant etcd evidence contamination falsely attributed to `hypershift#8871` | | `case-019` | Disjoint ClusterOperator intervals counted as one continuous streak, producing false `api#2920` and `api#2923` revert recommendations | | `case-020` | Compound GCP and Azure infrastructure causal chains falsely attributed to nearby `ovn-kubernetes#3298` and `cloud-provider-azure#164` changes | + +`case-015` through `case-017` are reserved by the eval additions in +[PR #626](https://github.com/openshift-eng/ai-helpers/pull/626). That PR should +use the same opaque directory naming before merge. diff --git a/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-001/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-001-4.22-nightly-ci-config-and-product-regression/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-001/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-002/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-002-5.0-nightly-all-new-failures/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-002/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-003-5.0-ci-cno-networkpolicy-revert/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-003/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-004/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-004-4.22-two-reverts/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-004/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-005/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-005-4.22-ci-cco-revert/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-005/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-006/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-006-4.22-ci-hypershift-revert/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-006/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-007-4.22-olm-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-007-4.22-olm-revert/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-007-4.22-olm-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-007/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-007-4.22-olm-revert/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-007/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-008/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-008-5.0-nightly-cmo-monitoring-revert/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-008/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-009/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-009-5.0-ci-hypershift-builder-fp/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-009/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-010/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-010-4.18-rejected-multiple-failures/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-010/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-011/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-011-5.0-ci-infra-only-no-candidates/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-011/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-012/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-012-4.20-rejected-streak/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-012/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-013/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-013-4.20-accepted-with-failures/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-013/input.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-014-5.0-nightly-step-registry-infra-change/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-014-5.0-nightly-step-registry-infra-change/annotations.yaml rename to plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml diff --git a/plugins/ci/evals/cases/payload-analysis/case-014-5.0-nightly-step-registry-infra-change/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-014/input.yaml similarity index 100% rename from plugins/ci/evals/cases/payload-analysis/case-014-5.0-nightly-step-registry-infra-change/input.yaml rename to plugins/ci/evals/cases/payload-analysis/case-014/input.yaml From 9a9f61c8210e72c6709ae510ac5656b56b746b1c Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 14:45:23 -0400 Subject: [PATCH 04/11] ci: bump plugin version for eval changes --- .claude-plugin/marketplace.json | 2 +- docs/index.html | 2 +- plugins/ci/.claude-plugin/plugin.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index fe0c68fd7..90668c162 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -88,7 +88,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", -"version": "0.0.77", + "version": "0.0.78", "category": "ci", "keywords": [ "prow", diff --git a/docs/index.html b/docs/index.html index 65ac4543a..ccf18fab3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -541,7 +541,7 @@

ai-helpers

{ "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", -"version": "0.0.77", + "version": "0.0.78", "has_readme": true, "commands": [ { diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index 77d1d9bdd..d96954e32 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.77", + "version": "0.0.78", "author": { "name": "github.com/openshift-eng" } From 5a0510484b6c7e0ef7d0d188a97e21f46e78478c Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 15:17:19 -0400 Subject: [PATCH 05/11] Add point-in-time payload eval guardrails --- .../case-001/annotations.yaml | 1 + .../payload-analysis/case-001/input.yaml | 1 + .../case-002/annotations.yaml | 1 + .../payload-analysis/case-002/input.yaml | 1 + .../case-003/annotations.yaml | 1 + .../payload-analysis/case-003/input.yaml | 1 + .../case-004/annotations.yaml | 1 + .../payload-analysis/case-004/input.yaml | 1 + .../case-005/annotations.yaml | 1 + .../payload-analysis/case-005/input.yaml | 1 + .../case-006/annotations.yaml | 1 + .../payload-analysis/case-006/input.yaml | 1 + .../case-007/annotations.yaml | 1 + .../payload-analysis/case-007/input.yaml | 1 + .../case-008/annotations.yaml | 1 + .../payload-analysis/case-008/input.yaml | 1 + .../case-009/annotations.yaml | 1 + .../payload-analysis/case-009/input.yaml | 1 + .../case-010/annotations.yaml | 1 + .../payload-analysis/case-010/input.yaml | 1 + .../case-011/annotations.yaml | 1 + .../payload-analysis/case-011/input.yaml | 1 + .../case-012/annotations.yaml | 1 + .../payload-analysis/case-012/input.yaml | 1 + .../case-013/annotations.yaml | 1 + .../payload-analysis/case-013/input.yaml | 1 + .../case-014/annotations.yaml | 1 + .../payload-analysis/case-014/input.yaml | 1 + .../case-018/annotations.yaml | 1 + .../payload-analysis/case-018/input.yaml | 1 + .../case-019/annotations.yaml | 1 + .../payload-analysis/case-019/input.yaml | 1 + .../case-020/annotations.yaml | 1 + .../payload-analysis/case-020/input.yaml | 1 + plugins/ci/evals/eval-payload-analysis.yaml | 157 +++++++++++++++++- 35 files changed, 185 insertions(+), 6 deletions(-) diff --git a/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml index 4513e9e6c..86cea25b3 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-03-27T04:56:37Z" expected_candidates: - pr_url: "https://github.com/openshift/cluster-authentication-operator/pull/839" diff --git a/plugins/ci/evals/cases/payload-analysis/case-001/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-001/input.yaml index e5b0b6ba0..c60ae9b41 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-001/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-001/input.yaml @@ -1 +1,2 @@ payload_tag: "4.22.0-0.nightly-2026-03-26-231124" +payload_completed_at: "2026-03-27T04:56:37Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml index 5f0d10f7e..48050954a 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 13 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-05-30T12:55:22Z" expected_candidates: - pr_url: "https://github.com/openshift/origin/pull/31006" diff --git a/plugins/ci/evals/cases/payload-analysis/case-002/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-002/input.yaml index a11a378e0..43fd6305f 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-002/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-002/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.nightly-2026-05-30-072431" +payload_completed_at: "2026-05-30T12:55:22Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml index f28f57c48..9b573e0a9 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 2 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-05-07T20:17:14Z" expected_candidates: - pr_url: "https://github.com/openshift/cluster-network-operator/pull/2959" diff --git a/plugins/ci/evals/cases/payload-analysis/case-003/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/input.yaml index d172d9ea0..0b65b6873 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-003/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-003/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.ci-2026-05-07-142711" +payload_completed_at: "2026-05-07T20:17:14Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml index d7d3a762d..cf0283d82 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-004/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-03-20T12:21:33Z" expected_candidates: - pr_url: "https://github.com/openshift/cluster-ingress-operator/pull/1354" diff --git a/plugins/ci/evals/cases/payload-analysis/case-004/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-004/input.yaml index 4c52ac2be..624346cd1 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-004/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-004/input.yaml @@ -1 +1,2 @@ payload_tag: "4.22.0-0.nightly-2026-03-20-053450" +payload_completed_at: "2026-03-20T12:21:33Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml index a3c538455..020f6c9ed 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-005/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-03-31T10:56:34Z" expected_candidates: - pr_url: "https://github.com/openshift/cloud-credential-operator/pull/978" diff --git a/plugins/ci/evals/cases/payload-analysis/case-005/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-005/input.yaml index 7d4ad2650..a74fb5fca 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-005/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-005/input.yaml @@ -1 +1,2 @@ payload_tag: "4.22.0-0.ci-2026-03-31-050515" +payload_completed_at: "2026-03-31T10:56:34Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml index ce90ad2ee..f13b21c47 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-006/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 2 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-03-31T22:55:24Z" expected_candidates: - pr_url: "https://github.com/openshift/hypershift/pull/7790" diff --git a/plugins/ci/evals/cases/payload-analysis/case-006/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-006/input.yaml index 764b8e3ba..673783280 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-006/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-006/input.yaml @@ -1 +1,2 @@ payload_tag: "4.22.0-0.ci-2026-03-31-170515" +payload_completed_at: "2026-03-31T22:55:24Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml index 991760bce..ed6df7de7 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-007/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-03-19T01:17:31Z" expected_candidates: - pr_url: "https://github.com/openshift/operator-framework-olm/pull/1256" diff --git a/plugins/ci/evals/cases/payload-analysis/case-007/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-007/input.yaml index 687db7221..a6041b45e 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-007/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-007/input.yaml @@ -1 +1,2 @@ payload_tag: "4.22.0-0.nightly-2026-03-18-161724" +payload_completed_at: "2026-03-19T01:17:31Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml index 1741ec525..f7483177c 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: "" expected_failed_job_count: 2 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-04-28T03:07:53Z" expected_candidates: - pr_url: "https://github.com/openshift/cluster-monitoring-operator/pull/2814" diff --git a/plugins/ci/evals/cases/payload-analysis/case-008/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-008/input.yaml index 867dce763..86cee74bf 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-008/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-008/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.nightly-2026-04-27-183150" +payload_completed_at: "2026-04-28T03:07:53Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml index 1713bd703..bc6d49b12 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-009/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 1 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-04-14T14:18:53Z" expected_candidates: [] diff --git a/plugins/ci/evals/cases/payload-analysis/case-009/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-009/input.yaml index 4f96beabc..623975f24 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-009/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-009/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.ci-2026-04-14-085906" +payload_completed_at: "2026-04-14T14:18:53Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml index 57211e2df..7da030cdb 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 4 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-05-14T16:40:01Z" notes: > Complex case: 4 failed blocking jobs (aws-ovn-serial, aws-ovn-single-node-upgrade, aws-ovn-upgrade, and one more). Tests the skill's ability to launch parallel diff --git a/plugins/ci/evals/cases/payload-analysis/case-010/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-010/input.yaml index f5dc61174..011526093 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-010/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-010/input.yaml @@ -1 +1,2 @@ payload_tag: "4.18.0-0.nightly-2026-05-14-085133" +payload_completed_at: "2026-05-14T16:40:01Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml index ea069d352..8c4660d01 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-011/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 1 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-05-15T00:00:06Z" expected_candidates: [] diff --git a/plugins/ci/evals/cases/payload-analysis/case-011/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-011/input.yaml index 87e3c3ace..793c148cc 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-011/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-011/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.ci-2026-05-14-181709" +payload_completed_at: "2026-05-15T00:00:06Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml index 12509e289..447b5c2d2 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-05-13T06:47:05Z" notes: > Rejection streak case: this payload is part of a multi-payload rejection streak (4 consecutive rejections: 2026-05-11-185554, 2026-05-12-035058, 2026-05-12-130101, diff --git a/plugins/ci/evals/cases/payload-analysis/case-012/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-012/input.yaml index 6e966cc4b..2330cc349 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-012/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-012/input.yaml @@ -1 +1,2 @@ payload_tag: "4.20.0-0.nightly-2026-05-12-225204" +payload_completed_at: "2026-05-13T06:47:05Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml index a5b1236ab..6b8451b83 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Accepted expected_failed_job_count: 1 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-05-13T15:35:28Z" notes: > Edge case: payload was Accepted despite 1 failed blocking job (hypershift-ovn-conformance-4.20). Tests that the skill correctly handles diff --git a/plugins/ci/evals/cases/payload-analysis/case-013/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-013/input.yaml index e6eae53ae..1c930730b 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-013/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-013/input.yaml @@ -1 +1,2 @@ payload_tag: "4.20.0-0.nightly-2026-05-13-064706" +payload_completed_at: "2026-05-13T15:35:28Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml index 73eddc911..d7ae3ebad 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-014/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 4 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-06-17T17:06:42Z" expected_candidates: - pr_url: "https://github.com/openshift/release/pull/78140" diff --git a/plugins/ci/evals/cases/payload-analysis/case-014/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-014/input.yaml index 03ab5f664..a178b9dec 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-014/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-014/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.nightly-2026-06-17-095118" +payload_completed_at: "2026-06-17T17:06:42Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml index 2e002a39b..1f53f088c 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 4 has_revert_candidates: true force_accept_expected: false +analysis_cutoff: "2026-07-23T07:44:48Z" expected_candidates: - pr_url: "https://github.com/openshift/oc/pull/2279" diff --git a/plugins/ci/evals/cases/payload-analysis/case-018/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-018/input.yaml index 76309d8c3..a9e6c146b 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-018/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-018/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.nightly-2026-07-22-233611" +payload_completed_at: "2026-07-23T07:44:48Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml index 7cb77b7e0..ebfedba70 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 3 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-07-25T16:05:26Z" expected_candidates: [] diff --git a/plugins/ci/evals/cases/payload-analysis/case-019/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-019/input.yaml index a16311ada..fad8ec9d4 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-019/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-019/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.ci-2026-07-25-102741" +payload_completed_at: "2026-07-25T16:05:26Z" diff --git a/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml index 45ef9f7c7..93bbe5b27 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml @@ -2,6 +2,7 @@ expected_phase: Rejected expected_failed_job_count: 2 has_revert_candidates: false force_accept_expected: false +analysis_cutoff: "2026-07-29T13:32:49Z" expected_candidates: [] diff --git a/plugins/ci/evals/cases/payload-analysis/case-020/input.yaml b/plugins/ci/evals/cases/payload-analysis/case-020/input.yaml index 3e243eb4d..d7b8057f0 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-020/input.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-020/input.yaml @@ -1 +1,2 @@ payload_tag: "5.0.0-0.nightly-2026-07-29-055948" +payload_completed_at: "2026-07-29T13:32:49Z" diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml index a8c9b053d..6525626df 100644 --- a/plugins/ci/evals/eval-payload-analysis.yaml +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -4,7 +4,7 @@ skill: ci:payload-analysis execution: mode: case - arguments: "{payload_tag} --snapshot-dir $EVAL_SNAPSHOT_DIR/{payload_tag}" + arguments: "{payload_tag} --snapshot-dir $EVAL_SNAPSHOT_DIR/{payload_tag} --as-of {payload_completed_at}" timeout: 3600 max_budget_usd: 25.0 env: @@ -30,11 +30,10 @@ runner: structured output files. NEVER write these files directly — the skills enforce the canonical schema. - When searching external sources (GitHub, Prow, GCS) for CI configuration - changes or PRs not found in the snapshot changelog, constrain searches to - changes merged before the payload creation timestamp. The payload tag - encodes its creation date (e.g., 4.22.0-0.nightly-2026-03-26-231124 was - created on 2026-03-26). + Perform a point-in-time analysis as of the `--as-of` timestamp. Do not use + later reverts, comments, subsequent payload outcomes, or the present-day + absence of a revert as causal evidence. Timestamp-bound every external + lookup and ignore results newer than the cutoff. models: judge: claude-opus-4-6 @@ -59,11 +58,14 @@ dataset: - input.yaml: YAML file with fields: - 'payload_tag' — a real payload tag from amd64.ocp.releases.ci.openshift.org (e.g., '5.0.0-0.nightly-2026-05-30-072431') + - 'payload_completed_at' — exact ISO-8601 evidence cutoff, derived from + the latest blocking-job completion in the payload - annotations.yaml: Expected outcomes and metadata for scoring: - 'expected_phase': Rejected|Ready|Accepted - 'expected_failed_job_count': integer - 'has_revert_candidates': boolean - 'force_accept_expected': boolean + - 'analysis_cutoff': the same ISO-8601 timestamp as payload_completed_at - 'expected_candidates': list of expected revert candidates with pr_url, component, min_confidence, expected_confidence, description, expected_failing_jobs @@ -262,6 +264,82 @@ judges: return (False, f"Missing Skill calls: {', '.join(missing)}. Found: {', '.join(found)}") return (True, f"Both required skills invoked: {', '.join(found)}") + - name: point_in_time_trace_hygiene + description: | + Hard gate against obvious evaluation leakage and mutable, unbounded + GitHub lookups. Immutable commit endpoints and timestamp-bounded commit + searches remain allowed. + check: | + import json, re + stdout = outputs.get("stdout", "") + violations = [] + + def github_url(value): + return re.search( + r"https?://(?:api\.|raw\.)?github(?:usercontent)?\.com", + value or "", + re.I, + ) + + for line in stdout.splitlines(): + try: + entry = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if entry.get("type") != "assistant": + continue + for block in entry.get("message", {}).get("content", []): + if block.get("type") != "tool_use": + continue + name = block.get("name", "") + tool_input = block.get("input", {}) + + if name == "Read": + path = tool_input.get("file_path", "") + if ( + path.endswith("annotations.yaml") + or "/evals/cases/payload-analysis/README.md" in path + or path.endswith("/comments.json") + ): + violations.append(f"Read evaluation/future-leaking file: {path}") + + if name == "WebFetch": + url = tool_input.get("url", "") + if github_url(url) or "claude-payload-agent" in url: + violations.append(f"Fetched mutable or post-hoc URL: {url}") + + if name != "Bash": + continue + command = tool_input.get("command", "") + if re.search( + r"(?:annotations\.yaml|evals/cases/payload-analysis/README\.md|" + r"/comments\.json)", + command, + re.I, + ): + violations.append("Bash read evaluation/future-leaking data") + + mutable_gh = ( + r"\bgh\s+pr\s+(?:view|list|status|checks|diff)\b", + r"\bgh\s+search\b", + r"\bgh\s+api\b[^\n]*(?:/pulls|/issues|/comments|/reviews)", + r"\b(?:curl|wget)\b[^\n]*" + r"(?:api\.)?github\.com", + ) + if any(re.search(pattern, command, re.I) for pattern in mutable_gh): + violations.append(f"Unbounded mutable GitHub lookup: {command[:180]}") + + if re.search(r"\bgh\s+api\b[^\n]*/commits\?", command, re.I): + if "until=" not in command: + violations.append( + f"GitHub commit search lacks an until cutoff: {command[:180]}" + ) + + if violations: + unique = list(dict.fromkeys(violations)) + return (False, "; ".join(unique[:8])) + return (True, "No obvious annotation leakage or unbounded mutable GitHub lookups") + - name: analysis_quality description: | LLM judge to assess root cause analysis depth, PR correlation accuracy, @@ -384,6 +462,69 @@ judges: correctly linked. No false positive revert recommendations for unrelated PRs. Cross-job correlation signal clearly documented. + - name: point_in_time_integrity + description: | + LLM judge assessing whether conclusions use only evidence that existed + when the payload completed, without post-hoc outcome leakage. + prompt: | + You are the evaluation-integrity judge for a historical OpenShift payload + analysis. The analysis must reconstruct what a release engineer could + have concluded at the payload completion timestamp in the annotations. + + Review the YAML results: + + {% for path, content in outputs.files.items() if path.endswith('.yaml') and 'payload-results' in path %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Review the HTML report: + + {% for path, content in outputs.files.items() if path.endswith('-summary.html') %} + ### {{ path }} + + {{ content }} + {% endfor %} + + Case metadata and exact cutoff: + + {{ annotations }} + + Judge evidence provenance, not whether hindsight happened to produce the + right answer. Evidence is admissible only when its event or observation + timestamp is at or before `analysis_cutoff`. Immutable code and CI + artifacts from runs completed by the cutoff are admissible. The report's + later generation timestamp is harmless. + + Treat these as post-hoc leakage when they occur after the cutoff: + - a later revert, its merge, or its experimental CI results; + - subsequent payload pass/fail outcomes; + - GitHub comments, reviews, status changes, or incident conclusions; + - present-day PR state or reasoning that a PR must be innocent because it + still has not been reverted; + - a previous payload-agent analysis generated after the cutoff. + + Also flag suspicious certainty whose only plausible basis is one of those + later facts, even when the report does not cite it candidly. Do not penalize + a conclusion independently established from contemporaneous logs, diffs, + and causal ordering merely because annotations also mention later + confirmation. + + Evaluate on a 1-5 scale: + + Score 1: A central diagnosis or revert decision explicitly depends on + post-cutoff evidence or leaked expected answers. + Score 2: Material post-cutoff evidence influences the result, even though + some contemporaneous evidence is present. + Score 3: No clear cheating, but important external evidence is undated or + provenance is too ambiguous to verify the cutoff. + Score 4: Point-in-time reasoning is clean; material evidence is + contemporaneous and later facts are excluded. + Score 5: Excellent temporal discipline — the cutoff is stated, evidence + provenance is auditable, and the report explicitly avoids or + quarantines tempting post-hoc signals. + thresholds: output_files_exist: min_pass_rate: 1.0 @@ -397,5 +538,9 @@ thresholds: min_mean: 3.5 revert_scoring_accuracy: min_mean: 3.0 + point_in_time_trace_hygiene: + min_pass_rate: 1.0 + point_in_time_integrity: + min_mean: 4.0 required_skill_invocations: min_pass_rate: 1.0 From b6bc3131e70f25860a074a8c57eb9a51f08c72ad Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 15:22:44 -0400 Subject: [PATCH 06/11] Close point-in-time eval trace gaps --- plugins/ci/evals/eval-payload-analysis.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml index 6525626df..076b16aad 100644 --- a/plugins/ci/evals/eval-payload-analysis.yaml +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -322,13 +322,26 @@ judges: mutable_gh = ( r"\bgh\s+pr\s+(?:view|list|status|checks|diff)\b", r"\bgh\s+search\b", + r"\bgh\s+api\s+graphql\b", r"\bgh\s+api\b[^\n]*(?:/pulls|/issues|/comments|/reviews)", r"\b(?:curl|wget)\b[^\n]*" r"(?:api\.)?github\.com", + r"\bgit\s+fetch\b[^\n]*(?:refs/)?pull/", ) if any(re.search(pattern, command, re.I) for pattern in mutable_gh): violations.append(f"Unbounded mutable GitHub lookup: {command[:180]}") + git_log_pattern = ( + r"\bgit(?:\s+-C\s+(?:\"[^\"]*\"|'[^']*'|\S+))?" + r"\s+log\b[^\n;&|]*" + ) + for match in re.finditer(git_log_pattern, command, re.I): + invocation = match.group(0) + if not re.search(r"--(?:until|before)(?:=|\s)", invocation, re.I): + violations.append( + f"Git history lookup lacks a cutoff: {invocation[:180]}" + ) + if re.search(r"\bgh\s+api\b[^\n]*/commits\?", command, re.I): if "until=" not in command: violations.append( From da5b85d0fcca0be7b2134e0e30d3a0e6b82b4da0 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 15:42:54 -0400 Subject: [PATCH 07/11] evals: enforce adversarial point-in-time constraints --- .../case-018/annotations.yaml | 16 ++ .../case-019/annotations.yaml | 18 ++ .../case-020/annotations.yaml | 24 ++ plugins/ci/evals/eval-payload-analysis.yaml | 264 ++++++++++++++++-- 4 files changed, 304 insertions(+), 18 deletions(-) diff --git a/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml index 1f53f088c..0fed500e1 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-018/annotations.yaml @@ -14,6 +14,22 @@ expected_candidates: - "aws-ovn-techpreview-serial-1of3" - "aws-ovn-techpreview-serial-2of3" +forbidden_revert_candidates: + - "https://github.com/openshift/hypershift/pull/8871" + +required_claims: + - name: "cross-tenant etcd evidence contamination" + any_of: + - "cross-tenant" + - "aggregates away namespace" + - "discards namespace identity" + - name: "target etcd pod did not roll" + any_of: + - "restartcount=0" + - "restart count was 0" + - "target etcd-0 has no restart" + - "no target statefulset rollout" + must_not_conclude: - "HyperShift PR #8871 caused the etcd leader-change failures by rolling or restarting the target HostedCluster etcd StatefulSet" - "The first payload where the aggregated HyperShift test name failed proves that PR #8871 introduced the underlying leader-election signature" diff --git a/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml index ebfedba70..1f548afca 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-019/annotations.yaml @@ -6,6 +6,24 @@ analysis_cutoff: "2026-07-25T16:05:26Z" expected_candidates: [] +forbidden_revert_candidates: + - "https://github.com/openshift/api/pull/2920" + - "https://github.com/openshift/api/pull/2923" + +required_claims: + - name: "disjoint interval arithmetic defect" + any_of: + - "disjoint waiting" + - "healthy gaps" + - "interval union" + - "sum of actual intervals" + - "summed waiting time" + - name: "test framework rather than product regression" + any_of: + - "test-framework arithmetic defect" + - "monitor test computes duration" + - "test defect" + must_not_conclude: - "openshift/api PR #2920 caused ClusterOperator availability regressions and should be reverted" - "openshift/api PR #2923 caused ClusterOperator availability regressions and should be reverted" diff --git a/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml index 93bbe5b27..cfdfbc346 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-020/annotations.yaml @@ -6,6 +6,30 @@ analysis_cutoff: "2026-07-29T13:32:49Z" expected_candidates: [] +forbidden_revert_candidates: + - "https://github.com/openshift/ovn-kubernetes/pull/3298" + - "https://github.com/openshift/cloud-provider-azure/pull/164" + +required_claims: + - name: "GCP infrastructure trigger precedes fallout" + any_of: + - "external vip reachability precedes" + - "vips were externally unreachable" + - "pre-upgrade network reachability loss" + - name: "Azure DNS trigger" + any_of: + - "dns 429" + - "dns throttling" + - name: "Azure provisioning failure precedes disk attach" + any_of: + - "osprovisioningtimedout" + - "provisioning timeout" + - name: "terminal symptoms are not the initial trigger" + any_of: + - "cleanup fallout" + - "amplifies the failure" + - "terminal symptom" + must_not_conclude: - "openshift/ovn-kubernetes PR #3298 caused the GCP external-container panic and should be reverted" - "openshift/cloud-provider-azure PR #164 introduced the empty VirtualMachine update and should be reverted" diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml index 076b16aad..de780a3b4 100644 --- a/plugins/ci/evals/eval-payload-analysis.yaml +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -69,6 +69,10 @@ dataset: - 'expected_candidates': list of expected revert candidates with pr_url, component, min_confidence, expected_confidence, description, expected_failing_jobs + - 'forbidden_revert_candidates': PR URLs that must not be recommended at + revert confidence or marked revert-eligible + - 'required_claims': causal findings that must appear in the analysis; + each entry supplies a name and a list of acceptable evidence phrases - 'notes': free text context Snapshots are stored as tar.gz archives in @@ -270,16 +274,114 @@ judges: GitHub lookups. Immutable commit endpoints and timestamp-bounded commit searches remain allowed. check: | - import json, re + import json, re, urllib.parse, yaml + from datetime import datetime, timezone + stdout = outputs.get("stdout", "") violations = [] + shell_vars = {} + + case_data = outputs.get("annotations", {}) - def github_url(value): - return re.search( - r"https?://(?:api\.|raw\.)?github(?:usercontent)?\.com", - value or "", - re.I, + def absolute_time(value): + value = urllib.parse.unquote((value or "").strip().strip("\"'")) + variable = re.fullmatch(r"\$(?:\{([A-Za-z_]\w*)\}|([A-Za-z_]\w*))", value) + if variable: + value = shell_vars.get(variable.group(1) or variable.group(2), "") + if not value or value.lower() in {"now", "today", "yesterday", "tomorrow"}: + return None + try: + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + return datetime.fromisoformat(value).replace(tzinfo=timezone.utc) + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + return None + return parsed + except ValueError: + return None + + cutoff_text = ( + case_data.get("analysis_cutoff", "") + if isinstance(case_data, dict) + else "" + ) + cutoff = absolute_time(cutoff_text) + + def remember_absolute_assignments(command): + assignment = re.compile( + r"(?:^|[\s;])([A-Za-z_]\w*)\s*=\s*" + r"(?:\"([^\"]+)\"|'([^']+)'|([^\s;&|]+))" ) + for match in assignment.finditer(command): + value = match.group(2) or match.group(3) or match.group(4) + if absolute_time(value): + shell_vars[match.group(1)] = value + + def check_bound(value, context): + parsed = absolute_time(value) + if parsed is None: + violations.append( + f"{context} uses a relative or unresolved cutoff: {value}" + ) + elif cutoff is not None and parsed > cutoff: + violations.append( + f"{context} exceeds analysis cutoff {cutoff_text}: {value}" + ) + + def is_github_url(value): + try: + host = (urllib.parse.urlsplit(value).hostname or "").lower() + except ValueError: + return False + return host in { + "github.com", + "api.github.com", + "raw.githubusercontent.com", + } + + def is_immutable_github_url(value): + try: + parsed = urllib.parse.urlsplit(value) + except ValueError: + return False + host = (parsed.hostname or "").lower() + path = parsed.path + sha = r"[0-9a-fA-F]{40}" + if host == "api.github.com": + if re.fullmatch( + rf"/repos/[^/]+/[^/]+/(?:git/)?commits/{sha}", path + ): + return True + if re.fullmatch(r"/repos/[^/]+/[^/]+/contents/.+", path): + ref = urllib.parse.parse_qs(parsed.query).get("ref", [""])[0] + return bool(re.fullmatch(sha, ref)) + if host == "raw.githubusercontent.com": + return bool(re.fullmatch(rf"/[^/]+/[^/]+/{sha}/.+", path)) + if host == "github.com": + return bool( + re.fullmatch( + rf"/[^/]+/[^/]+/(?:commit/{sha}(?:\.(?:diff|patch))?|" + rf"(?:blob|raw)/{sha}/.+)", + path, + ) + ) + return False + + def is_completed_agent_report(value): + return ( + "claude-payload-agent" in value + and "/artifacts/" in value + and re.search(r"payload-analysis-.+-summary\.html(?:\?.*)?$", value) + ) + + def check_external_url(url, context): + url = url.rstrip("),.;") + if is_github_url(url) and not is_immutable_github_url(url): + violations.append(f"{context} used mutable GitHub URL: {url}") + if "claude-payload-agent" in url and not is_completed_agent_report(url): + violations.append( + f"{context} used live/current Payload Agent URL: {url}" + ) for line in stdout.splitlines(): try: @@ -305,12 +407,12 @@ judges: if name == "WebFetch": url = tool_input.get("url", "") - if github_url(url) or "claude-payload-agent" in url: - violations.append(f"Fetched mutable or post-hoc URL: {url}") + check_external_url(url, "WebFetch") if name != "Bash": continue command = tool_input.get("command", "") + remember_absolute_assignments(command) if re.search( r"(?:annotations\.yaml|evals/cases/payload-analysis/README\.md|" r"/comments\.json)", @@ -324,35 +426,154 @@ judges: r"\bgh\s+search\b", r"\bgh\s+api\s+graphql\b", r"\bgh\s+api\b[^\n]*(?:/pulls|/issues|/comments|/reviews)", - r"\b(?:curl|wget)\b[^\n]*" - r"(?:api\.)?github\.com", r"\bgit\s+fetch\b[^\n]*(?:refs/)?pull/", ) if any(re.search(pattern, command, re.I) for pattern in mutable_gh): violations.append(f"Unbounded mutable GitHub lookup: {command[:180]}") + for url in re.findall(r"https?://[^\s'\"<>]+", command, re.I): + check_external_url(url, "Bash") + git_log_pattern = ( r"\bgit(?:\s+-C\s+(?:\"[^\"]*\"|'[^']*'|\S+))?" r"\s+log\b[^\n;&|]*" ) for match in re.finditer(git_log_pattern, command, re.I): invocation = match.group(0) - if not re.search(r"--(?:until|before)(?:=|\s)", invocation, re.I): + bounds = list( + re.finditer( + r"--(?:until|before)(?:=|\s+)" + r"(?P\"[^\"]+\"|'[^']+'|[^\s;&|]+)", + invocation, + re.I, + ) + ) + if not bounds: violations.append( f"Git history lookup lacks a cutoff: {invocation[:180]}" ) + for bound in bounds: + check_bound(bound.group("value"), "Git history lookup") - if re.search(r"\bgh\s+api\b[^\n]*/commits\?", command, re.I): - if "until=" not in command: + for api_call in re.finditer( + r"\bgh\s+api\b[^\n;|]*?/commits(?:\?|[\"'\s]|$)[^\n;|]*", + command, + re.I, + ): + invocation = api_call.group(0) + if re.search( + r"/commits/[0-9a-f]{40}(?:[\"'\s]|$)", + invocation, + re.I, + ): + continue + bounds = re.findall( + r"(?:[?&]|\\u0026)until=([^&\s'\"\\]+)", + invocation, + re.I, + ) + if not bounds: violations.append( - f"GitHub commit search lacks an until cutoff: {command[:180]}" + f"GitHub commit search lacks an until cutoff: " + f"{invocation[:180]}" ) + for bound in bounds: + check_bound(bound, "GitHub commit search") if violations: unique = list(dict.fromkeys(violations)) return (False, "; ".join(unique[:8])) return (True, "No obvious annotation leakage or unbounded mutable GitHub lookups") + - name: case_constraints + description: | + Machine-check expected candidates, forbidden high-confidence revert + recommendations, and required causal findings from case annotations. + check: | + import os, yaml + + case_data = outputs.get("annotations", {}) + if not isinstance(case_data, dict): + return (False, "Case annotations are unavailable to deterministic judge") + + all_files = { + **outputs.get("files", {}), + **outputs.get("modified_files", {}), + } + result_files = { + path: content for path, content in all_files.items() + if os.path.basename(path).startswith("payload-results-") + and path.endswith(".yaml") + } + if not result_files: + return (False, "No payload results YAML found") + try: + result = yaml.safe_load(next(iter(result_files.values()))) or {} + except yaml.YAMLError as exc: + return (False, f"Invalid payload results YAML: {exc}") + + candidates = result.get("candidates", []) or [] + by_url = { + str(candidate.get("pr_url", "")).rstrip("/").lower(): candidate + for candidate in candidates + if isinstance(candidate, dict) + } + failures = [] + + for expected in case_data.get("expected_candidates", []) or []: + url = str(expected.get("pr_url", "")).rstrip("/").lower() + candidate = by_url.get(url) + if candidate is None: + failures.append(f"Missing expected candidate {url}") + continue + minimum = int(expected.get("min_confidence", 0)) + actual = int(candidate.get("confidence_score", 0)) + if actual < minimum: + failures.append( + f"Expected candidate {url} scored {actual}, below {minimum}" + ) + + for url in case_data.get("forbidden_revert_candidates", []) or []: + normalized = str(url).rstrip("/").lower() + candidate = by_url.get(normalized) + if candidate is None: + continue + confidence = int(candidate.get("confidence_score", 0)) + eligible = candidate.get("revert_eligible") is True + if confidence >= 85 or eligible: + failures.append( + f"Forbidden revert candidate {normalized} remained actionable " + f"(confidence={confidence}, revert_eligible={eligible})" + ) + + generated = { + path: content for path, content in all_files.items() + if path.endswith("-summary.html") + or ( + os.path.basename(path).startswith("payload-results-") + and path.endswith(".yaml") + ) + or path.endswith("-autodl.json") + } + searchable = "\n".join(str(content) for content in generated.values()).lower() + for claim in case_data.get("required_claims", []) or []: + alternatives = [ + str(value).lower() + for value in claim.get("any_of", []) + if value + ] + if alternatives and not any(value in searchable for value in alternatives): + failures.append( + f"Missing required causal claim: {claim.get('name')}" + ) + + if failures: + return (False, "; ".join(failures)) + return ( + True, + "Expected, forbidden, and required causal constraints satisfied", + ) + - name: analysis_quality description: | LLM judge to assess root cause analysis depth, PR correlation accuracy, @@ -409,6 +630,8 @@ judges: rather than generic (just restating test names)? - If expected_candidates are listed in annotations, are they identified with evidence-based rationale? + - Are every required_claim and discriminating signal established from + evidence, and are must_not_conclude claims avoided? - Does the HTML report contain actionable information? - name: revert_scoring_accuracy @@ -452,7 +675,8 @@ judges: If expected_candidates are listed in annotations, those PRs should be identified with confidence >= min_confidence and linked to the expected_failing_jobs. If expected_candidates is empty, no false positive - revert recommendations should appear. + revert recommendations should appear. Any forbidden_revert_candidates + must remain below revert confidence and must not be marked revert-eligible. Evaluate on a 1-5 scale: @@ -506,9 +730,11 @@ judges: Judge evidence provenance, not whether hindsight happened to produce the right answer. Evidence is admissible only when its event or observation - timestamp is at or before `analysis_cutoff`. Immutable code and CI - artifacts from runs completed by the cutoff are admissible. The report's - later generation timestamp is harmless. + timestamp is at or before `analysis_cutoff`. SHA-pinning alone does not + make code contemporaneous: the cited commit must have existed by the + cutoff. CI artifacts and previous agent reports are admissible only when + their Prow jobs completed by the cutoff. Missing provenance caps the score + at 3. The report's later generation timestamp is harmless. Treat these as post-hoc leakage when they occur after the cutoff: - a later revert, its merge, or its experimental CI results; @@ -553,6 +779,8 @@ thresholds: min_mean: 3.0 point_in_time_trace_hygiene: min_pass_rate: 1.0 + case_constraints: + min_pass_rate: 1.0 point_in_time_integrity: min_mean: 4.0 required_skill_invocations: From 9837a1ed444ac147fb988cc95935923117437aa5 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 15:55:30 -0400 Subject: [PATCH 08/11] evals: reconcile payload case oracles --- .../case-001/annotations.yaml | 1 + .../case-002/annotations.yaml | 12 ++++++--- .../case-003/annotations.yaml | 8 +++--- .../case-008/annotations.yaml | 8 +++--- .../case-010/annotations.yaml | 8 +++--- .../case-012/annotations.yaml | 1 + .../case-013/annotations.yaml | 1 + plugins/ci/evals/eval-payload-analysis.yaml | 27 ++++++++++++++----- 8 files changed, 44 insertions(+), 22 deletions(-) diff --git a/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml index 86cea25b3..ea3441c1c 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-001/annotations.yaml @@ -17,6 +17,7 @@ expected_candidates: expected_ci_config_issues: - pr_url: "https://github.com/openshift/release/pull/76852" repo: "openshift/release" + min_confidence: 85 description: "Changed vSphere VCM default lease to multi-tenant, breaking UPI workflows that require single-tenant leases" expected_failing_jobs: - "install-analysis-all" diff --git a/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml index 48050954a..1dc5eaa6b 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-002/annotations.yaml @@ -25,9 +25,13 @@ notes: > same payload with 63 candidate PRs. openshift/origin#31006 ("add oc-mirror to extensionBinaries") caused widespread test failures across multiple job families — serial, techpreview-serial, metal-ipi, and potentially upgrade - jobs. The agent must identify this PR as the root cause despite the large - candidate pool (63 PRs). Key signals: cross-job correlation (same PR blamed - for failures in many independent jobs), error messages referencing test - binaries/extensions, and new failure mode not present in prior payloads. + jobs. The eight expected_failing_jobs above are the jobs this candidate is + expected to explain; the other five blocking failures have separate causes, + so the candidate-specific list intentionally does not equal the total + expected_failed_job_count. The agent must identify this PR as the root cause + despite the large candidate pool (63 PRs). Key signals: cross-job correlation + (same PR blamed for failures in many independent jobs), error messages + referencing test binaries/extensions, and new failure mode not present in + prior payloads. force_accept_expected is false because hours_since_baseline is 11.7h (under the 18h threshold) and failures are product regressions, not infrastructure. diff --git a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml index 9b573e0a9..5a361db83 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml @@ -1,5 +1,5 @@ expected_phase: Rejected -expected_failed_job_count: 2 +expected_failed_job_count: 3 has_revert_candidates: true force_accept_expected: false analysis_cutoff: "2026-05-07T20:17:14Z" @@ -11,9 +11,9 @@ expected_candidates: expected_confidence: 90 description: "Added remaining CNO NetworkPolicies that blocked egress from cloud-network-config-controller, causing CrashLoopBackOff due to API server timeout" expected_failing_jobs: - - "e2e-aws-ovn" - - "e2e-azure-ovn" - - "e2e-gcp-ovn" + - "aggregated-aws-ovn-upgrade-5.0-major" + - "aggregated-azure-ovn-upgrade-5.0-major" + - "aggregated-gcp-ovn-upgrade-5.0-micro" notes: > Textbook true positive. CNO #2959 added NetworkPolicies that blocked egress diff --git a/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml index f7483177c..d47f972c6 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-008/annotations.yaml @@ -1,4 +1,4 @@ -expected_phase: "" +expected_phase: Rejected expected_failed_job_count: 2 has_revert_candidates: true force_accept_expected: false @@ -11,10 +11,10 @@ expected_candidates: expected_confidence: 95 description: "Minimal and telemetry CP monitors change caused monitoring test regressions" expected_failing_jobs: - - "e2e-aws-ovn" + - "aws-ovn-techpreview-serial-1of3" notes: > True positive from the CMO monitoring regression series. CMO #2814 (MON-4517: Minimal and telemetry CP monitors) introduced monitoring - test failures. Reverted by #2901 on 2026-04-28. Expected phase is - empty because the payload was not formally Rejected. + test failures. Reverted by #2901 on 2026-04-28. The archived snapshot records + the payload phase as Rejected. diff --git a/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml index 7da030cdb..8dcb224e6 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-010/annotations.yaml @@ -3,8 +3,10 @@ expected_failed_job_count: 4 has_revert_candidates: false force_accept_expected: false analysis_cutoff: "2026-05-14T16:40:01Z" +expected_candidates: [] notes: > Complex case: 4 failed blocking jobs (aws-ovn-serial, aws-ovn-single-node-upgrade, - aws-ovn-upgrade, and one more). Tests the skill's ability to launch parallel - subagents, correlate multiple failures, and identify patterns across job families. - Previous payload was accepted, so these may be new failures. + aws-ovn-upgrade, and gcp-ovn-rt-upgrade-4.18-minor). Tests the skill's ability + to launch parallel subagents, correlate multiple failures, and identify + patterns across job families. Previous payload was accepted, so these may be + new failures. diff --git a/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml index 447b5c2d2..f1e10c3c6 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-012/annotations.yaml @@ -3,6 +3,7 @@ expected_failed_job_count: 3 has_revert_candidates: false force_accept_expected: false analysis_cutoff: "2026-05-13T06:47:05Z" +expected_candidates: [] notes: > Rejection streak case: this payload is part of a multi-payload rejection streak (4 consecutive rejections: 2026-05-11-185554, 2026-05-12-035058, 2026-05-12-130101, diff --git a/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml index 6b8451b83..1afa46b2c 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-013/annotations.yaml @@ -3,6 +3,7 @@ expected_failed_job_count: 1 has_revert_candidates: false force_accept_expected: false analysis_cutoff: "2026-05-13T15:35:28Z" +expected_candidates: [] notes: > Edge case: payload was Accepted despite 1 failed blocking job (hypershift-ovn-conformance-4.20). Tests that the skill correctly handles diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml index de780a3b4..243ccf5a6 100644 --- a/plugins/ci/evals/eval-payload-analysis.yaml +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -68,7 +68,11 @@ dataset: - 'analysis_cutoff': the same ISO-8601 timestamp as payload_completed_at - 'expected_candidates': list of expected revert candidates with pr_url, component, min_confidence, expected_confidence, description, - expected_failing_jobs + expected_failing_jobs. The failing-job list is the candidate-specific + subset it should explain and need not equal expected_failed_job_count. + - 'expected_ci_config_issues': optional list of causal CI-configuration + PRs outside the payload changelog, with pr_url, min_confidence, + description, and expected_failing_jobs - 'forbidden_revert_candidates': PR URLs that must not be recommended at revert confidence or marked revert-eligible - 'required_claims': causal findings that must appear in the analysis; @@ -520,7 +524,11 @@ judges: } failures = [] - for expected in case_data.get("expected_candidates", []) or []: + expected_causes = ( + (case_data.get("expected_candidates", []) or []) + + (case_data.get("expected_ci_config_issues", []) or []) + ) + for expected in expected_causes: url = str(expected.get("pr_url", "")).rstrip("/").lower() candidate = by_url.get(url) if candidate is None: @@ -630,6 +638,9 @@ judges: rather than generic (just restating test names)? - If expected_candidates are listed in annotations, are they identified with evidence-based rationale? + - If expected_ci_config_issues are listed, are those out-of-payload CI + changes identified as structured causal candidates and tied to the + expected jobs? - Are every required_claim and discriminating signal established from evidence, and are must_not_conclude claims avoided? - Does the HTML report contain actionable information? @@ -672,11 +683,13 @@ judges: {{ annotations }} - If expected_candidates are listed in annotations, those PRs should be - identified with confidence >= min_confidence and linked to the - expected_failing_jobs. If expected_candidates is empty, no false positive - revert recommendations should appear. Any forbidden_revert_candidates - must remain below revert confidence and must not be marked revert-eligible. + If expected_candidates or expected_ci_config_issues are listed in + annotations, those PRs should be identified with confidence >= + min_confidence and linked to the expected_failing_jobs. A CI-configuration + cause belongs in structured candidates even though it is outside the + payload changelog. If both lists are empty, no false positive revert + recommendations should appear. Any forbidden_revert_candidates must + remain below revert confidence and must not be marked revert-eligible. Evaluate on a 1-5 scale: From 14d30b245014d31e22865614d2073f36f39037ba Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 16:45:37 -0400 Subject: [PATCH 09/11] evals: correct shared payload scoring judge --- .claude-plugin/marketplace.json | 2 +- docs/index.html | 2 +- plugins/ci/.claude-plugin/plugin.json | 2 +- plugins/ci/evals/eval-payload-analysis.yaml | 32 ++++++++++++++----- .../ci/evals/test_eval_payload_analysis.py | 29 +++++++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 plugins/ci/evals/test_eval_payload_analysis.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 90668c162..c266c4aee 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -88,7 +88,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.78", + "version": "0.0.79", "category": "ci", "keywords": [ "prow", diff --git a/docs/index.html b/docs/index.html index ccf18fab3..5f3df1ac0 100644 --- a/docs/index.html +++ b/docs/index.html @@ -541,7 +541,7 @@

ai-helpers

{ "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.78", + "version": "0.0.79", "has_readme": true, "commands": [ { diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index d96954e32..c6ac8f0a8 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.78", + "version": "0.0.79", "author": { "name": "github.com/openshift-eng" } diff --git a/plugins/ci/evals/eval-payload-analysis.yaml b/plugins/ci/evals/eval-payload-analysis.yaml index 243ccf5a6..f72083416 100644 --- a/plugins/ci/evals/eval-payload-analysis.yaml +++ b/plugins/ci/evals/eval-payload-analysis.yaml @@ -654,14 +654,30 @@ judges: You are evaluating whether the payload analysis skill correctly identified revert candidates and applied its confidence scoring rubric. - The rubric awards points for: - - New failure mode: +30 (this specific failure wasn't present in prior payloads) - - Component exclusivity: +10 to +30 (fewer PRs touching same component = higher) - - Error message match: +40 (errors directly reference code changed by the PR) - - Multi-job correlation: +10 (same PR is candidate for multiple failed jobs) - - Presubmit coverage gap: +10 (failing scenario not in PR's presubmit tests) - - Single candidate: +10 (only one PR touches the affected component) - Maximum: 130 (capped at 100). Revert threshold: >= 85. + First compute the raw rubric score: + - New failure mode: +30 only when the minimal causal signature has a + verified boundary in raw artifacts and is plausibly attributable to + changed code. + - Component exclusivity: +30 for the sole modifier of the affected + component, +20 for 2-3 modifiers, or +10 for 4+ modifiers. The sole + modifier tier already covers the single-candidate signal. + - Error message match: +40 for a direct string, symbol, function, or + identifier match; +20-30 for the same executed code path; or +10 for + the same subsystem only. + - Multi-job correlation: +10 for independent executions with the same + causal signature. + - Presubmit coverage gap: +10 when the failing scenario is not covered. + The raw maximum is 120 and is capped at 100. + + Some treatment outputs additionally cap final confidence by causal + evidence: 100 for an executed end-to-end mechanism with alternatives + excluded, 80 for an incomplete mechanism, 60 for contextual correlation + only, and 20 when contradicted. When the output records such a cap, + verify that final confidence is the lower of the capped raw score and + causal-evidence cap. When it records `revert_gates`, verify that + `revert_eligible` is true only at confidence >= 85 with all five gates + passing. Do not penalize a control output solely because it uses the + legacy schema without causal-cap or revert-gate fields. Review the YAML results file: diff --git a/plugins/ci/evals/test_eval_payload_analysis.py b/plugins/ci/evals/test_eval_payload_analysis.py new file mode 100644 index 000000000..65f720234 --- /dev/null +++ b/plugins/ci/evals/test_eval_payload_analysis.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import os +import unittest + +import yaml + + +EVAL_PATH = os.path.join(os.path.dirname(__file__), "eval-payload-analysis.yaml") + + +class ScoringPromptTest(unittest.TestCase): + def test_shared_judge_uses_current_raw_rubric(self): + with open(EVAL_PATH) as stream: + config = yaml.safe_load(stream) + prompt = next( + judge["prompt"] + for judge in config["judges"] + if judge["name"] == "revert_scoring_accuracy" + ) + + self.assertIn("raw maximum is 120", prompt) + self.assertNotIn("Maximum: 130", prompt) + self.assertNotIn("Single candidate: +10", prompt) + self.assertIn("Do not penalize a control output", prompt) + + +if __name__ == "__main__": + unittest.main() From 5ec4babddd479b7aec6a4c2b02eefd3d5f40e6f4 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Thu, 30 Jul 2026 18:01:39 -0400 Subject: [PATCH 10/11] evals: restore underlying job match substrings --- .claude-plugin/marketplace.json | 2 +- docs/index.html | 2 +- plugins/ci/.claude-plugin/plugin.json | 2 +- .../evals/cases/payload-analysis/case-003/annotations.yaml | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c266c4aee..0d9badc35 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -88,7 +88,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.79", + "version": "0.0.80", "category": "ci", "keywords": [ "prow", diff --git a/docs/index.html b/docs/index.html index d084d3069..e52654ddf 100644 --- a/docs/index.html +++ b/docs/index.html @@ -541,7 +541,7 @@

ai-helpers

{ "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.79", + "version": "0.0.80", "has_readme": true, "commands": [ { diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index c6ac8f0a8..6cd9409dc 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.79", + "version": "0.0.80", "author": { "name": "github.com/openshift-eng" } diff --git a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml index 5a361db83..76a3eb80a 100644 --- a/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml +++ b/plugins/ci/evals/cases/payload-analysis/case-003/annotations.yaml @@ -11,9 +11,9 @@ expected_candidates: expected_confidence: 90 description: "Added remaining CNO NetworkPolicies that blocked egress from cloud-network-config-controller, causing CrashLoopBackOff due to API server timeout" expected_failing_jobs: - - "aggregated-aws-ovn-upgrade-5.0-major" - - "aggregated-azure-ovn-upgrade-5.0-major" - - "aggregated-gcp-ovn-upgrade-5.0-micro" + - "e2e-aws-ovn" + - "e2e-azure-ovn" + - "e2e-gcp-ovn" notes: > Textbook true positive. CNO #2959 added NetworkPolicies that blocked egress From 81cba1776794892b41877d691edfcb215de319eb Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 31 Jul 2026 06:05:06 -0400 Subject: [PATCH 11/11] evals: add pre-604 skill-set control --- .claude-plugin/marketplace.json | 2 +- docs/index.html | 74 +- plugins/ci/.claude-plugin/plugin.json | 2 +- plugins/ci/skills/analyze-disruption/SKILL.md | 4 +- .../skills/bulk-triage-regressions/SKILL.md | 284 ---- plugins/ci/skills/detect-permafail/SKILL.md | 884 ----------- .../skills/diagnose-job-run-symptoms/SKILL.md | 109 -- .../diagnose_job_run.py | 233 --- .../test_diagnose_job_run.py | 95 -- .../ci/skills/fetch-job-run-summary/SKILL.md | 2 +- .../ci/skills/fetch-prow-job-runs/SKILL.md | 158 -- .../fetch_prow_job_runs.py | 133 -- .../test_fetch_prow_job_runs.py | 69 - plugins/ci/skills/fetch-prowjob-json/SKILL.md | 4 +- .../skills/fetch-regression-details/SKILL.md | 2 +- plugins/ci/skills/list-symptoms/SKILL.md | 199 --- .../ci/skills/list-symptoms/list_symptoms.py | 118 -- .../list-symptoms/test_list_symptoms.py | 53 - plugins/ci/skills/manage-labels/SKILL.md | 150 -- .../ci/skills/manage-labels/manage_labels.py | 155 -- .../manage-labels/test_manage_labels.py | 57 - plugins/ci/skills/manage-symptoms/SKILL.md | 192 --- .../skills/manage-symptoms/manage_symptoms.py | 188 --- .../manage-symptoms/test_manage_symptoms.py | 101 -- plugins/ci/skills/payload-analysis/SKILL.md | 202 +-- .../ci/skills/payload-autodl-json/SKILL.md | 2 +- plugins/ci/skills/payload-snapshot/SKILL.md | 131 +- .../scripts/payload_snapshot.py | 1204 ++------------- .../scripts/test_collection_completeness.py | 354 ----- .../scripts/test_sippy_fallback.py | 322 ---- .../scripts/test_test_classification.py | 211 --- plugins/ci/skills/prow-job-analysis/SKILL.md | 185 --- .../prow_job_artifact_search.py | 605 -------- .../references/aggregated.md | 647 --------- .../prow-job-analysis/references/artifacts.md | 1096 -------------- .../references/ci-infrastructure-changes.md | 1139 --------------- .../references/cloud-provider-errors.md | 303 ---- .../references/disruption.md | 1273 ---------------- .../references/flaky-test-identification.md | 319 ---- .../references/hypershift.md | 186 --- .../references/install/general.md | 871 ----------- .../references/install/metal.md | 971 ------------- .../references/networking.md | 294 ---- .../references/operating-system-changes.md | 203 --- .../references/resource-exhaustion.md | 413 ------ .../references/test-extension-binaries.md | 814 ----------- .../references/test-failure.md | 237 --- .../prow-job-analysis/references/upgrade.md | 1188 --------------- .../prow-job-analyze-install-failure/SKILL.md | 711 +++++++++ .../SKILL.md | 459 ++++++ .../prow-job-analyze-test-failure/README.md | 13 + .../prow-job-analyze-test-failure/SKILL.md | 1289 +++++++++++++++++ .../skills/prow-job-artifact-search/SKILL.md | 259 ++++ .../prow_job_artifact_search.py | 332 +++++ .../prow-job-extract-must-gather/CHANGELOG.md | 54 + .../prow-job-extract-must-gather/README.md | 350 +++++ .../prow-job-extract-must-gather/SKILL.md | 490 +++++++ .../extract_archives.py | 202 +++ .../generate_html_report.py | 1289 +++++++++++++++++ .../ci/skills/reevaluate-job-runs/SKILL.md | 165 --- .../reevaluate_job_runs.py | 165 --- .../test_reevaluate_job_runs.py | 112 -- 62 files changed, 5617 insertions(+), 16711 deletions(-) delete mode 100644 plugins/ci/skills/bulk-triage-regressions/SKILL.md delete mode 100644 plugins/ci/skills/detect-permafail/SKILL.md delete mode 100644 plugins/ci/skills/diagnose-job-run-symptoms/SKILL.md delete mode 100644 plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py delete mode 100644 plugins/ci/skills/diagnose-job-run-symptoms/test_diagnose_job_run.py delete mode 100644 plugins/ci/skills/fetch-prow-job-runs/SKILL.md delete mode 100644 plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py delete mode 100644 plugins/ci/skills/fetch-prow-job-runs/test_fetch_prow_job_runs.py delete mode 100644 plugins/ci/skills/list-symptoms/SKILL.md delete mode 100644 plugins/ci/skills/list-symptoms/list_symptoms.py delete mode 100644 plugins/ci/skills/list-symptoms/test_list_symptoms.py delete mode 100644 plugins/ci/skills/manage-labels/SKILL.md delete mode 100644 plugins/ci/skills/manage-labels/manage_labels.py delete mode 100644 plugins/ci/skills/manage-labels/test_manage_labels.py delete mode 100644 plugins/ci/skills/manage-symptoms/SKILL.md delete mode 100644 plugins/ci/skills/manage-symptoms/manage_symptoms.py delete mode 100644 plugins/ci/skills/manage-symptoms/test_manage_symptoms.py delete mode 100644 plugins/ci/skills/payload-snapshot/scripts/test_collection_completeness.py delete mode 100644 plugins/ci/skills/payload-snapshot/scripts/test_sippy_fallback.py delete mode 100644 plugins/ci/skills/payload-snapshot/scripts/test_test_classification.py delete mode 100644 plugins/ci/skills/prow-job-analysis/SKILL.md delete mode 100755 plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py delete mode 100644 plugins/ci/skills/prow-job-analysis/references/aggregated.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/artifacts.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/ci-infrastructure-changes.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/cloud-provider-errors.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/disruption.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/flaky-test-identification.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/hypershift.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/install/general.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/install/metal.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/networking.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/operating-system-changes.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/resource-exhaustion.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/test-extension-binaries.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/test-failure.md delete mode 100644 plugins/ci/skills/prow-job-analysis/references/upgrade.md create mode 100644 plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md create mode 100644 plugins/ci/skills/prow-job-analyze-metal-install-failure/SKILL.md create mode 100644 plugins/ci/skills/prow-job-analyze-test-failure/README.md create mode 100644 plugins/ci/skills/prow-job-analyze-test-failure/SKILL.md create mode 100644 plugins/ci/skills/prow-job-artifact-search/SKILL.md create mode 100755 plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py create mode 100644 plugins/ci/skills/prow-job-extract-must-gather/CHANGELOG.md create mode 100644 plugins/ci/skills/prow-job-extract-must-gather/README.md create mode 100644 plugins/ci/skills/prow-job-extract-must-gather/SKILL.md create mode 100755 plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py create mode 100755 plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py delete mode 100644 plugins/ci/skills/reevaluate-job-runs/SKILL.md delete mode 100644 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py delete mode 100644 plugins/ci/skills/reevaluate-job-runs/test_reevaluate_job_runs.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 0d9badc35..f6807d699 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -88,7 +88,7 @@ "name": "ci", "source": "./plugins/ci", "description": "A plugin to work with OpenShift CI and analyze Prow job results", - "version": "0.0.80", + "version": "0.0.81", "category": "ci", "keywords": [ "prow", diff --git a/docs/index.html b/docs/index.html index e52654ddf..c8da2c761 100644 --- a/docs/index.html +++ b/docs/index.html @@ -541,7 +541,7 @@

ai-helpers

{ "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.80", + "version": "0.0.81", "has_readme": true, "commands": [ { @@ -734,24 +734,6 @@

ai-helpers

"description_html": "Analyze and compare disruption across one or more Prow CI job runs by examining interval data, audit logs, pod logs, and CPU metrics", "meta": "" }, - { - "name": "bulk-triage-regressions", - "description": "Use this skill for Component Readiness triage duty - holistically analyze and triage all untriaged regressions for a set of components in a view, clustering them into root-cause buckets", - "description_html": "Use this skill for Component Readiness triage duty - holistically analyze and triage all untriaged regressions for a set of components in a view, clustering them into root-cause buckets", - "meta": "" - }, - { - "name": "detect-permafail", - "description": "Analyze consecutive job failures to determine if they represent a permafail pattern versus flaky failures", - "description_html": "Analyze consecutive job failures to determine if they represent a permafail pattern versus flaky failures", - "meta": "" - }, - { - "name": "diagnose-job-run-symptoms", - "description": "Explain which Sippy Symptoms and failure Labels apply to a Prow CI job run, in plain language, given only the Prow URL", - "description_html": "Explain which Sippy Symptoms and failure Labels apply to a Prow CI job run, in plain language, given only the Prow URL", - "meta": "" - }, { "name": "fetch-jira-issue", "description": "Fetch JIRA issue details including status, assignee, comments, and progress classification", @@ -776,12 +758,6 @@

ai-helpers

"description_html": "Fetch recent release payloads from the OpenShift release controller", "meta": "" }, - { - "name": "fetch-prow-job-runs", - "description": "List Prow job runs from the Sippy API using its real filter syntax — find run IDs by job name, variant, result, or time window", - "description_html": "List Prow job runs from the Sippy API using its real filter syntax — find run IDs by job name, variant, result, or time window", - "meta": "" - }, { "name": "fetch-prowjob-json", "description": "Fetch and return key data from a Prow job's prowjob.json artifact given a Prow job URL", @@ -818,24 +794,6 @@

ai-helpers

"description_html": "Fetch test runs from Sippy API including outputs for AI-based similarity analysis", "meta": "" }, - { - "name": "list-symptoms", - "description": "List, search, and inspect Sippy Symptoms and Labels — known CI failure signatures — via the public Sippy API", - "description_html": "List, search, and inspect Sippy Symptoms and Labels — known CI failure signatures — via the public Sippy API", - "meta": "" - }, - { - "name": "manage-labels", - "description": "Create, update, or delete Sippy job run Labels (the human-readable tags applied by Symptoms) via the authenticated Sippy API", - "description_html": "Create, update, or delete Sippy job run Labels (the human-readable tags applied by Symptoms) via the authenticated Sippy API", - "meta": "" - }, - { - "name": "manage-symptoms", - "description": "Create, update, or delete Sippy Symptoms — known CI failure signatures that automatically label job runs — via the authenticated Sippy API", - "description_html": "Create, update, or delete Sippy Symptoms — known CI failure signatures that automatically label job runs — via the authenticated Sippy API", - "meta": "" - }, { "name": "oc-auth", "description": "Helper skill to retrieve OAuth tokens from the correct OpenShift cluster context when multiple clusters are configured", @@ -873,9 +831,15 @@

ai-helpers

"meta": "" }, { - "name": "prow-job-analysis", - "description": "Use this skill when debugging a failed Prow CI job.", - "description_html": "Use this skill when debugging a failed Prow CI job.", + "name": "prow-job-analyze-install-failure", + "description": "Analyze OpenShift installation failures in Prow CI jobs by examining installer logs, log bundles, and sosreports. Use when CI job fails \"install should succeed\" tests at bootstrap, cluster creation or other stages.", + "description_html": "Analyze OpenShift installation failures in Prow CI jobs by examining installer logs, log bundles, and sosreports. Use when CI job fails "install should succeed" tests at bootstrap, cluster creation or other stages.", + "meta": "" + }, + { + "name": "prow-job-analyze-metal-install-failure", + "description": "Analyze OpenShift bare metal installation failures in Prow CI jobs using dev-scripts artifacts. Use for jobs with \"metal\" in name, for debugging Metal3/Ironic provisioning, installation, or dev-scripts setup failures. You may also use the prow-job-analyze-install-failure skill with this one.", + "description_html": "Analyze OpenShift bare metal installation failures in Prow CI jobs using dev-scripts artifacts. Use for jobs with "metal" in name, for debugging Metal3/Ironic provisioning, installation, or dev-scripts setup failures. You may also use the prow-job-analyze-install-failure skill with this one.", "meta": "" }, { @@ -885,9 +849,21 @@

ai-helpers

"meta": "" }, { - "name": "reevaluate-job-runs", - "description": "Retroactively re-run Sippy Symptom detection on completed Prow CI job runs to apply or preview failure Labels", - "description_html": "Retroactively re-run Sippy Symptom detection on completed Prow CI job runs to apply or preview failure Labels", + "name": "prow-job-analyze-test-failure", + "description": "Analyze failed Prow CI tests by inspecting test code, downloading artifacts, and optionally integrating must-gather cluster diagnostics for root cause analysis", + "description_html": "Analyze failed Prow CI tests by inspecting test code, downloading artifacts, and optionally integrating must-gather cluster diagnostics for root cause analysis", + "meta": "" + }, + { + "name": "prow-job-artifact-search", + "description": "Search, list, and fetch artifacts from Prow CI job runs stored in GCS using the gcloud CLI", + "description_html": "Search, list, and fetch artifacts from Prow CI job runs stored in GCS using the gcloud CLI", + "meta": "" + }, + { + "name": "prow-job-extract-must-gather", + "description": "Extract and decompress must-gather archives from Prow CI job artifacts, generating an interactive HTML file browser with filters", + "description_html": "Extract and decompress must-gather archives from Prow CI job artifacts, generating an interactive HTML file browser with filters", "meta": "" }, { diff --git a/plugins/ci/.claude-plugin/plugin.json b/plugins/ci/.claude-plugin/plugin.json index 6cd9409dc..1914a2bc5 100644 --- a/plugins/ci/.claude-plugin/plugin.json +++ b/plugins/ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "ci", "description": "Tools for working with OpenShift CI and analyzing Prow job results", - "version": "0.0.80", + "version": "0.0.81", "author": { "name": "github.com/openshift-eng" } diff --git a/plugins/ci/skills/analyze-disruption/SKILL.md b/plugins/ci/skills/analyze-disruption/SKILL.md index b876e1e31..6c9295c12 100644 --- a/plugins/ci/skills/analyze-disruption/SKILL.md +++ b/plugins/ci/skills/analyze-disruption/SKILL.md @@ -37,7 +37,7 @@ The user will provide: - Validate at least one URL is provided 2. **Parse each URL** to extract bucket path, job name, and build ID - - Use the same URL parsing logic as the "prow-job-analysis" skill + - Use the same URL parsing logic as the "prow-job-artifact-search" skill - Accept both `prow.ci.openshift.org` and `gcsweb-ci` URL formats - Extract `build_id` and `job_name` from each URL @@ -128,7 +128,7 @@ gcloud storage cp "gs://test-platform-results/logs/{job_name}/{build_id}/artifac If `gcloud storage ls` doesn't find files, try the artifact search script: ```bash -python3 plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py \ +python3 plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py \ search "**/e2e-timelines_spyglass_*.json" ``` diff --git a/plugins/ci/skills/bulk-triage-regressions/SKILL.md b/plugins/ci/skills/bulk-triage-regressions/SKILL.md deleted file mode 100644 index 70b26dd20..000000000 --- a/plugins/ci/skills/bulk-triage-regressions/SKILL.md +++ /dev/null @@ -1,284 +0,0 @@ ---- -name: bulk-triage-regressions -description: Use this skill for Component Readiness triage duty - holistically analyze and triage all untriaged regressions for a set of components in a view, clustering them into root-cause buckets ---- - -# Bulk Triage Regressions - -## Input - -``` -bulk-triage-regressions [--components comp1 comp2 ...] [--auto-triage] -``` - -Example: `bulk-triage-regressions 5.0-main --components Installer Unknown` - -## Description - -This skill implements the **Component Readiness triage duty workflow**: it fetches *all* untriaged regressions for a set of components in a view (e.g., `5.0-main`, components `Installer` and `Unknown`), analyzes them **as a batch**, clusters them into **root-cause buckets**, and then triages each bucket to a single JIRA bug (existing or new). - -This differs from `/ci:analyze-regression` (which analyzes a single regression in depth). Triage duty requires a **holistic view**, because: - -1. **Many regressions, few root causes.** One product bug commonly opens 5–30 regressions across variants (different platforms, arches, featuresets, upgrade modes) and across "wrapper" tests (`install should succeed: overall`, `: cluster bootstrap`, `: cluster creation`, `verify the cluster readiness and stability`, mass-failure tests, etc.). Analyzing regressions one-by-one wastes effort and risks filing duplicate bugs. Cluster first, deep-dive once per cluster. - -2. **Component attribution is often wrong.** Regressions in `Installer` and `Unknown` are catch-all attributions. A failed installation or bootstrap is frequently caused by a *specific* component — e.g., a monitoring operator failing to go available blocks cluster creation, an etcd slowness issue breaks bootstrap, an MCO bug degrades nodes during install. The Sippy component label tells you *which test failed*, not *whose bug it is*. The real owner must be determined from artifacts (cluster operator status, log bundle, operator logs), and the JIRA bug must be filed against the **actual owning component**, not Installer. - -Use this skill when doing triage duty for a view, or whenever a user asks to "look at all untriaged regressions from " rather than a single regression ID. - -## Implementation - -**Script invocation rules**: Run Python skill scripts directly and analyze their JSON output with your own reasoning (pass `--format json` where the script offers the flag; scripts without it, such as `list_regressions.py`, emit JSON by default). Do not pipe script output through inline Python one-liners. Do not suppress stderr: if a script exits non-zero or returns invalid/empty JSON, stop and surface the error — an authentication or API failure must never be mistaken for an empty inventory ("nothing to triage"). - -**Authentication**: Read steps (listing, fetching details, test runs, GCS artifacts) require no auth — Phases 1–3 and a read-only report must work without any credentials. Write credentials are validated **only when writes are going to happen**: with `--auto-triage`, validate both up front (so an expired token surfaces before hours of analysis); otherwise validate at the start of Phase 4, before the first write. Sippy writes (creating/updating triage records) require a Bearer token from the DPCR cluster (`api.cr.j7t7.p1.openshiftapps.com:6443`) — see the `oc-auth` skill and the token-extraction snippet in `/ci:analyze-regression`; check with an authenticated GET against `https://sippy-auth.dptools.openshift.org/api/component_readiness/triages` (200 vs 401/403). - -JIRA writes (filing bugs, `set-release-blocker`, `add-jira-triage-link`) additionally require the `JIRA_USERNAME` and `JIRA_API_TOKEN` environment variables (API token from https://id.atlassian.com/manage-profile/security/api-tokens); verify with an authenticated GET against `https://redhat.atlassian.net/rest/api/3/myself` (Basic auth, 200 vs 401/403). If a required write credential is missing or invalid, pause before Phase 4 and ask the user to fix it — the analysis and report so far remain valid and must still be presented. - -### Phase 1: Collect the full batch - -1. **Load CI context**: Read the files in `plugins/ci/references/` (`jobs.md`, `tests.md`, `sippy-apis.md`) for conventions on tests, jobs, and Sippy APIs. - -2. **Parse arguments**: - - `view`: required, e.g. `5.0-main` - - `--components`: component filter list, e.g. `Installer Unknown`. Matching is case-insensitive and hierarchy-aware: a filter matches the full component name or any ` / `-separated segment of it, so `Installer` also covers `Installer / openshift-installer`, and `Networking` covers `Networking / ovn-kubernetes`, `Networking / router`, and every other `Networking / *` component. If omitted, ask the user which components the duty covers. - - `--auto-triage`: if present, triage buckets without per-bucket confirmation when confidence is high (see Phase 4). Default is to present findings and confirm before writing. - -3. **List regressions** with the `list-regressions` skill: - - ```bash - python3 plugins/teams/skills/list-regressions/list_regressions.py \ - --view --components - ``` - - Keep only **open, untriaged** regressions (empty `triages` array), but note recently-triaged ones — they are prime candidates for absorbing untriaged siblings. - - **Closed regressions are out of scope — even when untriaged.** A regression whose `closed` field is set has already resolved itself; do not inventory it, cluster it, deep-dive it, or recommend retroactive triage for it. The duty batch consists solely of open untriaged regressions. Closed regressions may be *consulted* as evidence (e.g., a closed sibling that shares a root cause with an open bucket, or a closed sibling whose existing triage/JIRA an open bucket should reuse — see Pitfalls), but they must never appear as bucket members, action items, or "leftovers" in the report. The only exception is the explicit closed-set audit mode (`--audit-closed`, see below), which the user must request by name — it is never part of a normal duty run. - -4. **Build a batch inventory table**: For every **open** untriaged regression record: regression ID, test name, component/capability, variants (Platform/Arch/Network/Topology/FeatureSet/Upgrade), opened date, failure/run counts. Present this table to the user up front so the scope of the duty run is visible. Do not include closed regressions in the inventory. - -5. **Stale-triage sweep (mandatory) — a 100% triaged board can still hide live defects.** For every **open, already-triaged** regression, compare its `last_failure` against the state of its triage's JIRA: a regression whose bug is Closed/Verified/resolved but which has failed *after* the resolution date is an alarm, not a statistic. Its fresh failure window is either a failed fix or — more often — a **different cause hiding behind the old triage record**. For each such regression, verify the recent runs' signature (armed Sippy symptoms are a cheap first oracle: dry-run `reevaluate` the newest runs — label hits map windows to known causes instantly; unlabeled recent runs mean a new, uninvestigated cause) and add the correct triage(s) for the new window rather than trusting the stale one. A real sweep found three "triaged" metal regressions pointing at a Closed toolchain bug while actively failing daily from two entirely new causes (a build-cluster capacity outage and a same-day console regression) — invisible in every untriaged-count metric. - -6. **Long-lived wrapper regressions are cause *timelines*, not single buckets.** An install-wrapper regression that stays open for weeks accumulates causes: one real record carried four (toolchain era → operator-flap window → CRI-O node bug → console startup fatal), each window separately verified and separately triaged. When a previously-analyzed regression shows new `last_failure` dates, re-verify the new window from scratch — never assume the existing triage covers it. - -### Phase 2: Cluster into candidate buckets (cheap signals first) - -Before any deep log analysis, group regressions using signals already in hand: - -- **Same test, different variants** — almost always one bucket. -- **Same variant fingerprint, different tests** — e.g., `install should succeed: overall` + `: cluster creation` + `verify the cluster readiness and stability` all failing on `azure/amd64/techpreview` starting the same day is one bucket. Wrapper tests fail together. -- **Same opened date** — regressions opened the same day across components often share one payload-level cause. -- **Shared job runs** — fetch details for each regression (`fetch-regression-details` skill) and compare `job_runs` `prowjob_run_id`s. Regressions observed in the same failed runs are strong candidates for one bucket — but this is a clustering signal, not proof: the same run can contain independent defects or be a mass failure, so Phase 3 validation is still required. Also run the `fetch-related-triages` skill per regression; `same_last_failure` and `similarly_named_test` matches feed the clustering, and `triaged_matches` with confidence ≥5 immediately suggest an existing triage/bug for the whole bucket. -- **Symptom labels** — `fetch-regression-details` returns `label_summary` (per job) and `job_labels` (per run). A label shared by most failed runs of several regressions is a strong bucket signal, and labels are precise (human-written matchers over artifacts). An empty label list means nothing was *detected* — not that nothing is wrong, and often just that the runs were never swept: Phase 3 opens with an active dry-run reevaluation that closes this gap. -- **Mass-failure marker**: high `test_failures` counts in `job_runs` mean the regression is likely collateral of a bigger event, not an independent issue. - -Output of this phase: a **draft bucket list**, each bucket with member regression IDs, the shared fingerprint (test/variant/date/job-run overlap), and any candidate existing triage or JIRA bug. - -Treat buckets as hypotheses — Phase 3 must confirm or split them. Do not merge buckets merely because both are "install failures"; installs fail at different stages for unrelated reasons. - -### Phase 3: Deep-dive each bucket (confirm root cause and real owner) - -For each bucket, pick 2–5 representative failed job runs (spread across jobs/variants; include the newest) and analyze. 2–5 runs are sufficient **only when they yield a consistent result** (same error signature / failure stage across all of them). If the sample is mixed or unclear — different errors, different stages, or an inconclusive owner — **extend the sample to 10–20 runs** before concluding; a small ambiguous sample must never be the basis for splitting/merging a bucket or attributing an owner. - -**Symptom-catalog oracle first (cheap, decisive — run before any artifact dive or subagent dispatch).** The passive `label_summary` from Phase 2 only shows labels already applied by past sweeps; recent runs are typically unswept and show nothing. Actively probe the catalog instead: `POST https://sippy-auth.dptools.openshift.org/api/jobs/runs/reevaluate` with `{"prow_job_build_ids": [...], "dry_run": true}` (DPCR Bearer token) for 3–5 representative runs per bucket, spread across platforms. Reevaluation scans the runs' GCS artifacts server-side against every armed symptom matcher and returns `symptoms_matched` per run — one call can attribute an entire bucket to a known incident (e.g., a payload-blocker label firing on all sampled runs across all platforms) in seconds. A real duty run resolved its whole board with one dry-run call, while a parallel run on the same wave that skipped this step spent 264 shell commands and 7 subagents re-deriving the same known cause per bucket and timed out before writing its report. Matched symptoms are a *hypothesis with a named cause* — still verify per the steps below (the label names the incident; confirm the causal chain to the bucket's specific test) — but they dictate where to look first and usually collapse the deep-dive to a single confirmatory artifact read. No matches means the cause is not yet cataloged — proceed with the full evidence ladder. - -**Deep-dive at least one CI sample per bucket — always, even at high confidence.** A Sippy `triaged_matches` confidence of 10 or a same-day triaged sibling is a *hypothesis*, not a verdict: Sippy matches on test names and shared job runs, which produces conf=10 for the same test name across unrelated platforms and root causes (see Pitfalls). Before accepting any disposition — including "extend existing triage" — read the actual failure evidence for at least one representative run of the bucket (failure output at minimum; installer logs / artifacts for install wrappers) and confirm it matches the target triage's root cause. You have deeper analysis capabilities than Sippy's heuristics — use your own judgement on the raw evidence, and if it contradicts the Sippy categorization, trust the evidence and re-bucket. - -1. **Failure outputs**: `fetch-test-runs` skill with the bucket's test IDs and job run IDs — check whether error messages are consistent within the bucket. >90% same error is **strong evidence of** a single cause, not confirmation: wrapper tests and mass failures print identical error text for unrelated defects (e.g., "nodes not ready" covers disk, memory, and network deaths alike). Confirm with items 2–3 below before treating the bucket as one cause; inconsistent errors ⇒ split the bucket. - -2. **Job run context**: `fetch-job-run-summary` skill per representative run — is the regressed test isolated, part of a consistent co-failure set, or one of hundreds of random failures? For `Unknown`-component and mass-failure regressions this is where the *real* component reveals itself: read the names of the co-failing tests. - -3. **Install/bootstrap failures — mandatory artifact dig**: For any bucket whose tests include `install should succeed` (any stage) or bootstrap/cluster-readiness wrappers, invoke the `prow-job-analysis` skill per representative run. Do not stop at Sippy's generic "install failed" wrapper. From the GCS artifacts determine: - - **Failure stage**: infrastructure provisioning / bootstrap / cluster creation (operators rolling out) / stability window. - - **The blocking condition**: for cluster-creation failures, read `clusteroperators.json` (or the installer log's "Cluster operator X is not available" lines) and the failing operator's pod logs from the log bundle. For bootstrap failures, read the bootstrap log bundle (etcd, bootkube, release-image pulls). - - **Bootstrap-era logs are full of normal transients — validate every causal theory against the gathered end-state.** During any bootstrap, operator logs contain scary-looking messages that resolve on their own ("the server could not find the requested resource (post routes.route.openshift.io)" before openshift-apiserver is up, static pods "0 nodes at revision 0" early on). Quoting one of these as the root cause is only valid if the *final* cluster state agrees. **The end-state oracle for bootstrap-stage failures is `clusteroperators.json`, not pod existence**: check etcd's `StaticPodsAvailable`/revision status and openshift-apiserver's `APIServicesAvailable` directly — a bootstrap-timed-out run whose gather still shows `etcd Available=False: 0 nodes are active; 3 nodes are at revision 0` was blocked by etcd no matter what else looks broken. Do **not** infer "etcd/apiserver were fine" from workload pods running: during bootstrap the bootstrap-node etcd/apiserver serve the control plane, so pods schedule and run happily while cluster etcd never deploys (this exact inference mis-refuted a correct etcd attribution in a real duty run). A crash-looping container's `*_previous.log` is primary evidence for *that container's* failure — but before assigning ownership, check whether its error names a missing upstream dependency (e.g., router looping on `failed to list *v1.Route: the server could not find the requested resource` is a **victim** of the Route API being unregistered, not a router bug). - - **"Operators were not stable" — count condition transitions before classifying.** "Healthy at gather time" does NOT imply a harmless transient. Grep the install log for the operator's condition lines and look at `LastTransitionTime`/`DurationSinceTransition`: hundreds of transitions with `DurationSinceTransition=1s` across the stability window means the operator is **flapping continuously** (a sync-loop product bug that will never pass the stability check — permafail), whereas a single long `Progressing=True` window that eventually clears is a slow rollout (flaky race against the timeout). These two have opposite classifications and dispositions; a real duty run mislabeled a ~1 Hz flap (1600+ transitions in 30 minutes) as a "self-resolved slow rollout" by only looking at the gather-time state and one FailedMount event. - - **The real owner**: the component whose operator/pods are actually failing. Examples from past duty: "cluster creation failed" ⇒ monitoring operator degraded ⇒ **Monitoring** bug; "bootstrap failed" ⇒ `etcdserver: request timed out` on Azure ⇒ **etcd** bug; nodes degraded during install ⇒ **MCO** bug; quota/DNS/cloud-API errors ⇒ **ci-infra**, not a product bug at all. - - **Stability-window failures — "operator X not available" is a symptom, not a root cause.** For cluster-readiness/stability wrappers that fail on a ClusterVersion or ClusterOperator condition, two extra steps are mandatory before assigning an owner: - 1. **Recover the underlying condition message when the junit output is vague.** Outputs like `clusterversion not available: False` (empty reason) carry no cause. Run this literal check against the e2e step's `build-log.txt` (and, if present, the monitor-intervals JSON under the step's `artifacts/junit/`): - - ```bash - curl -s /build-log.txt | grep -oE '(Failing|Available|Degraded)=(True|False)[:,][^"\\]{0,160}' | sort | uniq -c | sort -rn | head - ``` - - The condition *messages* recovered this way (e.g., a controller error string) name the real culprit. This step is complete only when the report quotes the recovered message for every vague-output run. Never attribute such a run from its co-failing tests: co-failures on techpreview jobs are usually unrelated background noise, and correlation with them has produced wrong owners in past duty runs. - 2. **Ask *why* the operator went unavailable, not just *which* operator.** If everything is healthy at gather time, the wrapper caught a transient flap: read the operator's own pod log around the transition timestamps and find the trigger. Distinguish (a) the operator genuinely failing (⇒ product bug for that operator) from (b) a routine reconciliation/rollout triggered by a cluster mutation (config/secret change, node roll). A rollout of a single-replica deployment (e.g., image-registry on vSphere RWO storage) flaps Available by design. **A rollout-flap disposition must name the mutator, not stop at "the operator detected a configuration change":** quote the operator-log line identifying *which object changed* (grep the operator pod log for `object changed` / secret and config names and read the surrounding lines), then identify *who wrote it* — audit logs (verb `update`/`patch` on that object: user + userAgent) or the job's test-harness step logs (search them for the object name or the value being set). If a test step caused the mutation mid-run, the bucket is `test` owned by that suite, not a product bug against the flapping operator; filing "spurious rollouts" against the operator without naming the mutator is an incomplete deep-dive. - -4. **Test-interference buckets — read the offending test's source before describing its scope.** When the root cause is another test's or tool's behavior (a test creates pods/namespaces that break an invariant check, leaks resources, reboots nodes, etc.), do not infer *why* the failure is confined to certain variants from the regression's variant labels — correlation with a variant (techpreview, platform, upgrade mode) is not evidence of how the offending code is gated. Instead: - - Locate the offending code (`gh search code` in `openshift/origin` or the relevant repo for the test name, namespace prefix, or error string) and **read its skip/gate conditions** (`skipIf...`, `e2eskipper.Skipf`, feature-gate checks, platform checks, suite membership). - - State the *actual* gating in the report (e.g., "gated on baremetal platform via `skipIfNotBaremetal`"), and if that gating is broader than the regressed variants, say so — the regression's variant slice then reflects job-scheduling or sample-size effects, not the blast radius, and other variants are also at risk. - - Check the file's merge history (`gh api repos///commits?path=...`) — a merge date matching the regression onset both confirms the attribution and identifies the owning team. - - Never write "X-only" (techpreview-only, platform-only, arch-only) about a test or tool in the report or a bug unless the source code gating has been read and confirms it. - -5. **Onset and suspect PRs** (when the bucket has a crisp start date): follow the "Determine Regression Start Date" and "Identify Suspect PRs in Payload" procedures from `/ci:analyze-regression` (first failing run → payload tag via `fetch-prowjob-json` → `fetch-new-prs-in-payload` → up to 5 candidate PRs vetted with `gh`). A LIKELY PR both strengthens the bucket and tells you the owning component/repo. - -6. **Cross-check globally**: `fetch-test-report` skill (with `--no-collapse`) for the bucket's main test — confirms whether the issue is variant-specific or global, and surfaces `open_bugs` that may already cover the bucket. - -7. **Check Slack context (optional — only when Slack access is available)**: The TRT/release-oversight team discusses ongoing payload and CI issues in **#forum-ocp-release-oversight** (https://redhat.enterprise.slack.com/archives/C01CQA76KMX). Search/read the **last 14 days** of messages there for the bucket's signature (test name, error message, operator, platform, payload tag) — known payload-wide events, infra outages, and in-flight fixes are usually discussed there before triages/bugs exist, and a thread often names the owning team or an existing OCPBUGS ticket. If the agent has no Slack access (no Slack tooling/credentials), **omit this step entirely** — do not block or ask for access. - -After deep-dive, finalize buckets. **Depth is mandatory, not optional: no bucket may be finalized at LOW or MEDIUM confidence, and the duty run must not end with open "action items" like "needs artifact deep-dive" or "spot-check installer logs first".** When one regression's failed runs split into multiple distinct failure signatures, every signature must be root-caused independently — each may get its own triage record (a regression can carry several), and no run may be left labeled "ambiguous"/"unclear" in a bucket claimed at HIGH confidence: an unexplained run either gets dug into until it joins a signature, or the bucket's confidence is honestly downgraded and the digging continues. **Never bundle an unexplained sub-pattern into another pattern's triage "pragmatically" or "since the regression is open anyway"** — triage covers only the runs whose root cause it actually explains; attaching unexplained runs to it hides them from the next duty shift and mis-scopes the bug. If a sub-pattern remains unexplained, the whole bucket is not finalizable — keep digging. If confidence is not HIGH after the steps above, keep digging until it is — escalate through the evidence ladder yourself: raw failure outputs → job-run summaries → GCS artifacts (`prow-job-artifact-search`, install/test-failure analysis skills) → **audit logs** (grep for the failing object/namespace to identify the creating user and userAgent — this reliably resolves "who created this pod/namespace" questions) → junit timing correlation (what else ran in the same window) → suspect-PR vetting. Triage taking longer is acceptable; leaving an unexplained bucket is not. The only permitted low-confidence outcome is when the evidence is genuinely exhausted (artifacts expired, logs missing), and then the report must say exactly what was checked and what was missing. - -Additional finalization rules (each has caused a wrong disposition in a real duty run): - -- **"Resolved"/"stopped"/"no recurrence" claims require the full run list, not a sample.** Before classifying a signature as resolved or transient, enumerate *all* of the regression's `job_runs` by date and confirm the newest runs' signature. A signature absent from a 3–4 run sample of a 20-run regression is not evidence it stopped. And when a sub-test starts "passing" recently, verify recent runs actually **reach that stage**: a bootstrap failure cannot "self-resolve" while newer runs of the same job die earlier at infrastructure provisioning — the earlier failure masks the later stage, it does not fix it. -- **"Leave untriaged" is not a permitted disposition for a bucket with an identified root cause and owner.** If the deep-dive named the mechanism and the responsible component/suite, the bucket gets a triage (to an existing or new issue) — "collateral of noisy runs" is only a valid leftover justification when the failure has *no independent mechanism* (pure co-occurrence). A failure that is deterministically produced by another test's behavior has an independent mechanism and must be triaged as `test`. -- **Extend, don't duplicate.** When an existing triage record already covers the bucket's bug, extend that triage with the new regression IDs (`--triage-id`); do not create a second triage record pointing at the same JIRA. -- **Subagent outputs must be verified, not trusted.** If bucket deep-dives are delegated (subagents, parallel tasks), the orchestrator must check each returned bucket against this section's requirements before accepting it — in particular that the mandatory quotes are present (recovered condition messages for vague wrappers, mutator identification for rollout flaps, transition counts for stability failures, `*_previous.log` reads for crash-looping containers). A missing mandatory quote means the sub-analysis is incomplete and must be redone, regardless of how confident its prose sounds. Batch size is never a reason to skip mandatory steps. -- **Never end your turn to "wait" for dispatched subagents — the session ends the moment you stop.** There is no background execution across turns: an assistant message that ends with a status narration ("waiting for X analysis to complete") and no tool call terminates the run, and in CI the harness will tear the process down at that point. A real duty run burned 15 minutes and $12 of analysis and produced **no report** because the orchestrator ended its turn "waiting" for AWS/GCP subagents. Collect every subagent's result *within* the turn that needs it, and treat the duty report as a hard checkpoint: if the run were killed right after your current message, the report file must already exist on disk — write intermediate versions early and update them, rather than deferring all writing to a final step that may never come. - -Each bucket must have: -- Member regression IDs (re-check the untriaged list — new siblings may have opened during analysis) -- Root cause summary (one paragraph) and failure classification (permafail / flaky / resolved / recent) -- **Owning component** (may differ from the Sippy component — state both) -- Triage type: `product` / `test` / `ci-infra` / `product-infra` -- Disposition: existing triage to extend / existing JIRA to create a triage for / new JIRA needed / no action (resolved or pure infra noise — say so explicitly and leave untriaged only with justification) - -### Phase 4: Search for existing bugs, then triage each bucket - -For each bucket, before filing anything new: - -1. Check `triaged_matches` from `fetch-related-triages` (confidence ≥5 with an open JIRA is the default target) — but never act on a match, even conf=10, without the Phase 3 per-bucket CI-sample verification confirming the root cause actually matches. -2. Check `open_bugs` from the test report. -3. Search Jira for the root-cause signature (error message, operator name, `component-regression` label) in OCPBUGS against the **owning component** — the right bug may exist under Monitoring/etcd/MCO even though the regression sits under Installer. -4. **Component-scoped JIRA listing (mandatory — keyword search is not enough).** Owning teams describe defects in *developer* vocabulary that shares no keywords with the CI-side symptom (real duplicate: the CI symptom was "baremetal CO flaps Progressing / Applying metal3 resources", the existing bug was titled "Do not require metal3-static-ip-manager to progress in vmedia"). After determining the owning component, list its recent bugs regardless of keywords and *read the summaries*: - - ``` - project = OCPBUGS AND component = "" AND created >= -21d ORDER BY created DESC - ``` - - A bug whose creation date falls inside the bucket's failure window, on the owning component, is a duplicate candidate even with zero keyword overlap — open it and compare mechanisms. -5. **Owning-repo merge-history check (mandatory when onset or cessation is dated).** Query the owning repo for PRs merged around the bucket's onset *and* cessation dates: - - ```bash - gh pr list --repo / --state merged --search "merged:" --json number,title,mergedAt - ``` - - A merge at the **cessation** boundary is likely the fix — its `OCPBUGS-*` title prefix names the existing bug: triage to that bug instead of filing a new one. A merge at the **onset** boundary is a suspect trigger. The phrase "resolved by unidentified payload change" is banned from reports and bugs unless this check was run and came back empty — a real duplicate bug was filed because a signature that stopped on 07-21 was written off as "unidentified", while the owning repo had merged `OCPBUGS-xxxxx:`-titled fix at exactly that boundary. For **currently-live** breakage, also scan the owning repo's *newest* merges for revert PRs: a fresh `Revert "..."` title citing a TRT/OCPBUGS key hands you the trigger PR, the tracking ticket, and the expected recovery time in one query (a real console startup fatal was trigger-identified and triage-targeted entirely from its same-day revert's title). - -Then act (this is where `--auto-triage` applies; without it, confirm each bucket with the user): - -- **Extend existing triage**: `triage-regression` skill with `--triage-id` (additive merge is automatic; pass only the new IDs). -- **New triage to existing bug**: `triage-regression` skill with `--url`, `--type`, and a one-sentence `--description` (<120 chars). -- **New bug**: file with `/jira:create bug` (the `create` skill from the jira plugin) against the **owning component**, label `component-regression`, description per the bug-filing template in `/ci:analyze-regression` ("Prepare Bug Filing Recommendations" section: full test names in `{code}` blocks, test IDs, regression IDs, variants, error signature, Sippy test-details **UI** links for every member regression, suspect PRs). **Every JIRA issue or comment created by this workflow must end with an AI-attribution footer as a separate, visually marked block** — not a sentence buried in the text: place it after a divider, as its own paragraph or note panel, e.g. a `rule` followed by a `panel` (type `note`) in ADF containing "**AI-generated content:** This bug was filed by AI as part of Component Readiness triage duty. Please verify before acting on it." **Release blocker is conditional on the triage type and impact, not automatic**: mark the bug a release blocker (`set-release-blocker` skill) only for `product` bugs whose failures block or materially degrade blocking/informing payload jobs; `test` bugs (races, invariant-scan interference) and `ci-infra` issues (cloud capacity, registry outages) are **not** release blockers — state the blocker decision and its one-line justification in the report. Then create the triage record. -- Always finish a triage by running the `add-jira-triage-link` skill to put the triage URL into the JIRA description. The skill *appends* to the description — if the description ends with the AI-attribution footer, the appended link would land after it. After running the skill, verify the footer is still the final block; if not, move the footer back to the end of the description (or insert the triage link before the footer in the same update). -- While triaging each bucket, note whether its signature is symptom-worthy (crisp grep-able line in a durable artifact) — the Phase 5 report must carry a symptom proposal for every bucket where it is (see "Label the bucket's signature in Sippy"). - -With `--auto-triage`, only act autonomously when confidence is high: consistent error signature across the bucket, and either a confidence ≥5 triaged match or an unambiguous existing open bug. Buckets requiring a *new* bug, or with mixed signals, are always presented for confirmation. - -### Phase 5: Duty report - -Present a final report: - -1. **Inventory**: N untriaged regressions found → M buckets. -2. **Per bucket**: member regression IDs, root cause, owning component (vs. Sippy component), classification, evidence highlights (error signature, stage, representative run links), action taken (triage ID + JIRA link) or recommendation awaiting confirmation. -3. **Leftovers**: regressions deliberately left untriaged (resolved / one-off flake / inconclusive) with justification and what evidence would change the call. -4. **Proposed Sippy symptoms** (mandatory section, even if empty): for every bucket whose root cause has a crisp, grep-able single-line signature in a durable artifact, include a concrete symptom proposal per the "Label the bucket's signature in Sippy" section below — label name, matcher, file pattern, validation pair, retro-apply run set. Proposing costs nothing (creation still requires explicit user confirmation); a duty shift that root-caused a bucket and did not propose a symptom for an obviously grep-able signature has left cheap future-triage value on the table. If no bucket qualifies, say so and why (e.g., signature only visible via timing correlation, artifact expires, no unique string). -5. **Cross-cutting observations**: payload-wide events, infra instability windows, techpreview-only patterns — useful context for the next duty shift. -6. **Session usage** (when available): if the run is orchestrated by a harness that captures usage telemetry (e.g. the CI job appends a "Session usage" section with model, turns, token counts, and cost after the session ends), do not fabricate these numbers yourself — the model cannot observe its own final token totals mid-session. In interactive runs simply omit the section. - -### Label the bucket's signature in Sippy (propose always, create on confirmation) - -When a bucket has a crisp, grep-able artifact signature (an error string or log line unique to the root cause), define a Sippy **symptom** so future runs are labeled automatically and the signature can be applied retroactively. Symptoms have repeatedly paid for themselves in real duty runs: a validated symptom **adjudicated a disputed root-cause attribution** (two competing theories for the same runs — the matcher proved which one was present), and retro-applying two symptoms to one regression's 20 runs **machine-verified a 19-vs-1 bucket split**. Good candidates: recurring infra events that spawn regressions across many job families, single-occurrence product signatures worth a tripwire (n=1 bugs), and any signature that a "known recurring family" keeps regenerating. These are writes: **always present the proposed label/symptom definitions and the target run list to the user and get explicit confirmation first** — `--auto-triage` does *not* authorize them. - -0. **Check for existing coverage before creating anything.** List the current catalog (`GET .../api/jobs/labels`, `GET .../api/jobs/symptoms`) and search it for the bucket's signature — the label/symptom may already exist from the shift that first triaged the bug (a real duty run proposed a console-crash symptom whose label *and* both matchers already existed; the duplicate-key 400 from the label POST was what revealed it). A dry-run `reevaluate` of one bucket run is an equivalent cheap probe: if existing symptoms fire on the run, the bucket is already covered and nothing should be created. - -1. Create a label and symptom via the authenticated API (DPCR Bearer token, same as triage writes): `POST https://sippy-auth.dptools.openshift.org/api/jobs/labels` (`{"label_title": ..., "explanation": ...}` — the title becomes the ID), then `POST .../api/jobs/symptoms` (`{"summary": " - ", "matcher_type": "string|regex|file", "file_pattern": "", "match_string": ..., "label_ids": [...]}`). The symptom ID is derived from the summary; gzipped artifacts are decompressed transparently. To change a symptom, DELETE and re-POST (PUT does not reliably update). Reference the JIRA key in both the label explanation and the symptom summary. -2. **Beware benign-transient false positives — presence is not pathology.** Before finalizing a matcher, grep the target file in a *healthy* run: many error lines appear benignly during normal startup — sometimes once (a router logs one probe failure while warming up), sometimes **in bulk** (a healthy install's etcd-operator log contained a proposed match string 120 times as a normal bootstrap warm-up transient; the failed run differed only in that the line persisted to log-end). String/regex matchers cannot count occurrences or check persistence, so if the line can appear benignly at all, re-anchor to an artifact that only exists — or only contains the string — in the failure mode: a container's `*_previous.log` (only present after a crash/restart) instead of its current log, an events.json string that healthy runs never emit, or best of all the **gather-time end state** in `clusteroperators.json` (e.g., `StaticPodsAvailable: 0 nodes are active` appears there only when etcd truly never deployed — zero hits in healthy runs by construction, because gather runs after the failure window). -3. **Validate with a dry run** against one known-positive and one known-negative run — the best negative is a *near miss*: a failed run of the same job family with a different root cause (it catches over-broad matchers that a green run would not). `POST .../api/jobs/runs/reevaluate` with `{"prow_job_build_ids": [...], "dry_run": true}`. Proceed only if **both** outcomes are correct; otherwise refine the matcher and re-validate. Note: reevaluation runs *all* symptoms, so unrelated pre-existing labels may legitimately appear on your negative — check only that *your* symptom's matching is correct. -4. Then apply with `"dry_run": false` to the bucket's `prowjob_run_id`s. The API caps requests at 50 IDs, but reevaluation scans GCS artifacts server-side and large batches time out at the gateway (504) — use **batches of 5–10** with a retry, check each result's `status`, and stop and report on the first failed batch or any non-`success` result rather than continuing. -5. Match the **underlying error**, not a transient side effect: a matcher keyed on a crash/panic goes silently false-negative the moment a partial fix removes the crash while the defect persists. If the primary evidence lives in pod logs that sometimes fail to gather (dying clusters), add a fallback symptom against an artifact that survives (e.g., `pods.json` state/reason strings) mapped to the same label. When the same signature can surface under different step names (e.g., devscripts-driven vs plain IPI installs), create one symptom per file pattern, all mapped to the same label. - -## Closed-set audit mode (`--audit-closed`) — justify why every closed regression closed - -``` -bulk-triage-regressions --components ... --audit-closed -``` - -An explicitly-requested, **read-only** mode that answers a different question than triage duty: not "who owns this failure?" but **"why did this regression close, and can we prove it?"** Every closed untriaged regression must end the audit in exactly one of these justification classes: - -| Class | Meaning | Proof required | -|---|---|---| -| `fixed` | A fix merged; closure follows it | Fix PR/bug with merge/resolution date at or before the close boundary | -| `event-ended` | An infra/payload event ended | Event window bracketing the failures (bad payload, repo outage, cloud capacity, credentials) | -| `intermittent-cause-open` | Pass rate recovered but the defect is still open | Signature matched to an open bug; state explicitly that closure ≠ resolution | -| `collateral` | Closed with the window of a tracked sibling event | Signature or run overlap with the tracked event | -| `evidence-expired` | Run history and artifacts are gone | List exactly what was checked (regression details, GCS) and the statistical argument (age, sibling consistency) — flagged, never silently absorbed | - -No regression may be left as "unknown". `evidence-expired` is the only permitted terminal state without a mechanism, and it must be flagged in its own report section. - -### Scale technique (a closed set is 5–30x a duty batch — per-regression deep-dives do not scale) - -1. **Inventory and cluster first**: fetch details for all members (parallel, ~8 workers is safe for the details endpoint), then cluster by (test-family, platform, featureset, close-window). Expect heavy super-clusters — in a real audit, **one bad nightly payload explained 21% of the entire closed set** (installer stamped version "0.0" panicking on every platform for one day). Always look for same-day open/close waves across platforms before analyzing anything individually. -2. **Bypass the Sippy runs API for bulk signature extraction — it rate-limits (HTTP 429) far below audit volume, and backoff does not help at this scale.** Go to GCS directly (unthrottled, parallel-safe): - - *Install-family tests*: classify from the installer log (`artifacts/*/ipi-install-install*/build-log.txt`, devscripts equivalent on metal) with an ordered signature catalog (most-specific first). Seed the catalog from the view's known events and extend it with whatever the duty history has verified (ign-push 403s, cipherSuites/rev-0, toolchain download, quay pulls, registry.ci 5xx, quota, capacity, provisioning, per-operator stabilization). Guard against substring traps — a real run mis-binned 31 regressions because `lease` matched inside `release`. - - *Everything else*: parse the junit XML under the e2e step (`artifacts/*//artifacts/junit/junit_e2e*.xml`) with a real XML parser (regex over XML mis-matched ~95% of testcases in practice) and take the `` of the exact testcase. - - Sample the 2 newest failed runs per regression; extend only where signatures disagree. -3. **Match against the full triage catalog**: fetch all existing triage records once (`/api/component_readiness/triages`) and index their regressions by test name — a closed untriaged record is very often the untriaged sibling of a triaged one, which supplies the bug and the closure mechanism for free. -4. **Date-anchor every closure claim**: `fixed` requires the fix date ≤ close boundary (JIRA `resolutiondate`, `gh pr list --search "merged:"` on the owning repo); `event-ended` requires the last failing run to fall inside the event window. The Phase 4 component-scoped JIRA listing and merge-history checks apply here unchanged. -5. **Checkpoint long-running collection to disk and run it detached** (nohup + periodic JSON dumps): bulk GCS extraction takes tens of minutes, harness timeouts will kill foreground loops, and two concurrent writers on one results file destroy each other — one writer, atomic-ish checkpoints, verify counts after every stage. - -### Audit report structure - -1. Executive summary: category table (name, member count, one-line verdict). -2. Method: exactly which artifact paths / APIs were used and why. -3. Per-category narrative: mechanism, proof, closure explanation, and for `intermittent-cause-open` the explicit warning that siblings will reopen. -4. **Per-regression appendix — one row per member, no exceptions** (ID, test, platform/featureset, opened→closed, category, evidence sample). -5. Honest-limitations section listing every `evidence-expired` member. - -### Audit-specific pitfalls - -- **Sippy retention creates a hard evidence horizon**: regressions from the view's first tracking week may have no run history at all, and GCS artifacts expire (~3 months). Do not let the horizon silently shrink the audit — count and flag such members. -- **The view-baseline start date masquerades as a mass event**: many unrelated regressions "open" on the view's first tracking day. Check whether a suspicious same-day open wave is simply the earliest date in the dataset before hunting for a common cause. -- **"Closed" is not a verdict**: a closed regression whose signature maps to a still-open bug (`intermittent-cause-open`) is a prediction of future regressions — surface these in the summary so the next duty shift expects them. - -## Pitfalls (learned from real duty runs) - -- **Left = newest** in `pass_sequence` strings. Misreading direction inverts "regressed" vs "resolved". -- **Do not file bugs against Installer by default.** In practice a majority of `install should succeed` regressions in duty batches were owned by other components or were infra noise. The installer is the messenger. -- **`Unknown` component regressions** (e.g., `verify the cluster readiness and stability`, `verify all machines should be in Running state`) are wrappers; the co-failing tests and operator states identify the owner. -- **One bucket can span components**: Monitoring + Test Framework + Unknown + Installer regressions have all belonged to a single MCO bug. Don't let the component column fragment a bucket. -- **Techpreview variants** often fail for techpreview-only reasons (new feature gates); check whether the same job without techpreview passes before assuming a general regression. -- **A regression's variant slice is not the offending code's gating.** Real example: a QoS-invariant regression appeared only under `metal/techpreview`, and the report claimed the interfering SecurityPenetration tests were "techpreview-gated" — reading `penetration.go` showed they are gated on **baremetal platform**, so every metal job (any featureset) was exposed. Claims like "techpreview-only" or "platform-only" about a test/tool must come from its source-code skip/gate conditions (Phase 3 item 4), never from the variants Sippy happened to flag. -- **API vs UI URLs**: convert `test_details_url` to the `sippy-ng` UI form before putting it in bugs or reports, by replacing the base `https://sippy.dptools.openshift.org/api/component_readiness/test_details` with `https://sippy-auth.dptools.openshift.org/sippy-ng/component_readiness/test_details` (query parameters are identical). -- **Re-list before writing**: new regressions open continuously; refresh the untriaged list right before creating/updating triages so siblings opened mid-analysis are included. -- **Check closed/dropped siblings too.** A regression that looks novel is often a *new open instance* of a root cause whose earlier sibling was already triaged and has since dropped out of the active view (its regression closed). `fetch-related-triages` and a `--test-name` query without an open-only filter find these; reuse the existing triage/JIRA instead of filing a new bug. -- **Identical `prowjob_run_id` sets are the strongest clustering signal — but still not proof.** When two regressions were opened from literally the same job runs, treat them as one draft bucket, then confirm in Phase 3 that the failure outputs actually point at one cause: a single run can carry independent defects (two unrelated tests failing for unrelated reasons), and in mass-failure runs co-occurrence is largely coincidental. Only merge into one triage after the error signatures/artifacts agree. -- **The failing monitor/test is often just the messenger.** Example: pod-to-service and host-to-service connectivity tests (attributed to Networking) failed because the *test preparation* poller pods could not be created due to `etcdserver: request timed out` — an etcd-on-Azure product issue, not a networking bug. Always read the actual error text, including setup/preparation errors, before trusting the test's subject area. -- **"Operator not available" during a stability window is not automatically that operator's bug.** Real example: `verify the cluster readiness and stability` failed with "Cluster operator image-registry is not available" on vSphere; the draft disposition was a product bug against Image Registry. The operator log showed the trigger: the job's test harness replaced the cluster pull secret mid-run, the operator re-synced `installation-pull-secrets` and rolled its single-replica deployment — a by-design flap. Correct disposition: `test` bug against the harness. Before filing against a flapping operator, find the mutation that triggered the reconcile. -- **A vague wrapper message hides a specific controller error — extract it, don't guess.** Real example: runs failing with bare `clusterversion not available: False` were drafted as "UDN CRD breakage" because UDN tests co-failed in the same jobs. The monitor intervals actually showed `Failing=True: failed to list agentic runs: no matches for kind "AgenticRun"` — a CVO controller bug with a merged fix, unrelated to UDN. When the junit output has an empty reason, grep the build log/monitor intervals for the condition-change events and their messages before attributing. -- **Bootstrap-transient theories that contradict the end-state.** Real example: an Azure bootstrap timeout was attributed to an etcd-operator "cipherSuites not found" race, and a GCP one to "monitoring can't create Routes (Route API not registered)" — the initial refutation used `pods.json` ("workloads running ⇒ etcd fine") and pinned it on crash-looping routers instead. `clusteroperators.json` later proved the refutation wrong: etcd was at `0 nodes active / revision 0` in every run — the etcd theory was **correct**, the router loop was a victim (it looped on `failed to list *v1.Route`, i.e., the Route API was never registered), and the bug got mis-attributed to the router change for weeks. Two lessons: pods running proves nothing about cluster etcd during bootstrap, and a crash-looping component whose error names a missing upstream dependency is a victim, not an owner. -- **A validated Sippy symptom detects a signature, not a cause.** The router probe-loop symptom fired on 30/33 runs of the etcd bucket above — a true detection, because the loop really happens — yet triaging by that label alone attributed victim runs to the router bug. When a symptom's signature can be produced downstream of a different defect, its label text must say so, and a second symptom keyed on the *upstream* discriminator (here: the etcd cipherSuites/revision-0 evidence) should exist to adjudicate. -- **Flapping ≠ slow rollout, and a small sample ≠ cessation.** Real example: `not stable: [baremetal]` was classified "slow rollout, self-resolved" from 3 sampled runs that looked healthy at gather — the install log actually showed the operator re-transitioning `Progressing` every ~1 second (1600+ times in 30 minutes), a permafail sync-loop bug present in 19 of 20 runs including the newest. Count condition transitions and enumerate all run dates before choosing between flaky-transient and permafail. -- **Identifying a root cause and then leaving the regression untriaged is a contradiction.** Real example: a QoS-invariant failure was accurately root-caused to debug pods created by SecurityPenetration tests, then dispositioned "leave untriaged — mass-failure collateral". Deterministic test interference is an independent mechanism: it gets a `test` triage and a bug, regardless of how noisy the surrounding runs are. -- **No triage record ≠ no bug, and keyword search ≠ component search.** Real example: a CBO Progressing-flap bucket had no related triage and no keyword hits, so a new bug was filed — duplicating a 8-day-old bug on the owning component whose summary used only developer vocabulary ("static-ip-manager should be optional"), with the fix already merged at exactly the bucket's cessation date. The component-scoped JIRA listing and the owning-repo merge-history check (Phase 4 items 4–5) exist because both would have caught it; run them before every new-bug disposition, and treat an unexplained cessation as a strong hint that a fix already landed somewhere. -- **A negative grep for one error string does not make a signature "new" — run the symptom catalog before declaring a distinct mechanism.** Real example: a bootstrap failure flooding the etcd-operator log with `TargetConfigController missing env var values` (5,471×) was flagged as a new mechanism because a grep for the known bug's `getCipherSuites` error string found nothing — but a dry-run `reevaluate` of the run fired the existing OCPBUGS-94106 symptom, proving the flood was a *downstream* consequence of the same defect (empty observedConfig → cipherSuites lookup fails → EnvVarController publishes nothing → TargetConfigController starves). Known-cause bugs surface through several messenger strings; the armed symptom catalog encodes the reliable ones. Before dispositioning any signature as new/distinct, dry-run `reevaluate` a representative run and treat every label hit as a known-cause hypothesis to check. -- **Known recurring families**: some failure signatures come back shift after shift and usually already have a triage — e.g., etcd slowness / slow fdatasync on Azure masters, quay.io 502s / `ImagePullNeverCompletes` on metal jobs (ci-infra), transient cloud quota or DNS provisioning errors. Search existing triages for the signature before opening anything. -- **Similar symptom ≠ same bucket.** Two image-pull triages can coexist for different causes (e.g., a quay ci-infra outage vs. an MCO bootimage product bug). Match on the full signature — error text, platform, job family, timing — not just the headline symptom, and pick the triage whose root cause matches, not the first one found. -- **Triage type follows the root cause, not the component**: a flaky test with an external dependency is `test` even if it looks like a product failure; an etcd timeout that also hits customers is `product`; a registry outage is `ci-infra` even when it kills installs. -- **A closed/MODIFIED bug can still be the right triage target** when the failures predate the fix landing; check the fix-merge date against the newest failed run before dismissing it — but if failures continue after the fix, that's a failed fix (analysis_status -1000) and needs the bug reopened or a new one. -- **Sample runs only from the regression's own `job_runs` list.** Broad Sippy job-filter URLs ("all jobs failing test X") sweep in unrelated jobs and post-fix eras; analyzing "3 random jobs" from such a list has produced three confidently wrong root causes in one shift. Every run you cite must be a member of the regression (and predate any candidate fix). -- **A CI step pod that never started is a build-cluster problem, not a product one.** If the prow build log shows `Deleting pod that failed to start` or `pod pending for more than 1h: pod has not been scheduled` (with `0/N nodes are available: ... untolerated taints / unschedulable`), the OpenShift installer never ran — no product artifact exists to analyze. Classify as `ci-infra`, and name the build cluster from `prowjob.json` (`.spec.cluster`, e.g. build10) so the infra team knows where the capacity problem is. Trivial setup steps (rbac, hosted-loki) failing this way across a job family for days is a capacity outage worth its own tracking ticket. -- **`Unknown` conditions are not resource pressure.** When a kubelet stops posting status, *all* node conditions flip to `Unknown`, and the MCO controller logs `Reporting unready: ... OutOfDisk=Unknown` — this is a stale-status marker, not disk exhaustion. Only `DiskPressure/MemoryPressure=True` (or eviction/OOM events, df output, sosreport sar) is evidence of actual resource pressure. -- **Superficially different per-run mechanisms can share one upstream cause — and two defects can chain.** In one incident, runs died variously of maxPods saturation, memory livelock, and an OVS flow storm; all were downstream of a single namespace-deletion blockage, which also amplified an otherwise-transient network defect into node death. When run-level root causes look "completely different", look one level up (what leaked, what was stuck, what accumulated) before declaring them unrelated; and when a symptom vanishes without its component's fix merging, suspect an interacting defect that got fixed instead. - -## Arguments - -- ``: Component Readiness view name (e.g., `5.0-main`). Required. -- `--components`: Space-separated component name filters, case-insensitive and hierarchy-aware (e.g., `Installer` also matches `Installer / openshift-installer`; `Networking` matches `Networking / ovn-kubernetes`, `Networking / router`, ...). Required in practice for duty scoping. -- `--auto-triage`: Allow high-confidence buckets to be triaged without per-bucket confirmation. New bug filing always requires confirmation. -- `--audit-closed`: Read-only closed-set audit mode (see "Closed-set audit mode") — justifies why every closed untriaged regression closed. Mutually exclusive with normal duty triage and with `--auto-triage`; performs no writes. - -## See Also - -- Related Command: `/ci:analyze-regression` — single-regression deep dive; this skill orchestrates its techniques across a batch -- Related Skill: `create` (jira plugin) — file new JIRA bugs via `/jira:create bug` (`plugins/jira/skills/create/SKILL.md`) -- Related Skill: `list-regressions` (teams plugin) — batch listing (`plugins/teams/skills/list-regressions/SKILL.md`) -- Related Skill: `fetch-regression-details` (`plugins/ci/skills/fetch-regression-details/SKILL.md`) -- Related Skill: `fetch-related-triages` (`plugins/ci/skills/fetch-related-triages/SKILL.md`) -- Related Skill: `fetch-test-runs` (`plugins/ci/skills/fetch-test-runs/SKILL.md`) -- Related Skill: `fetch-job-run-summary` (`plugins/ci/skills/fetch-job-run-summary/SKILL.md`) -- Related Skill: `prow-job-analysis` — GCS artifact analysis for install/bootstrap failures (`plugins/ci/skills/prow-job-analysis/SKILL.md`) -- Related Skill: `fetch-test-report` (`plugins/ci/skills/fetch-test-report/SKILL.md`) -- Related Skill: `triage-regression` (`plugins/ci/skills/triage-regression/SKILL.md`) -- Related Skill: `add-jira-triage-link` (`plugins/ci/skills/add-jira-triage-link/SKILL.md`) -- Related Skill: `set-release-blocker` (`plugins/ci/skills/set-release-blocker/SKILL.md`) -- Related Skill: `oc-auth` (`plugins/ci/skills/oc-auth/SKILL.md`) -- TRT Documentation: https://docs.ci.openshift.org/docs/release-oversight/troubleshooting-failures/ diff --git a/plugins/ci/skills/detect-permafail/SKILL.md b/plugins/ci/skills/detect-permafail/SKILL.md deleted file mode 100644 index a5a6fc820..000000000 --- a/plugins/ci/skills/detect-permafail/SKILL.md +++ /dev/null @@ -1,884 +0,0 @@ ---- -name: detect-permafail -description: Analyze consecutive job failures to determine if they represent a permafail pattern versus flaky failures ---- - -# Detect Permafail - -## When to Use This Skill - -Use this skill when you have 2-10 consecutive failures of the same job and need to determine if the failures represent a systematic/permanent failure (permafail) versus a flaky failure. This is critical for CI/CD pipeline analysis to distinguish between: - -- **Permafail**: A systematic failure affecting the same test(s) or infrastructure issue, detected by analyzing comparable runs (same failure type): - - 2-3 comparable runs: All must have the same failure (100% match) - - 4-5 comparable runs: At least 4 must have the same failure (80% match) - - 6-10 comparable runs: At least ceil(count × 0.7) must have the same failure (70% match) -- **Flaky**: Non-deterministic failures with varying root causes, or failures that don't meet the permafail thresholds - -## Prerequisites - -- Access to OpenShift CI Prow job artifacts via gcsweb URLs -- Access to the `Bash` tool for running `plugins/ci/scripts/classify-job-failures.py` -- Python with the `requests` package available for artifact fetching -- Knowledge of Prow artifact structure (from `fetch-prowjob-json` and `prow-job-artifact-search` skills) -- 2-10 URLs pointing to consecutive job failures (from Prow/OpenShift CI, ordered newest to oldest) -- Job name context to verify consistency across all failures -- PR information to provide context for analysis - -## Implementation Steps - -### Step 1: Validate Inputs - -**Standard Mode (URL-based):** - -Verify that all required inputs are present with expected types and constraints: -- `failure_urls`: Array of 2-10 strings matching Prow job URL pattern `https://prow.ci.openshift.org/view/gs////` where path may be `logs/`, `pr-logs/pull/`, or other GCS paths (must be consecutive runs, ordered newest to oldest) -- `job_name`: Non-empty string identifier of the job being analyzed -- `pr_info`: Object containing PR number (integer) and repository context (string) -- Each URL must match the Prow job URL pattern above - -Reject requests if: -- URLs count is less than 2 or more than 10 -- Job names don't match across all URLs (validate via prowjob.json metadata, not path position) -- PR context is missing - -Notes: -- URL ordering (newest first) is assumed but not validated - the frontend provides them in this order -- Job name validation should check prowjob.json metadata or verify the name appears in the extracted GCS path, not assume a fixed path position (presubmit URLs include org/repo/pr segments) - -**Offline/Eval Mode (Pre-normalized Signatures):** - -For testing or offline analysis, the skill can accept pre-normalized failure signatures directly instead of fetching from URLs. This mode skips Steps 2-4 (artifact fetching and classification) and proceeds directly to Step 5 (threshold analysis). - -Input format: -- `signatures`: Array of 2-10 pre-normalized signature objects (see Step 4 output format) -- Each signature must be either test_failure format `{type: "test_failure", url: "...", tests: [...], test_count: N}` or infra_failure format `{type: "infra_failure", url: "...", error: "...", error_hash: "..."}` - -When signatures are provided, skip to Step 5 and apply threshold logic directly. Use this mode only for evals or when signatures have been extracted by another tool. - -### Step 2: Classify Job Failures with Script - -For standard URL-based analysis, call the deterministic classifier script to fetch artifacts and produce normalized signatures: - -```bash -"$SKILL_DIR/../../scripts/classify-job-failures.py" --json-input '{ - "failure_urls": ["https://prow.ci.openshift.org/view/gs/..."], - "job_name": "pull-ci-openshift-origin-master-e2e-aws", - "pr_info": {"pr_number": 12345, "repository": "openshift/origin"} -}' -``` - -The script performs the artifact-based work that does not require AI judgment: - -1. Parses the full GCS path from each Prow URL. -2. Fetches `prowjob.json` from gcsweb. -3. Browses the artifacts directory. -4. Classifies each run as `test_failure` or `infra_failure` based on artifacts, not error text alone. -5. Extracts failing test names from junit/build logs or normalized infrastructure errors. -6. Returns the Step 4 normalized signature array. - -If the script exits non-zero, report its JSON error and do not continue to threshold analysis unless at least two valid signatures are available from another trusted source. - -### Step 3: Artifact Classification Rules - -The classifier follows these rules: - -- **TEST_FAILURE** when artifacts contain `openshift-e2e-test/`, `e2e-*`, `junit/`, junit XML files, `openshift-tests-*`, or `monitor-test-*`. -- **INFRA_FAILURE** only when artifacts show setup/build/gather output without test artifacts, or artifacts are empty/unavailable after a valid job fetch. -- MonitorTest failures count as test failures because MonitorTests run during e2e execution. -- Classification must be artifact-based. Do not classify from Prow descriptions or error messages alone. - -### Step 4: Use Normalized Failure Signatures - -Use the script output directly as the normalized signature array. - -**For test failures:** -```json -{ - "type": "test_failure", - "url": "job_url", - "tests": ["failing_test_name1", "failing_test_name2"], - "test_count": 2 -} -``` - -**For infrastructure failures:** -```json -{ - "type": "infra_failure", - "url": "job_url", - "error": "normalized_error_message", - "error_hash": "md5_of_error_message" -} -``` - -### Step 5: Compare Signatures for Permafail Pattern - -Apply permafail detection logic using **failure-type-based thresholds** - the denominator is the count of comparable runs (same failure type), not the matching group size: - -**Detection Thresholds (based on comparable run count):** -- 2-3 comparable runs: All must match (100% match required) -- 4-5 comparable runs: At least 4 must match (80% match required) -- 6-10 comparable runs: At least ceil(count × 0.7) must match (70% threshold) - -**For Test Failures:** -1. Count total test_failure signatures (this is the **denominator**) -2. Extract all unique test names from test_failure signatures -3. For each unique test name, count how many test_failure signatures contain it (this is the **numerator**) -4. Apply threshold based on the **denominator** (total test_failure count): - - If denominator is 2-3: numerator must equal denominator (100%) - - If denominator is 4-5: numerator must be ≥4 (80%) - - If denominator is 6-10: numerator must be ≥ceil(denominator × 0.7) (70%) -5. If ANY test name meets its threshold: **PERMAFAIL = TRUE** - - Report which test(s) met the threshold and their occurrence count -6. If no test meets the threshold: **PERMAFAIL = FALSE** - -Example: 8 test_failure signatures, "TestNetworkPolicy" appears in 6 → 6/8 = 75% → needs ceil(8×0.7)=6 → 6≥6 ✓ PERMAFAIL - -**For Infrastructure Failures:** -1. Count total infra_failure signatures (this is the **denominator**) -2. Extract error messages and group similar errors (exact match or >70% character similarity using normalized Levenshtein: `similarity = 1 - (levenshtein_distance(a, b) / max(len(a), len(b)))`. Treat two normalized errors as similar when `similarity > 0.70`. If both strings are empty, do not group them as similar; require a non-empty normalized error.) -3. For each error group, count how many infra_failure signatures contain it (this is the **numerator**) -4. Apply threshold based on the **denominator** (total infra_failure count): - - If denominator is 2-3: numerator must equal denominator (100%) - - If denominator is 4-5: numerator must be ≥4 (80%) - - If denominator is 6-10: numerator must be ≥ceil(denominator × 0.7) (70%) -5. If ANY error group meets its threshold: **PERMAFAIL = TRUE** - - Report the error message and occurrence count -6. If no error group meets the threshold: **PERMAFAIL = FALSE** - -Example: 5 infra_failure signatures - 3 have "operator X timeout", 2 have random errors → 3/5 = 60% → needs 4/5 (80%) → 3<4 → NOT PERMAFAIL - -**For Mixed Failure Types:** -1. Separate signatures by type (test_failure vs infra_failure) -2. For each type, count total signatures (the **denominator** for that type) -3. Apply threshold logic to each type independently: - - **Test failures**: If ≥2 test_failure signatures exist, check if any test appears frequently enough to meet the threshold based on test_failure count - - **Infra failures**: If ≥2 infra_failure signatures exist, check if any error appears frequently enough to meet the threshold based on infra_failure count -4. **Minimum runs requirement**: Need at least 2 runs of the same type to establish a permafail pattern. A single failure of either type is insufficient. -5. If either type meets the threshold AND has ≥2 runs: **PERMAFAIL = TRUE** - - Report the dominant pattern (the one that triggered permafail) - - Explain which runs contributed and which were ignored (e.g., "All 4 runs that reached e2e tests failed on TestNetworkPolicy. 3 other runs failed during cluster setup and are not relevant to this test failure pattern.") -6. If neither type meets criteria: **PERMAFAIL = FALSE** - -**Example Scenario 1: PERMAFAIL (4 test, 3 infra):** -- 7 total runs provided -- 3 runs: infra_failure (cluster creation failed) -- 4 runs: test_failure (all failing on TestNetworkPolicy) - -**Analysis:** -- Test failures: 4 runs → use 4/5 threshold (80%) → need 4 matching → have 4/4 (100%) ✓ PERMAFAIL -- Infra failures: 3 runs → use 3/3 threshold (100%) → different errors → NOT permafail -- Verdict: **PERMAFAIL = TRUE** (test failure group met criteria) -- Reason: "All 4 runs that reached e2e tests failed on TestNetworkPolicy (100% match). 3 additional runs failed during infrastructure setup and are not relevant to this test failure pattern." - -**Example Scenario 2: NOT PERMAFAIL (1 test, 6 diverse infra):** -- 7 total runs provided -- 6 runs: infra_failure (each with different errors: cluster creation timeout, AWS quota, network issue, pod eviction, storage failure, DNS timeout) -- 1 run: test_failure (failed on TestNetworkPolicy) - -**Analysis:** -- Test failures: 1 run → INSUFFICIENT (need minimum 2 runs to establish pattern) -- Infra failures: 6 runs with all different errors → no single error appears ≥5 times → NOT permafail -- Verdict: **PERMAFAIL = FALSE** -- Reason: "Only 1 out of 7 runs reached e2e tests and failed on '[sig-arch] daemonset cni-sysctl-allowlist-ds maxUnavailable requirement'. The other 6 runs failed during infrastructure setup, each with different unrelated issues (no consistent infra pattern). A single test failure is insufficient to establish a permafail pattern - we need multiple runs with consistent failures to confirm systematic breakage." - -### Step 6: Generate Verdict and Return JSON - -Construct the final response object with: -- Boolean verdict: `permafail` (true/false) -- Reason string explaining the determination. Always include explicit slash-format ratios for the dominant pattern and strongest non-matching pattern, such as `7/10`, `4/4`, `5/6`, or `2/10`. Do not write only `7 of 10`; include `7/10`. -- Complete failure signatures array -- Common tests array (if applicable and permafail=true) -- Confidence score as a JSON number from 0.0 to 1.0, representing confidence in the final verdict, not the raw match ratio. Never use strings such as `"high"`, `"medium"`, or `"0.95"`. For example, a clearly non-permafail `2/10` pattern can still have confidence around 0.70 because the verdict is well supported. -- Stable `failure_type` value: - - `test_failure` when the final verdict is driven by test-failure analysis, or only test failures are present - - `infra_failure` when the final verdict is driven by infrastructure-failure analysis, or only infrastructure failures are present - - `mixed` when both test and infrastructure failures are present and no single type meets the permafail threshold -- `match_ratio`: Slash-format string for the dominant pattern, such as `"7/10"`, `"4/4"`, `"5/6"`, or `"2/10"`. -- `threshold_required`: Integer count required for the comparable-run group to qualify as permafail. -- `matching_runs`: Integer numerator for the dominant or strongest pattern. -- `comparable_runs`: Integer denominator for the comparable-run group. - -The final JSON must be machine-checkable. The ratio-bearing fields must be top-level fields, not nested under an analysis object. Use this exact style: - -```json -{ - "permafail": true, - "confidence": 0.95, - "reason": "7/10 test_failure runs failed TestNetworkPolicy, meeting the required 7/10 threshold.", - "failure_type": "test_failure", - "match_ratio": "7/10", - "matching_runs": 7, - "comparable_runs": 10, - "threshold_required": 7, - "signatures": [] -} -``` - -For a mixed non-permafail, use `failure_type: "mixed"` and include the word `insufficient` in the reason when one failure type has fewer than 2 comparable runs. - -Do not set `confidence` equal to the match percentage. Confidence means confidence in the verdict: - -- Clear non-permafail with complete data and a strongest pattern below threshold, such as `2/10`, should use `confidence: 0.70` or higher. -- Exact threshold permafail, such as `7/10` test failures or `5/6` infra failures, should use `confidence: 0.85` or higher. -- All comparable runs matching, such as `4/4`, should use `confidence: 0.99`. -- Ambiguous or incomplete data should use lower confidence. - -## Output Format - -The skill returns a JSON object with this schema: - -```json -{ - "permafail": true, - "confidence": 0.95, - "reason": "3/3 test_failure runs show the same failing test 'test_node_scale' - consistent permanent failure", - "failure_type": "test_failure", - "match_ratio": "3/3", - "matching_runs": 3, - "comparable_runs": 3, - "threshold_required": 3, - "signatures": [ - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "tests": ["test_node_scale"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "tests": ["test_node_scale"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "tests": ["test_node_scale"], - "test_count": 1 - } - ], - "common_tests": ["test_node_scale"] -} -``` - -### For Infrastructure Failure (permafail=true) - -```json -{ - "permafail": true, - "confidence": 0.92, - "reason": "All 3 runs fail at cluster creation with identical error: 'Insufficient quota for machine type n1-standard-4'", - "failure_type": "infra_failure", - "signatures": [ - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "error": "Insufficient quota for machine type n1-standard-4", - "error_hash": "a1b2c3d4e5f6g7h8" - }, - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "error": "Insufficient quota for machine type n1-standard-4", - "error_hash": "a1b2c3d4e5f6g7h8" - }, - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/bucket/logs/...", - "error": "Insufficient quota for machine type n1-standard-4", - "error_hash": "a1b2c3d4e5f6g7h8" - } - ] -} -``` - -### For Non-Permafail (mixed or varying failures) - -```json -{ - "permafail": false, - "confidence": 0.88, - "reason": "Mixed failure types detected: 2/3 runs are test_failure, 1/3 is infra_failure. Inconsistent pattern indicates flaky behavior, not a systematic permafail. Test failures: 2/3 (threshold: 2/3). Infra failures: 1/3 (threshold: 1/3).", - "failure_type": "mixed", - "match_ratio": "2/3", - "matching_runs": 2, - "comparable_runs": 3, - "threshold_required": 2, - "signatures": [ - { - "type": "test_failure", - "url": "...", - "tests": ["test_networking"], - "test_count": 1 - }, - { - "type": "infra_failure", - "url": "...", - "error": "Pod evicted due to memory pressure", - "error_hash": "x1y2z3a4b5c6d7e8" - }, - { - "type": "test_failure", - "url": "...", - "tests": ["test_storage", "test_deployment"], - "test_count": 2 - } - ] -} -``` - -## Failure Signature Format - -### Test Failure Signature - -```json -{ - "type": "test_failure", - "url": "string (job URL)", - "tests": ["array", "of", "failing_test_names"], - "test_count": "integer (length of tests array)" -} -``` - -**Fields:** -- `type`: Always "test_failure" -- `url`: The Prow job URL for this run -- `tests`: Array of test names extracted from failure logs (deduplicated) -- `test_count`: Count of unique failing tests - -### Infrastructure Failure Signature - -```json -{ - "type": "infra_failure", - "url": "string (job URL)", - "error": "string (normalized error message)", - "error_hash": "string (MD5 hash of normalized error)" -} -``` - -**Fields:** -- `type`: Always "infra_failure" -- `url`: The Prow job URL for this run -- `error`: Normalized error message with timestamps and build IDs removed -- `error_hash`: MD5 hash for fast similarity comparison - -## Permafail Detection Logic - -### Test Failure Logic - -Apply the threshold rules from Step 5 based on the number of test_failure signatures (the comparable run count): - -1. Count total test_failure signatures (this is N, the **denominator**) -2. Collect all unique test names from test_failure signatures -3. For each unique test name, count how many test_failure signatures contain it (the **numerator**) -4. Check if any test meets the threshold: - - **N=2-3**: Test must appear in ALL N runs (100% match required) - - **N=4-5**: Test must appear in ≥4 runs (80% match required) - - **N=6-10**: Test must appear in ≥ceil(N × 0.7) runs (70% match required) -5. **Permafail = TRUE** if ANY test meets the threshold -6. Set confidence based on match strength: - - 0.99 if all test_failure runs have identical test set - - 0.85 or higher if threshold is met exactly (e.g., 3/3, 4/5, 7/10) - - 0.92 if threshold is exceeded (e.g., 5/5, 8/10) - - 0.70 or higher if no test meets threshold but the non-permafail verdict is clear -7. In the `reason`, include the strongest test ratio in slash form, for example `7/10 test_failure runs failed TestNetworkPolicy` or `2/10 test_failure runs failed TestNetworkPolicy`. -8. Include top-level `match_ratio`, `matching_runs`, `comparable_runs`, and `threshold_required` fields for the strongest test pattern. - -### Infrastructure Failure Logic - -Apply the threshold rules from Step 5 based on the number of infra_failure signatures (the comparable run count): - -1. Count total infra_failure signatures (this is N, the **denominator**) -2. Extract error messages and group similar errors (exact hash match or >70% string similarity) -3. For each error group, count how many infra_failure signatures contain it (the **numerator**) -4. Apply threshold based on **N** (total infra_failure count, not the error group size): - - **N=2-3**: Error must appear in ALL N runs (100% required) - - **N=4-5**: Error must appear in ≥4 runs (80% required) - - **N=6-10**: Error must appear in ≥ceil(N × 0.7) runs (70% required) -5. **Permafail = TRUE** if ANY error group meets its threshold - - Example: 5 total infra_failure signatures, 3 with "operator X timeout", 2 with random errors - - Denominator N=5 → needs 4/5 (80%) → 3/5 = 60% < 80% → NOT PERMAFAIL -6. Set confidence based on match strength: - - 0.99 if all infra_failure runs have identical error hash - - 0.85 or higher if the threshold is met exactly - - 0.92 if threshold is met with >80% string similarity - - 0.88 if threshold is met with >70% string similarity - - 0.70 or higher if no error group meets threshold but the non-permafail verdict is clear -7. In the `reason`, include the strongest infra ratio in slash form, for example `5/6 infra_failure runs share operator authentication timeout` or `1/6 infra_failure runs share the strongest error`. -8. Include top-level `match_ratio`, `matching_runs`, `comparable_runs`, and `threshold_required` fields for the strongest infra pattern. - -### Mixed Type Logic - -When both test_failure and infra_failure types are present: -1. **Analyze each group independently** using their respective thresholds -2. **Test failures**: Check if test_failure signatures have common failing tests -3. **Infra failures**: Check if infra_failure signatures have common errors -4. If **either group meets the permafail criteria**: **PERMAFAIL = TRUE** - - Report the pattern that triggered permafail (tests or infra) - - Explain the breakdown (e.g., "4 of 4 test runs failed on the same test; 3 other runs failed during setup") - - Set `failure_type` to the type that triggered the permafail: `test_failure` or `infra_failure` - - Use the triggering type's top-level ratio fields. For example, if 4/4 test failures match and infra failures are noise, set `failure_type: "test_failure"`, `match_ratio: "4/4"`, `matching_runs: 4`, `comparable_runs: 4`, and the test threshold. -5. If **neither group meets criteria**: **PERMAFAIL = FALSE** - - Reason: "No consistent pattern found in test failures or infrastructure failures" - - If both test_failure and infra_failure signatures are present, set `failure_type` to `mixed`, not `flaky` -6. In mixed cases, include slash-format ratios for the evaluated groups in the `reason`, such as `1/1 test_failure runs` and `1/6 infra_failure runs`. - -**Key Principle**: Infrastructure failures (cluster setup, resource quota, network issues) are **orthogonal** to test failures. A PR can have a systematic test failure (permafail) even if some runs fail during infrastructure setup. Analyze each type separately and detect permafails in either category. - -## Error Handling - -### Scenario 1: Artifact Fetch Failure - -If artifact fetching fails for a job URL: -- Return status: "error" -- Return error message with specific failure reason (network error, 404, timeout, etc.) -- Continue with remaining jobs when at least 2 jobs have not yet been attempted -- Do NOT return a permafail verdict when fewer than 2 jobs completed successfully - -**Response:** -```json -{ - "status": "error", - "error": "Failed to fetch artifacts for job: 404 Not Found at gcsweb URL", - "action": "verify_job_url_validity" -} -``` - -### Scenario 2: Timeout on Job Analysis - -If the classifier script times out or returns an incomplete result: -- Report the script error. -- Continue only if at least 2 valid signatures are available from another trusted source. -- If only 1 or 0 signatures are available: Return error. - -**Response:** -```json -{ - "status": "error", - "error": "Analysis timeout: Only 2 of 3 jobs analyzed successfully. Insufficient data for permafail determination.", - "completed_jobs": 2, - "action": "retry_with_single_job" -} -``` - -### Scenario 3: Invalid Job URLs - -If URL validation fails: -- Return status: "error" -- Return specific validation error message -- Do NOT attempt analysis - -**Response:** -```json -{ - "status": "error", - "error": "Invalid job URL format: 'url3' is not a valid Prow job URL", - "invalid_url": "url3", - "action": "provide_valid_urls" -} -``` - -### Scenario 4: Job Names Don't Match - -If the job_name parameter doesn't match the actual job names extracted from URLs: -- Return status: "error" -- Return the expected vs actual job names - -**Response:** -```json -{ - "status": "error", - "error": "Job name mismatch. Expected 'pull-ci-job-xyz' but found 'pull-ci-job-abc' in run 2", - "expected_job": "pull-ci-job-xyz", - "actual_job": "pull-ci-job-abc", - "action": "provide_matching_job_urls" -} -``` - -### Scenario 5: Insufficient Comparable Runs (Different Types) - -If analysis completes but there are insufficient comparable runs for permafail determination (e.g., 1 test_failure + 1 infra_failure): -- Return permafail: false -- Explain that neither failure type has ≥2 comparable runs -- This is NOT an error - the analysis succeeded but found no same-type pattern - -**Response:** -```json -{ - "permafail": false, - "confidence": 0.70, - "reason": "Insufficient comparable runs: 1/2 test failures, 1/2 infra failures. Cannot establish a permafail pattern with only one run of each type. Need at least 2 runs of the same failure type to determine if failures are systematic. Test: 1/2 (threshold: 2/2). Infra: 1/2 (threshold: 2/2).", - "failure_type": "mixed", - "match_ratio": "1/2", - "matching_runs": 1, - "comparable_runs": 2, - "threshold_required": 2, - "signatures": [ - { - "type": "test_failure", - "url": "...", - "tests": ["[sig-network] test"] - }, - { - "type": "infra_failure", - "url": "...", - "error": "cluster creation failed" - } - ] -} -``` - -### Scenario 6: Failure to Extract Failure Details - -If the classifier output doesn't contain expected failure information: -- Mark this run as "incomplete" -- Continue with other runs -- If ≥2 runs have valid failure data, proceed with analysis -- Otherwise, return error - -**Response:** -```json -{ - "status": "incomplete", - "warning": "Run 1 analysis incomplete: could not extract failure details", - "completed_jobs": 2, - "incomplete_jobs": 1, - "permafail": "unknown", - "recommendation": "Review job logs manually or retry analysis" -} -``` - -## Examples - -### Example 1: Permafail - Identical Failing Test - -**Input:** -```json -{ - "failure_urls": [ - "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234567", - "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234568", - "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234569" - ], - "job_name": "pull-ci-openshift-origin-master-e2e-aws", - "pr_info": { - "pr_number": 12345, - "repository": "openshift/origin" - } -} -``` - -**Subagent analysis results for all 3 runs:** -- Run 1: Failed tests = ["[sig-api] API discovery should provide capability information"] -- Run 2: Failed tests = ["[sig-api] API discovery should provide capability information"] -- Run 3: Failed tests = ["[sig-api] API discovery should provide capability information"] - -**Output:** -```json -{ - "permafail": true, - "confidence": 0.99, - "reason": "3/3 consecutive runs fail with identical test: '[sig-api] API discovery should provide capability information'. This is a systematic permanent failure.", - "failure_type": "test_failure", - "match_ratio": "3/3", - "matching_runs": 3, - "comparable_runs": 3, - "threshold_required": 3, - "signatures": [ - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234567", - "tests": ["[sig-api] API discovery should provide capability information"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234568", - "tests": ["[sig-api] API discovery should provide capability information"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/pull-ci-openshift-origin-master-e2e-aws/1234569", - "tests": ["[sig-api] API discovery should provide capability information"], - "test_count": 1 - } - ], - "common_tests": ["[sig-api] API discovery should provide capability information"] -} -``` - -### Example 2: Permafail Despite Mixed Failure Types - -**Input:** -```json -{ - "failure_urls": [ - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1111", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1112", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1113", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1114", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1115", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1116", - "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1117" - ], - "job_name": "periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node", - "pr_info": { - "pr_number": 3186, - "repository": "openshift/ovn-kubernetes" - } -} -``` - -**Subagent analysis results:** -- Run 1: Infrastructure failure = "Cluster creation timeout" (infra_failure) -- Run 2: Failed tests = ["[sig-network] Networking should provide connectivity"] (test_failure) -- Run 3: Infrastructure failure = "AWS quota exceeded" (infra_failure) -- Run 4: Failed tests = ["[sig-network] Networking should provide connectivity"] (test_failure) -- Run 5: Failed tests = ["[sig-network] Networking should provide connectivity"] (test_failure) -- Run 6: Infrastructure failure = "Cluster creation timeout" (infra_failure) -- Run 7: Failed tests = ["[sig-network] Networking should provide connectivity"] (test_failure) - -**Analysis:** -- 7 total runs: 3 infra_failures, 4 test_failures -- Test failures: 4/4 (100%) have identical failing test -- Infra failures: 3 runs, but different errors (not a permafail pattern in infra) -- Verdict: **PERMAFAIL = TRUE** based on test failure group - -**Output:** -```json -{ - "permafail": true, - "confidence": 0.99, - "reason": "All 4 runs that reached e2e tests failed on '[sig-network] Networking should provide connectivity' (100% match). 3 additional runs failed during infrastructure setup (cluster creation, AWS quota) and are not relevant to this test failure pattern. This is a systematic test failure caused by the PR changes.", - "failure_type": "test_failure", - "signatures": [ - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1111", - "error": "Cluster creation timeout", - "error_hash": "a1b2c3d4" - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1112", - "tests": ["[sig-network] Networking should provide connectivity"], - "test_count": 1 - }, - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1113", - "error": "AWS quota exceeded", - "error_hash": "e5f6g7h8" - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1114", - "tests": ["[sig-network] Networking should provide connectivity"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1115", - "tests": ["[sig-network] Networking should provide connectivity"], - "test_count": 1 - }, - { - "type": "infra_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1116", - "error": "Cluster creation timeout", - "error_hash": "a1b2c3d4" - }, - { - "type": "test_failure", - "url": "https://prow.ci.openshift.org/view/gs/..../logs/periodic-ci-openshift-release-main-ci-4.19-e2e-aws-upgrade-ovn-single-node/1117", - "tests": ["[sig-network] Networking should provide connectivity"], - "test_count": 1 - } - ], - "common_tests": ["[sig-network] Networking should provide connectivity"] -} -``` - -### Example 3: NOT Permafail - Insufficient Matching Infra Errors - -**Input:** 5 runs where 3 have identical operator installation failure, 2 have random infra issues - -**Subagent analysis results:** -- Run 1: Infrastructure failure = "operator authentication timeout waiting for operator to reach Available=True" (infra_failure) -- Run 2: Infrastructure failure = "AWS quota exceeded for instance type m5.xlarge" (infra_failure) -- Run 3: Infrastructure failure = "operator authentication timeout waiting for operator to reach Available=True" (infra_failure) -- Run 4: Infrastructure failure = "operator authentication timeout waiting for operator to reach Available=True" (infra_failure) -- Run 5: Infrastructure failure = "Pod evicted due to memory pressure on node ip-10-0-1-2" (infra_failure) - -**Analysis:** -- 5 total infra_failure signatures (denominator = 5) -- "operator authentication timeout" appears in 3 runs (numerator = 3) -- Threshold for N=5 (4-5 runs): need at least 4 matching (80% required) -- 3/5 = 60% < 80% → NOT PERMAFAIL - -**Output:** -```json -{ - "permafail": false, - "confidence": 0.70, - "reason": "Infrastructure failures are not consistent enough. 3/5 runs failed with 'operator authentication timeout waiting for operator to reach Available=True' (60% match), but with 5 infra failures at least 4/5 must match (80% required). The presence of 2 different random errors indicates flaky infrastructure rather than systematic failure.", - "failure_type": "infra_failure", - "match_ratio": "3/5", - "matching_runs": 3, - "comparable_runs": 5, - "threshold_required": 4, - "signatures": [ - { - "type": "infra_failure", - "url": "...", - "error": "operator authentication timeout waiting for operator to reach Available=True", - "error_hash": "a1b2c3d4" - }, - { - "type": "infra_failure", - "url": "...", - "error": "AWS quota exceeded for instance type m5.xlarge", - "error_hash": "e5f6g7h8" - }, - { - "type": "infra_failure", - "url": "...", - "error": "operator authentication timeout waiting for operator to reach Available=True", - "error_hash": "a1b2c3d4" - }, - { - "type": "infra_failure", - "url": "...", - "error": "operator authentication timeout waiting for operator to reach Available=True", - "error_hash": "a1b2c3d4" - }, - { - "type": "infra_failure", - "url": "...", - "error": "Pod evicted due to memory pressure on node ip-10-0-1-2", - "error_hash": "i9j0k1l2" - } - ] -} -``` - -### Example 4: Non-Permafail - No Consistent Pattern - -**Input:** 3 runs with different test failures - -**Subagent analysis results:** -- Run 1: Failed tests = ["[sig-network] networking should support networking"] (test_failure) -- Run 2: Failed tests = ["[sig-storage] storage should support volumes"] (test_failure) -- Run 3: Failed tests = ["[sig-api] API discovery should work"] (test_failure) - -**Output:** -```json -{ - "permafail": false, - "confidence": 0.70, - "reason": "No consistent failure pattern detected. 0/3 runs show matching tests - each of the 3 runs failed with different tests: networking, storage, API discovery. This indicates flaky/non-deterministic behavior rather than a systematic permafail.", - "failure_type": "test_failure", - "match_ratio": "0/3", - "matching_runs": 0, - "comparable_runs": 3, - "threshold_required": 3, - "signatures": [ - { - "type": "test_failure", - "url": "...", - "tests": ["[sig-network] networking should support networking"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "...", - "tests": ["[sig-storage] storage should support volumes"], - "test_count": 1 - }, - { - "type": "test_failure", - "url": "...", - "tests": ["[sig-api] API discovery should work"], - "test_count": 1 - } - ] -} -``` - -### Threshold Logic Validation Examples - -**Table-driven test cases demonstrating correct threshold behavior:** - -| Total URLs | Test Failures | Infra Failures | Matching Test Count | Verdict | Reason | -|------------|---------------|----------------|---------------------|---------|---------| -| 10 | 10 | 0 | 2 same test | NOT permafail | 2/10 = 20% < 70% threshold (need ≥7) | -| 10 | 10 | 0 | 7 same test | PERMAFAIL | 7/10 = 70% ≥ 70% threshold ✓ | -| 7 | 4 | 3 | 4 same test (in test bucket) | PERMAFAIL | 4/4 = 100% ≥ 80% threshold (4-5 runs need ≥4) ✓ | -| 7 | 1 | 6 | 1 test, all 6 infra diverse | NOT permafail | Only 1 test_failure (need ≥2), and infra errors are all different | -| 7 | 1 | 6 | 1 test, 5 same infra error | PERMAFAIL | 5/6 = 83% ≥ ceil(6×0.7)=5 (70% threshold) → infra bucket meets threshold ✓ | - -## Technical Details - -### Artifact Analysis Approach - -This skill analyzes Prow job artifacts directly using techniques from existing CI skills: - -1. **URL parsing** - Extract job name and build ID from Prow URLs -2. **Artifact fetching** - Use `plugins/ci/scripts/classify-job-failures.py` to fetch prowjob.json and browse artifacts -3. **Classification** - Use artifact-based detection (junit files, test directories) to classify failures -4. **Failure extraction** - Parse junit XML or build logs to extract test names or error messages - -This approach follows patterns from `fetch-prowjob-json`, `prow-job-artifact-search`, and `prow-job-analyze-test-failure` skills. - -### Script Execution Strategy - -The classifier script performs deterministic artifact analysis before AI threshold reasoning: - -```text -failure_urls ━━━ classify-job-failures.py ━━━ normalized signatures ━━━ Step 5 threshold analysis -``` - -**Benefits:** -- Keeps artifact fetching and parsing deterministic. -- Avoids spending agent reasoning on mechanical classification. -- Produces the same normalized signature schema used by offline eval cases. - -**Synchronization:** -- Run the script once with all failure URLs. -- Require at least 2 valid normalized signatures to proceed. -- Stop and report the script's JSON error if URL validation or artifact fetching fails. - -### Error Message Normalization - -Normalize infrastructure error messages for comparison: - -1. Remove timestamps: `2025-05-12T14:32:10Z` → "" -2. Remove build IDs: `build-12345-xyz` → "" -3. Remove resource names with IDs: `pod-abc123xyz` → "pod-*" -4. Remove request/limit values: Numbers in memory/CPU specs → "" -5. Keep: Error classification, core message, error type - -**Example normalization:** -```text -Input: "Pod evicted at 2025-05-12T14:32:10Z (build-12345): insufficient memory (512M < 1Gi required)" -Output: "Pod evicted: insufficient memory (* < *Gi required)" -``` - -### Confidence Scoring - -Confidence reflects how certain the permafail verdict is: - -- **0.99**: All 3 runs have identical failure signature (test names or error hashes) -- **0.95**: All 3 runs have ≥1 common failing test in test_failure -- **0.92**: All 3 runs have >80% similar error messages in infra_failure -- **0.85**: 2 of 3 runs share common failure or mixed types detected -- **0.70**: Insufficient data or ambiguous failure patterns - -Use confidence to determine remediation priority: -- Confidence ≥ 0.95: High priority permafail, block PR merge -- Confidence 0.85-0.94: Medium priority, warn but allow manual override -- Confidence < 0.85: Low confidence verdict, require manual review diff --git a/plugins/ci/skills/diagnose-job-run-symptoms/SKILL.md b/plugins/ci/skills/diagnose-job-run-symptoms/SKILL.md deleted file mode 100644 index c3597b5c3..000000000 --- a/plugins/ci/skills/diagnose-job-run-symptoms/SKILL.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: diagnose-job-run-symptoms -description: Explain which Sippy Symptoms and failure Labels apply to a Prow CI job run, in plain language, given only the Prow URL ---- - -# Diagnose Job Run Symptoms - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -This skill takes a Prow job run URL and explains which symptoms matched the run and what each applied label means — including the matched file and text. - -## When to Use This Skill - -Use this skill when: - -- A CI job failed and you want to know if it is a **known failure mode** before debugging it from scratch -- You are about to create a new symptom and want to check what already matched the run (see `manage-symptoms`) -- You want a plain-language explanation of the labels shown on a run in the Sippy UI or Spyglass - -## Prerequisites - -1. **Default mode** (already-applied labels): only network access to public GCS (`https://storage.googleapis.com`) and the public Sippy API (`https://sippy.dptools.openshift.org`) — **no authentication required**. - -2. **Deep mode** (`--deep`, server-side dry-run rescan): a Bearer token from the DPCR cluster. - - Must be logged into the DPCR cluster via `oc login` - - Cluster API: `https://api.cr.j7t7.p1.openshiftapps.com:6443` - - Use the `oc-auth` skill to obtain the token (see the token-acquisition snippet in `reevaluate-job-runs/SKILL.md`); prefer `export SIPPY_TOKEN=...` over `--token` — argv is visible in process listings - -3. **Python 3**: Python 3.6 or later, standard library only. - -## Implementation Steps - -### Step 1: Default mode — explain already-applied labels - -```bash -python3 plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py \ - "https://prow.ci.openshift.org/view/gs/test-platform-results/logs//" -``` - -For each applied label the summary output shows: the label title and explanation, the symptom that applied it (summary, matcher type, file pattern, match string), and the actual matched file and matched text from the run's artifacts. - -### Step 2: Deep mode — server-side rescan - -Use when the run predates the current symptom set or shows no labels. This asks Sippy to re-scan the run with `dry_run: true` — it **writes nothing** and reports what would match now: - -```bash -python3 plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py \ - "" --deep -``` - -Note: deep mode reports label IDs only — the matched file/text detail is only available in default mode (it comes from the GCS artifacts). - -### Step 3: Nothing matched? - -If default mode finds no labels, first suggest `--deep`: the run may simply never have been scanned, and default mode cannot distinguish that from "scanned, nothing matched". If a deep rescan also finds nothing and the user has identified the failure cause, guide them to the `manage-symptoms` skill to create a new symptom (and `reevaluate-job-runs` to apply it retroactively to past runs). - -**Arguments**: -- `prow_url`: Prow job run URL (`https://prow.ci.openshift.org/view/gs/...`, positional, required) - -**Options**: -- `--deep`: Server-side dry-run rescan via the reevaluate API (requires a token) -- `--token `: Bearer token from the oc-auth skill (only needed with `--deep`; optional if the `SIPPY_TOKEN` environment variable is set, which is preferred — argv is visible in process listings; `--token` takes precedence) -- `--format json|summary`: Output format (default: summary) - -## API Details - -**Default mode** uses the public GCS JSON API (no auth): - -- List label artifacts: `GET https://storage.googleapis.com/storage/v1/b/{bucket}/o?prefix={path}/artifacts/job_labels/` -- Download each object: `GET https://storage.googleapis.com/storage/v1/b/{bucket}/o/{object}?alt=media` - -**GCS `job_labels` artifact schema (verified live 2026-07):** each `*.json` object under `artifacts/job_labels/` contains one wrapped entry (a `label-summary.html` file is also present and skipped): - -```json -{ - "symptom_label_v1": { - "symptom": {"id": "...", "summary": "...", "matcher_type": "string", - "file_pattern": "...", "match_string": "...", "label_ids": ["..."]}, - "label": {"id": "...", "label_title": "...", "explanation": "..."}, - "file_match": "artifacts/.../nodes.json", - "text_match": "the exact text that matched" - } -} -``` - -The script also cross-references `GET /api/jobs/labels` and `GET /api/jobs/symptoms` on the public Sippy API to enrich entries with current explanations (embedded copies are snapshots from labeling time). - -**Deep mode** calls `POST https://sippy-auth.dptools.openshift.org/api/jobs/runs/reevaluate` with `{"prow_job_build_ids": [""], "dry_run": true}` — same request/response as the `reevaluate-job-runs` skill. - -## Error Handling - -- **Bad / non-Prow URL**: Must contain `/view/gs/` and end in a numeric build ID (exit 1 client-side). -- **GCS 404 or no `job_labels` artifacts**: The run may be too old (artifacts pruned) or was never scanned — suggest `--deep` to rescan server-side. -- **No labels found**: Not an error — the script prints guidance (try `--deep` first, then create a new symptom via `manage-symptoms`). -- **401/403 or HTML login page in deep mode**: Token missing/expired (the SSO proxy may return a login page instead of 401) — refresh via the `oc-auth` skill. -- **HTML gateway error page or non-JSON body in deep mode**: Transient gateway error (likely 504) — retry later. -- **Sippy API unreachable**: exit 1 with a clear message. - -**Exit Codes**: -- `0`: Diagnosis completed (including "no labels found") -- `1`: Validation, network, or API error - -## See Also - -- Related Skill: `list-symptoms` (browse/search the symptom and label catalogs) -- Related Skill: `manage-symptoms` (create a new symptom when nothing matched) -- Related Skill: `reevaluate-job-runs` (apply symptoms retroactively to past runs) -- Related Skill: `oc-auth` (token for deep mode) -- Related Skill: `prow-job-analyze-test-failure` (deeper manual failure analysis) diff --git a/plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py b/plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py deleted file mode 100644 index ae1fb7445..000000000 --- a/plugins/ci/skills/diagnose-job-run-symptoms/diagnose_job_run.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Explain which Sippy Symptoms/Labels apply to a Prow job run. - -Default mode reads already-applied labels from the run's public GCS artifacts -(no auth). Deep mode (--deep --token) asks Sippy to re-scan the run server-side -with dry_run=true and reports what would match now. - -GCS artifact schema (verified live 2026-07): each object under -artifacts/job_labels/*.json contains a single wrapped entry: - {"symptom_label_v1": {"symptom": {...}, "label": {...}, - "file_match": "", "text_match": ""}} -""" -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request - -READ_BASE = "https://sippy.dptools.openshift.org/api/jobs" -REEVALUATE_URL = "https://sippy-auth.dptools.openshift.org/api/jobs/runs/reevaluate" -GCS_API = "https://storage.googleapis.com/storage/v1/b" - - -def resolve_token(arg_token, env=None): - """Return the Bearer token from --token or the SIPPY_TOKEN env var. - - --token takes precedence over the environment variable. Prefer the env - var: command-line arguments are visible in process listings. - """ - env = os.environ if env is None else env - return arg_token or env.get("SIPPY_TOKEN") or None - - -def parse_prow_url(url): - marker = "/view/gs/" - if marker not in url: - raise ValueError("expected a Prow job URL containing '/view/gs/', got %r" % url) - url = url.strip().split("#", 1)[0].split("?", 1)[0] - rest = url.split(marker, 1)[1].strip("/") - parts = rest.split("/") - if len(parts) < 2 or not parts[-1].isdigit(): - raise ValueError("could not parse bucket/path/build_id from %r" % url) - return parts[0], "/".join(parts[1:]), parts[-1] - - -def normalize_label_entry(entry): - """Normalize one job_labels JSON entry to a flat match dict. - - Handles the observed wrapped schema ({"symptom_label_v1": {...}}) plus a - flat fallback (keys like label_id/id and symptom_id) for robustness. - """ - inner = entry.get("symptom_label_v1") if isinstance(entry, dict) else None - if isinstance(inner, dict): - label = inner.get("label") or {} - symptom = inner.get("symptom") or {} - return { - "label_id": label.get("id"), - "label": label, - "symptom_id": symptom.get("id"), - "symptom": symptom, - "file_match": inner.get("file_match"), - "text_match": inner.get("text_match"), - "raw": entry, - } - if isinstance(entry, dict): - return { - "label_id": entry.get("label_id") or entry.get("id"), - "label": entry.get("label") or {}, - "symptom_id": entry.get("symptom_id"), - "symptom": entry.get("symptom") or {}, - "file_match": entry.get("file_match"), - "text_match": entry.get("text_match"), - "raw": entry, - } - return {"label_id": None, "label": {}, "symptom_id": None, "symptom": {}, - "file_match": None, "text_match": None, "raw": entry} - - -def classify_response(body): - """Classify a deep-mode response body. Returns (parsed_json, error). - - Mirrors reevaluate_job_runs.py: an HTML body is either an SSO login page - (expired token — the proxy redirects instead of returning 401) or a - gateway error page; anything else must be valid JSON. - """ - if body.lstrip().startswith("<"): - if "log in" in body.lower(): - return None, ("got an SSO login page instead of JSON — token is " - "missing/expired; use the oc-auth skill to refresh it") - return None, "gateway returned an HTML error page (likely 504 timeout); retry later" - try: - return json.loads(body), None - except ValueError: - return None, "server returned a non-JSON response body" - - -def get_json(url): - with urllib.request.urlopen(url, timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) - - -def fetch_applied_labels(bucket, path): - """Return list of label entries from gs://bucket/path/artifacts/job_labels/*.json.""" - prefix = "%s/artifacts/job_labels/" % path - list_url = "%s/%s/o?prefix=%s" % (GCS_API, bucket, urllib.parse.quote(prefix, safe="")) - listing = get_json(list_url) - entries = [] - for obj in listing.get("items", []): - if not obj["name"].endswith(".json"): - continue - media = "%s/%s/o/%s?alt=media" % (GCS_API, bucket, urllib.parse.quote(obj["name"], safe="")) - data = get_json(media) - if isinstance(data, list): - entries.extend(data) - else: - entries.append(data) - return entries - - -def index_by_id(items): - return {i.get("id"): i for i in items} - - -def main(): - p = argparse.ArgumentParser(description="Diagnose which Sippy symptoms/labels apply to a job run") - p.add_argument("prow_url", help="Prow job run URL (https://prow.ci.openshift.org/view/gs/...)") - p.add_argument("--deep", action="store_true", - help="Server-side dry-run rescan via the reevaluate API (requires --token)") - p.add_argument("--token", help="Bearer token, required with --deep " - "(or set SIPPY_TOKEN env var, preferred; use oc-auth skill)") - p.add_argument("--format", choices=["json", "summary"], default="summary") - args = p.parse_args() - - token = resolve_token(args.token) - if args.deep and not token: - print("Error: --deep requires a token — pass --token or set the SIPPY_TOKEN " - "environment variable (preferred; use the oc-auth skill to obtain " - "one)", file=sys.stderr) - return 1 - - try: - bucket, path, build_id = parse_prow_url(args.prow_url) - except ValueError as e: - print("Error: %s" % e, file=sys.stderr) - return 1 - - try: - labels_catalog = index_by_id(get_json("%s/labels" % READ_BASE)) - symptoms_catalog = index_by_id(get_json("%s/symptoms" % READ_BASE)) - except (urllib.error.HTTPError, urllib.error.URLError, ValueError) as e: - print("Error: cannot reach Sippy API: %s" % e, file=sys.stderr) - return 1 - - report = {"build_id": build_id, "mode": "deep" if args.deep else "applied", "matches": []} - - if args.deep: - req = urllib.request.Request( - REEVALUATE_URL, - data=json.dumps({"prow_job_build_ids": [build_id], "dry_run": True}).encode("utf-8"), - headers={"Content-Type": "application/json", "Authorization": "Bearer %s" % token}, - method="POST") - try: - with urllib.request.urlopen(req, timeout=300) as resp: - body = resp.read().decode("utf-8") - except urllib.error.HTTPError as e: - hint = " (token expired? use oc-auth)" if e.code in (401, 403) else "" - print("Error: HTTP %d: %s%s" % (e.code, e.reason, hint), file=sys.stderr) - return 1 - except urllib.error.URLError as e: - print("Error: failed to connect to Sippy API: %s" % e.reason, file=sys.stderr) - return 1 - result, err = classify_response(body) - if err: - print("Error: %s" % err, file=sys.stderr) - return 1 - report["reevaluate_results"] = result.get("results", []) - for r in result.get("results", []): - for lid in r.get("labels_applied") or []: - report["matches"].append({"label_id": lid, - "label": labels_catalog.get(lid, {})}) - else: - try: - entries = fetch_applied_labels(bucket, path) - except (urllib.error.HTTPError, urllib.error.URLError, ValueError) as e: - print("Error: cannot read GCS artifacts for this run: %s" % e, file=sys.stderr) - return 1 - for entry in entries: - m = normalize_label_entry(entry) - # Enrich with the live catalogs (embedded copies may lack - # explanation text that was added after the run was labeled). - if m["label_id"] and labels_catalog.get(m["label_id"]): - m["label"] = labels_catalog[m["label_id"]] - if m["symptom_id"] and symptoms_catalog.get(m["symptom_id"]): - m["symptom"] = symptoms_catalog[m["symptom_id"]] - report["matches"].append(m) - - if args.format == "json": - print(json.dumps(report, indent=2)) - return 0 - - print("Symptom diagnosis for run %s (%s mode)" % (build_id, report["mode"])) - print("=" * 60) - if not report["matches"]: - print("No symptom labels found for this run.") - if not args.deep: - print("The run may never have been scanned (default mode cannot distinguish") - print("that from 'scanned, nothing matched') — try --deep --token \"$TOKEN\"") - print("for a server-side rescan with the current symptom set.") - print("If you have identified the failure cause, consider creating a new") - print("symptom with the manage-symptoms skill so future runs are auto-labeled.") - return 0 - for m in report["matches"]: - label = m.get("label") or {} - print("Label: %s — %s" % (m.get("label_id"), label.get("label_title", "(unknown label)"))) - if label.get("explanation"): - print(" Meaning: %s" % label["explanation"]) - sym = m.get("symptom") or {} - if sym: - print(" Matched symptom: %s (%s)" % (sym.get("id"), sym.get("summary"))) - print(" Rule: %s matcher on %s" % (sym.get("matcher_type"), sym.get("file_pattern"))) - if sym.get("match_string"): - print(" Pattern: %s" % sym.get("match_string")) - if m.get("file_match"): - print(" Matched file: %s" % m["file_match"]) - if m.get("text_match"): - print(" Matched text: %s" % m["text_match"].strip()) - print() - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/diagnose-job-run-symptoms/test_diagnose_job_run.py b/plugins/ci/skills/diagnose-job-run-symptoms/test_diagnose_job_run.py deleted file mode 100644 index ba00049cf..000000000 --- a/plugins/ci/skills/diagnose-job-run-symptoms/test_diagnose_job_run.py +++ /dev/null @@ -1,95 +0,0 @@ -import diagnose_job_run -import pytest -from diagnose_job_run import parse_prow_url, normalize_label_entry, classify_response - -def test_parse_standard_prow_url(): - url = ("https://prow.ci.openshift.org/view/gs/test-platform-results/logs/" - "periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn/1856789012345678848") - bucket, path, build_id = parse_prow_url(url) - assert bucket == "test-platform-results" - assert path == ("logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn/" - "1856789012345678848") - assert build_id == "1856789012345678848" - -def test_parse_pr_job_url(): - url = ("https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/" - "openshift_origin/29000/pull-ci-openshift-origin-master-e2e/1856789012345678848/") - bucket, path, build_id = parse_prow_url(url) - assert bucket == "test-platform-results" - assert build_id == "1856789012345678848" - assert path.startswith("pr-logs/pull/") and path.endswith(build_id) - -def test_rejects_non_prow_url(): - with pytest.raises(ValueError): - parse_prow_url("https://example.com/foo") - -def test_parse_url_with_query_string_and_fragment(): - url = ("https://prow.ci.openshift.org/view/gs/test-platform-results/logs/" - "some-job/1856789012345678848?tab=x#top") - bucket, path, build_id = parse_prow_url(url) - assert bucket == "test-platform-results" - assert build_id == "1856789012345678848" - assert path == "logs/some-job/1856789012345678848" - -def test_classify_response_login_page(): - _, err = classify_response("Please Log in to continue") - assert err and "oc-auth" in err - -def test_classify_response_other_html(): - _, err = classify_response("504 Gateway Time-out") - assert err and "HTML error page" in err - -def test_classify_response_non_json_plaintext(): - _, err = classify_response("service unavailable") - assert err and "non-JSON" in err - -def test_classify_response_valid_json(): - parsed, err = classify_response('{"results": [{"status": "success"}]}') - assert err is None - assert parsed["results"][0]["status"] == "success" - -WRAPPED_ENTRY = { - "symptom_label_v1": { - "symptom": {"id": "KubeletVersionSkew1355", "summary": "kubelet version skew 1.35.5", - "matcher_type": "string", - "file_pattern": "artifacts/*e2e*/gather-extra/artifacts/nodes.json", - "match_string": "\"kubeletVersion\": \"v1.35.3\"", - "label_ids": ["KubeletVersion1353"]}, - "label": {"id": "KubeletVersion1353", "label_title": "kubeletVersion 1.35.3", - "explanation": ""}, - "file_match": "artifacts/e2e-metal/gather-extra/artifacts/nodes.json", - "text_match": " \"kubeletVersion\": \"v1.35.3\",", - } -} - -def test_normalize_wrapped_symptom_label_v1(): - m = normalize_label_entry(WRAPPED_ENTRY) - assert m["label_id"] == "KubeletVersion1353" - assert m["symptom_id"] == "KubeletVersionSkew1355" - assert m["label"]["label_title"] == "kubeletVersion 1.35.3" - assert m["symptom"]["matcher_type"] == "string" - assert m["file_match"].endswith("nodes.json") - assert "v1.35.3" in m["text_match"] - assert m["raw"] is WRAPPED_ENTRY - -def test_normalize_flat_fallback(): - m = normalize_label_entry({"label_id": "InfraFailure", "symptom_id": "AWSAuth"}) - assert m["label_id"] == "InfraFailure" - assert m["symptom_id"] == "AWSAuth" - -def test_normalize_flat_fallback_id_key(): - assert normalize_label_entry({"id": "InfraFailure"})["label_id"] == "InfraFailure" - -def test_normalize_garbage_returns_empty_match(): - m = normalize_label_entry("not-a-dict") - assert m["label_id"] is None and m["symptom_id"] is None - - -def test_resolve_token_arg_wins_over_env(): - assert diagnose_job_run.resolve_token("argtok", {"SIPPY_TOKEN": "envtok"}) == "argtok" - -def test_resolve_token_falls_back_to_env(): - assert diagnose_job_run.resolve_token(None, {"SIPPY_TOKEN": "envtok"}) == "envtok" - -def test_resolve_token_none_when_unset(): - assert diagnose_job_run.resolve_token(None, {}) is None diff --git a/plugins/ci/skills/fetch-job-run-summary/SKILL.md b/plugins/ci/skills/fetch-job-run-summary/SKILL.md index 47ef5415c..7749059e2 100644 --- a/plugins/ci/skills/fetch-job-run-summary/SKILL.md +++ b/plugins/ci/skills/fetch-job-run-summary/SKILL.md @@ -142,4 +142,4 @@ python3 plugins/ci/skills/fetch-job-run-summary/fetch_job_run_summary.py 2030845 - Related Skill: `fetch-test-runs` - Fetch raw outputs for a specific test across multiple job runs - Related Skill: `fetch-regression-details` - Fetch Component Readiness regression details -- Related Skill: `ci:prow-job-analysis` - Comprehensive analysis of test and install failures in a Prow job +- Related Command: `/ci:analyze-prow-job-test-failure` - Deep analysis of test failures in a Prow job diff --git a/plugins/ci/skills/fetch-prow-job-runs/SKILL.md b/plugins/ci/skills/fetch-prow-job-runs/SKILL.md deleted file mode 100644 index c36bd8100..000000000 --- a/plugins/ci/skills/fetch-prow-job-runs/SKILL.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -name: fetch-prow-job-runs -description: List Prow job runs from the Sippy API using its real filter syntax — find run IDs by job name, variant, result, or time window ---- - -# Fetch Prow Job Runs - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -This skill finds Prow job **runs** (and their `prow_id` values) via the Sippy `/api/jobs/runs` endpoint. It exists because this API does **not** accept ad-hoc query parameters like `?job_name=` or `?period=` — those do not exist and will be silently ignored or rejected. Filtering is done through a JSON `filter` parameter with a fixed set of column fields and operators (listed below), and this skill's script builds that JSON with verified fields and operators so you never have to guess. - -## When to Use This Skill - -Use this skill when you need to: - -- Find run IDs (`prow_id`) to feed into `reevaluate-job-runs` after creating or updating a symptom -- Check how widespread a failure is (e.g. all failed runs of a job family in the last day) -- List recent runs matching a platform/network/etc. variant -- Get the Prow URL of recent runs of a given job - -## Prerequisites - -1. **Network Access**: The Sippy API must be accessible at `https://sippy.dptools.openshift.org` - - No authentication required -2. **Python 3**: Python 3.6 or later, standard library only - -## Implementation Steps - -Invoke the script with flags matching the question: - -```bash -script_path="plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py" - -# Metal-platform runs in the last 24 hours -python3 "$script_path" --release 5.0 --variant Platform:metal --since-hours 24 --format summary - -# Failed runs of jobs whose name contains e2e-metal -python3 "$script_path" --release 5.0 --job-contains e2e-metal --result F --format summary - -# Pipe run IDs into symptom reevaluation -python3 "$script_path" --release 5.0 --job-contains e2e-metal --result F --since-hours 48 --ids-only \ - | xargs python3 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py --dry-run -``` - -Flags: -- `--release ` (required): OpenShift release, e.g. `5.0` -- `--job-contains ` (repeatable): job name must contain each substring -- `--variant ` (repeatable): run must have each variant entry, e.g. `Platform:metal` -- `--result `: overall result code (`S` success, `F` failure, `n` infra failure — lowercase n) -- `--since-hours `: only runs newer than N hours ago -- `--filter-json `: escape hatch — raw JSON array of extra filter items merged with the generated ones -- `--limit `: max rows (default 100) -- `--ids-only`: print just `prow_id` per line (pipeline-friendly) -- `--format {json,summary}`: output format (default `json`) - -All filter items are combined with `AND`. - -## API Details - -### Endpoint - -```text -GET https://sippy.dptools.openshift.org/api/jobs/runs?release={release}&filter={urlencoded_json}&limit={n}&sortField=timestamp&sort=desc -``` - -No authentication required. - -### Filter JSON Shape - -```json -{"items": [{"columnField": "name", "operatorValue": "contains", "value": "e2e-metal"}], "linkOperator": "and"} -``` - -### Verified Column Fields and Operators - -Do NOT invent other query parameters or fields — only these are verified to work: - -| columnField | operatorValue | value | -|---|---|---| -| `name` | `contains` | job-name substring | -| `variants` | `has entry` | `Key:Value`, e.g. `Platform:metal` | -| `timestamp` | `>` / `<` | epoch **milliseconds** as a string | -| `overall_result` | `equals` | `S` (success), `F` (failure), `n` (infra failure — lowercase) | - -An unknown `columnField` yields HTTP 400 with a "column does not exist" message; the script surfaces that message verbatim. - -### Response Row Fields - -`{"rows": [...]}`; each row has `prow_id` (string), `job`, `url`, `variants[]`, `overall_result`, `failed`, `succeeded`, `infrastructure_failure`, `test_failures`, `timestamp` (epoch millis), `cluster`. - -## Error Handling - -### Case 1: Unknown Column (HTTP 400) - -```text -Error: HTTP 400 from Sippy API: {"code":400,"message":"could not filter or sort data: error updating query for filter ... column does not exist"} -``` - -Use only the column fields in the table above. Exits 1. - -### Case 2: Network Error - -```text -Error: failed to connect to Sippy API: [Errno -2] Name or service not known -``` - -Exits 1. Check connectivity to `sippy.dptools.openshift.org`. - -### Case 3: Empty Results - -No matching runs prints `[]` (JSON), `Total: 0` (summary), or nothing (`--ids-only`) and exits 0 — not an error. - -**Exit Codes:** -- `0`: Success (including empty results) -- `1`: Error (HTTP error, network error, invalid `--filter-json`) - -## Examples - -### Example 1: Metal Runs in the Last 24 Hours - -```bash -python3 plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py \ - --release 5.0 --variant Platform:metal --since-hours 24 --limit 5 --format summary -``` - -**Expected Output (excerpt):** -```text -n periodic-ci-openshift-release-main-nightly-5.0-e2e-metal-ovn-ha-cert-rotation-shutdown-5y-age-90d 2080684960133419008 1784908873000 -Total: 5 -``` - -### Example 2: Failed Runs of a Job Family - -```bash -python3 plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py \ - --release 5.0 --job-contains e2e-metal --result F --format summary -``` - -### Example 3: Feed Run IDs to Reevaluation - -```bash -python3 plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py \ - --release 5.0 --job-contains e2e-metal --since-hours 48 --ids-only \ - | xargs python3 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py --dry-run -``` - -## Notes - -- Results are sorted newest-first (`sortField=timestamp&sort=desc`) -- `timestamp` filter values must be epoch **milliseconds** (the script computes this for `--since-hours`) -- Repeatable flags AND together; there is no OR support in this script (use `--filter-json` if you need custom items) - -## See Also - -- Related Skill: `reevaluate-job-runs` (re-run symptom detection on the run IDs found here) -- Related Skill: `diagnose-job-run-symptoms` (explain which symptoms/labels apply to a run) -- Related Skill: `list-symptoms` (browse the symptom/label catalog) -- Related Skill: `fetch-regression-details` (another source of job run IDs, from regressions) diff --git a/plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py b/plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py deleted file mode 100644 index 0f556e6f0..000000000 --- a/plugins/ci/skills/fetch-prow-job-runs/fetch_prow_job_runs.py +++ /dev/null @@ -1,133 +0,0 @@ -"""List Prow job runs from the Sippy API using its filter syntax. - -The /api/jobs/runs endpoint takes a JSON "filter" query parameter — NOT -ad-hoc query params like ?job_name= or ?period= (those do not exist). -Unknown columnField values yield HTTP 400 "column does not exist"; that -message is surfaced verbatim. -""" -import argparse -import json -import sys -import time -import urllib.error -import urllib.parse -import urllib.request - -BASE_URL = "https://sippy.dptools.openshift.org/api/jobs/runs" - - -def since_millis(hours, now_ms=None): - """Return the epoch-milliseconds timestamp `hours` ago from now_ms.""" - if now_ms is None: - now_ms = int(time.time() * 1000) - return int(now_ms - hours * 3600 * 1000) - - -def build_filter(job_contains, variants, result, since_ms, extra_items): - """Build the Sippy filter dict. All items are ANDed together.""" - items = [] - for substr in job_contains: - items.append({"columnField": "name", "operatorValue": "contains", "value": substr}) - for variant in variants: - items.append({"columnField": "variants", "operatorValue": "has entry", "value": variant}) - if result is not None: - items.append({"columnField": "overall_result", "operatorValue": "equals", "value": result}) - if since_ms is not None: - items.append({"columnField": "timestamp", "operatorValue": ">", "value": str(since_ms)}) - items.extend(extra_items) - return {"items": items, "linkOperator": "and"} - - -def extract_ids(rows): - return [row["prow_id"] for row in rows] - - -def fetch_runs(release, filter_dict, limit): - params = { - "release": release, - "filter": json.dumps(filter_dict), - "limit": str(limit), - "sortField": "timestamp", - "sort": "desc", - } - url = "%s?%s" % (BASE_URL, urllib.parse.urlencode(params)) - try: - with urllib.request.urlopen(url, timeout=60) as resp: - body = resp.read().decode("utf-8") - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode("utf-8", errors="replace") - except (OSError, ValueError): - detail = "" - raise RuntimeError("HTTP %d from Sippy API: %s" % (e.code, detail.strip())) from e - except urllib.error.URLError as e: - raise RuntimeError("failed to connect to Sippy API: %s" % e.reason) from e - try: - data = json.loads(body) - except ValueError as e: - raise RuntimeError("server returned a non-JSON response body") from e - return data.get("rows") or [] - - -def print_summary(rows): - for row in rows: - print("%s %s %s %s" % ( - row.get("overall_result", "?"), - row.get("job", "?"), - row.get("prow_id", "?"), - row.get("timestamp", "?"))) - print("Total: %d" % len(rows)) - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="List Prow job runs from Sippy using proper filter syntax") - parser.add_argument("--release", required=True, help="OpenShift release, e.g. 5.0") - parser.add_argument("--job-contains", action="append", default=[], - help="substring the job name must contain (repeatable, ANDed)") - parser.add_argument("--variant", action="append", default=[], - help="variant entry like Platform:metal (repeatable, ANDed)") - parser.add_argument("--result", help="overall_result code: S=success, F=failure, n=infra failure") - parser.add_argument("--since-hours", type=float, - help="only runs newer than N hours ago") - parser.add_argument("--filter-json", - help="raw JSON array of extra filter items, merged with generated ones") - parser.add_argument("--limit", type=int, default=100) - parser.add_argument("--ids-only", action="store_true", - help="print only prow_id values, one per line") - parser.add_argument("--format", choices=["json", "summary"], default="json") - args = parser.parse_args(argv) - - extra_items = [] - if args.filter_json: - try: - extra_items = json.loads(args.filter_json) - except ValueError as e: - print("Error: --filter-json is not valid JSON: %s" % e, file=sys.stderr) - return 1 - if not isinstance(extra_items, list): - print("Error: --filter-json must be a JSON array of filter items", file=sys.stderr) - return 1 - - since_ms = since_millis(args.since_hours) if args.since_hours is not None else None - filter_dict = build_filter(args.job_contains, args.variant, args.result, - since_ms, extra_items) - try: - rows = fetch_runs(args.release, filter_dict, args.limit) - except RuntimeError as e: - print("Error: %s" % e, file=sys.stderr) - return 1 - - if args.ids_only: - for prow_id in extract_ids(rows): - print(prow_id) - elif args.format == "summary": - print_summary(rows) - else: - print(json.dumps(rows, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/fetch-prow-job-runs/test_fetch_prow_job_runs.py b/plugins/ci/skills/fetch-prow-job-runs/test_fetch_prow_job_runs.py deleted file mode 100644 index a5dacadb1..000000000 --- a/plugins/ci/skills/fetch-prow-job-runs/test_fetch_prow_job_runs.py +++ /dev/null @@ -1,69 +0,0 @@ -from fetch_prow_job_runs import build_filter, since_millis, extract_ids - - -def test_build_filter_empty(): - f = build_filter([], [], None, None, []) - assert f == {"items": [], "linkOperator": "and"} - - -def test_build_filter_job_contains(): - f = build_filter(["e2e-metal", "ipi"], [], None, None, []) - assert f["items"] == [ - {"columnField": "name", "operatorValue": "contains", "value": "e2e-metal"}, - {"columnField": "name", "operatorValue": "contains", "value": "ipi"}, - ] - assert f["linkOperator"] == "and" - - -def test_build_filter_variants(): - f = build_filter([], ["Platform:metal", "Network:ovn"], None, None, []) - assert f["items"] == [ - {"columnField": "variants", "operatorValue": "has entry", "value": "Platform:metal"}, - {"columnField": "variants", "operatorValue": "has entry", "value": "Network:ovn"}, - ] - - -def test_build_filter_result(): - f = build_filter([], [], "F", None, []) - assert f["items"] == [ - {"columnField": "overall_result", "operatorValue": "equals", "value": "F"}, - ] - - -def test_build_filter_since_ms_is_string(): - f = build_filter([], [], None, 1750000000000, []) - assert f["items"] == [ - {"columnField": "timestamp", "operatorValue": ">", "value": "1750000000000"}, - ] - - -def test_build_filter_extra_items_merged_last(): - extra = [{"columnField": "cluster", "operatorValue": "equals", "value": "build09"}] - f = build_filter(["e2e"], [], None, None, extra) - assert f["items"][-1] == extra[0] - assert len(f["items"]) == 2 - - -def test_build_filter_all_combined(): - f = build_filter(["e2e"], ["Platform:metal"], "n", 123, []) - fields = [i["columnField"] for i in f["items"]] - assert fields == ["name", "variants", "overall_result", "timestamp"] - assert f["linkOperator"] == "and" - - -def test_since_millis(): - now_ms = 1_750_000_000_000 - assert since_millis(24, now_ms) == now_ms - 24 * 3600 * 1000 - - -def test_since_millis_fractional_hours(): - assert since_millis(0.5, 7_200_000) == 7_200_000 - 1_800_000 - - -def test_extract_ids(): - rows = [{"prow_id": "111", "job": "a"}, {"prow_id": "222"}] - assert extract_ids(rows) == ["111", "222"] - - -def test_extract_ids_empty(): - assert extract_ids([]) == [] diff --git a/plugins/ci/skills/fetch-prowjob-json/SKILL.md b/plugins/ci/skills/fetch-prowjob-json/SKILL.md index ebd58e06f..9dfe9b908 100644 --- a/plugins/ci/skills/fetch-prowjob-json/SKILL.md +++ b/plugins/ci/skills/fetch-prowjob-json/SKILL.md @@ -175,4 +175,6 @@ Variant: ci-4.22 ## See Also -- Related Skill: `ci:prow-job-analysis` (comprehensive Prow CI job failure analysis — test, install, disruption, and more) +- Related Command: `/ci:analyze-prow-job-test-failure` (analyzes test failures in a Prow job) +- Related Command: `/ci:analyze-prow-job-install-failure` (analyzes install failures in a Prow job) +- Related Skill: `ci:prow-job-analyze-test-failure` (detailed test failure analysis) diff --git a/plugins/ci/skills/fetch-regression-details/SKILL.md b/plugins/ci/skills/fetch-regression-details/SKILL.md index 266595cf3..f6222cb05 100644 --- a/plugins/ci/skills/fetch-regression-details/SKILL.md +++ b/plugins/ci/skills/fetch-regression-details/SKILL.md @@ -469,5 +469,5 @@ API URL: https://sippy.dptools.openshift.org/api/... ## See Also - Component Readiness API Documentation: https://sippy.dptools.openshift.org/api/docs -- Related Skill: `ci:prow-job-analysis` (comprehensive Prow CI job failure analysis) +- Related Skill: `ci:analyze-prow-job-test-failure` (for analyzing individual test failures) - Related Command: `/ci:analyze-regression` (uses this skill) diff --git a/plugins/ci/skills/list-symptoms/SKILL.md b/plugins/ci/skills/list-symptoms/SKILL.md deleted file mode 100644 index 74f855071..000000000 --- a/plugins/ci/skills/list-symptoms/SKILL.md +++ /dev/null @@ -1,199 +0,0 @@ ---- -name: list-symptoms -description: List, search, and inspect Sippy Symptoms and Labels — known CI failure signatures — via the public Sippy API ---- - -# List Symptoms - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -This skill lists, searches, and fetches symptoms and labels using the public (read-only, no auth) Sippy API. - -## When to Use This Skill - -Use this skill when you need to: - -- Browse the catalog of known CI failure signatures (symptoms) -- Check whether a failure pattern already has a symptom before creating a new one (avoid duplicates) -- Look up what a label like `InfraFailure` means (its title and explanation) -- Find which symptoms apply a given label -- Inspect a single symptom or label by its ID - -## Prerequisites - -1. **Network Access**: The Sippy API must be accessible at `https://sippy.dptools.openshift.org` - - No authentication required - - Check: `curl -s https://sippy.dptools.openshift.org/api/jobs/labels | head -c 200` - -2. **Python 3**: Python 3.6 or later - - Check: `python3 --version` - - Uses only standard library (no external dependencies) - -## Implementation Steps - -Invoke the script with the flags that match the user's question: - -```bash -script_path="plugins/ci/skills/list-symptoms/list_symptoms.py" - -# List all symptoms (JSON by default) -python3 "$script_path" - -# List all symptoms as a human-readable summary -python3 "$script_path" --format summary - -# Search symptoms by text (matches id, summary, and match_string, case-insensitive) -python3 "$script_path" --search "credentials" --format summary - -# Find all symptoms that apply a given label -python3 "$script_path" --label InfraFailure --format summary - -# Fetch a single symptom by ID -python3 "$script_path" --id AWSCouldNotValidateAccessCredentials - -# List all labels instead of symptoms -python3 "$script_path" --labels --format summary - -# Fetch a single label by ID -python3 "$script_path" --labels --id InfraFailure -``` - -Flags: -- `--id `: fetch a single symptom (or label with `--labels`) by ID -- `--search `: case-insensitive text search over id/summary/match_string (for labels: id/label_title/explanation) -- `--label `: only symptoms that apply this label ID -- `--matcher-type {string,regex,none,cel}`: only symptoms of this matcher type -- `--labels`: operate on labels instead of symptoms -- `--format {json,summary}`: output format (default `json`) - -## API Details - -### Endpoints - -```text -GET https://sippy.dptools.openshift.org/api/jobs/symptoms -GET https://sippy.dptools.openshift.org/api/jobs/symptoms/{id} -GET https://sippy.dptools.openshift.org/api/jobs/labels -GET https://sippy.dptools.openshift.org/api/jobs/labels/{id} -``` - -### Example Symptom JSON - -```json -{ - "id": "AWSCouldNotValidateAccessCredentials", - "summary": "AWS could not validate access credentials", - "matcher_type": "string", - "file_pattern": "build-log.txt", - "match_string": "api error AuthFailure: AWS was not able to validate the provided access credentials", - "label_ids": ["InfraFailure"], - "created_by": "kenzhang", - "updated_by": "kenzhang", - "updated_at": "2026-04-27T16:09:12.660547Z" -} -``` - -### Symptom Fields - -| Field | Description | -|---|---| -| `id` | Immutable identifier (generated from the summary on create) | -| `summary` | Short, unique human-readable description (≤200 chars) | -| `matcher_type` | One of `string`, `regex`, `none` (file exists), `cel` (expression over label names) | -| `file_pattern` | Glob over the job run's artifact files, e.g. `**/build-log.txt` (not used for `cel`) | -| `match_string` | Substring, regular expression, or CEL expression depending on `matcher_type` | -| `label_ids` | Label IDs applied to a run when the symptom matches | -| `created_by`, `updated_by`, `updated_at` | Metadata about who created/last modified the symptom and when | - -### Label Fields - -| Field | Description | -|---|---| -| `id` | Immutable identifier (≤80 chars) | -| `label_title` | Unique human-readable title | -| `explanation` | Markdown explanation of what the label means | -| `hide_display_contexts` | UI contexts where the label is hidden (subset of `spyglass`, `metrics`, `jaq-options`) | - -## Error Handling - -### Case 1: Not Found (404) - -```bash -python3 list_symptoms.py --id NoSuchSymptom -# Error: not found: https://sippy.dptools.openshift.org/api/jobs/symptoms/NoSuchSymptom (use list mode to see valid IDs) -``` - -Exits 1 and suggests listing symptoms/labels to find valid IDs. - -### Case 2: Network Error - -```bash -# Error: failed to connect to Sippy API: [Errno -2] Name or service not known -``` - -Exits 1. Check network connectivity to `sippy.dptools.openshift.org`. - -### Case 3: Empty Results - -Filters that match nothing print `[]` (JSON) or `No results.` (summary) and exit 0 — this is not an error. - -**Exit Codes:** -- `0`: Success (including empty results) -- `1`: Error (404, network error, etc.) - -## Examples - -### Example 1: List All Symptoms - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --format summary -``` - -**Expected Output (excerpt):** -```text -Symptom: AWSCouldNotValidateAccessCredentials - Summary: AWS could not validate access credentials - Matcher: string - File pattern: build-log.txt - Match string: api error AuthFailure: AWS was not able to validate the provided access credentials - Labels: InfraFailure - Updated by: kenzhang at 2026-04-27T16:09:12.660547Z - -Total: 42 -``` - -### Example 2: Search for an Existing Symptom Before Creating One - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --search "credentials" --format summary -``` - -### Example 3: Symptoms That Apply a Label - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --label InfraFailure --format summary -``` - -### Example 4: Fetch One Symptom by ID - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --id AWSCouldNotValidateAccessCredentials -``` - -### Example 5: List All Labels - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --labels --format summary -``` - -## Notes - -- `--search`, `--label`, and `--matcher-type` filtering is performed **client-side**: the script fetches the full list and filters locally -- This skill is read-only; to create, update, or delete symptoms or labels use the `manage-symptoms` and `manage-labels` skills -- No authentication is required — the public Sippy instance serves all read endpoints - -## See Also - -- Related Skill: `manage-symptoms` (create/update/delete symptoms) -- Related Skill: `manage-labels` (create/update/delete labels) -- Related Skill: `diagnose-job-run-symptoms` (explain which symptoms/labels apply to a job run) diff --git a/plugins/ci/skills/list-symptoms/list_symptoms.py b/plugins/ci/skills/list-symptoms/list_symptoms.py deleted file mode 100644 index 05138903f..000000000 --- a/plugins/ci/skills/list-symptoms/list_symptoms.py +++ /dev/null @@ -1,118 +0,0 @@ -"""List, search, and fetch Sippy Symptoms and Labels (read-only, no auth).""" -import argparse -import json -import sys -import urllib.error -import urllib.parse -import urllib.request - -BASE_URL = "https://sippy.dptools.openshift.org/api/jobs" - - -def item_url(resource, item_id): - """Build the URL for a single symptom/label, URL-encoding the ID.""" - return "%s/%s/%s" % (BASE_URL, resource, urllib.parse.quote(item_id, safe="")) - - -def fetch(url): - req = urllib.request.Request(url, headers={"Accept": "application/json"}) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) - except urllib.error.HTTPError as e: - if e.code == 404: - print("Error: not found: %s (use list mode to see valid IDs)" % url, file=sys.stderr) - else: - print("Error: HTTP %d from %s: %s" % (e.code, url, e.reason), file=sys.stderr) - sys.exit(1) - except urllib.error.URLError as e: - print("Error: failed to connect to Sippy API: %s" % e.reason, file=sys.stderr) - sys.exit(1) - - -def filter_symptoms(symptoms, search=None, label=None, matcher_type=None): - out = [] - for s in symptoms: - if search: - hay = " ".join([s.get("id", ""), s.get("summary", ""), s.get("match_string", "") or ""]).lower() - if search.lower() not in hay: - continue - if label and label not in (s.get("label_ids") or []): - continue - if matcher_type and s.get("matcher_type") != matcher_type: - continue - out.append(s) - return out - - -def summarize_symptom(s): - lines = ["Symptom: %s" % s.get("id")] - lines.append(" Summary: %s" % s.get("summary")) - lines.append(" Matcher: %s" % s.get("matcher_type")) - if s.get("file_pattern"): - lines.append(" File pattern: %s" % s.get("file_pattern")) - if s.get("match_string"): - lines.append(" Match string: %s" % s.get("match_string")) - lines.append(" Labels: %s" % ", ".join(s.get("label_ids") or [])) - lines.append(" Updated by: %s at %s" % (s.get("updated_by"), s.get("updated_at"))) - return "\n".join(lines) - - -def filter_labels(labels, search=None): - if not search: - return labels - out = [] - for label in labels: - hay = " ".join([label.get("id", ""), label.get("label_title", ""), - label.get("explanation", "") or ""]).lower() - if search.lower() in hay: - out.append(label) - return out - - -def summarize_label(label): - lines = ["Label: %s" % label.get("id")] - lines.append(" Title: %s" % label.get("label_title")) - if label.get("explanation"): - lines.append(" Explanation: %s" % label.get("explanation")) - if label.get("hide_display_contexts"): - lines.append(" Hidden in: %s" % ", ".join(label["hide_display_contexts"])) - return "\n".join(lines) - - -def main(): - p = argparse.ArgumentParser(description="List/search Sippy symptoms and labels") - p.add_argument("--id", help="Fetch a single symptom (or label with --labels) by ID") - p.add_argument("--search", help="Case-insensitive text search over id/summary/match_string") - p.add_argument("--label", help="Only symptoms that apply this label ID") - p.add_argument("--matcher-type", choices=["string", "regex", "none", "cel"]) - p.add_argument("--labels", action="store_true", help="List labels instead of symptoms") - p.add_argument("--format", choices=["json", "summary"], default="json") - args = p.parse_args() - - if args.labels and (args.label or args.matcher_type): - print("Error: --label and --matcher-type only apply to symptoms and cannot " - "be combined with --labels", file=sys.stderr) - return 1 - - resource = "labels" if args.labels else "symptoms" - if args.id: - items = [fetch(item_url(resource, args.id))] - else: - items = fetch("%s/%s" % (BASE_URL, resource)) - if args.labels: - items = filter_labels(items, args.search) - else: - items = filter_symptoms(items, args.search, args.label, args.matcher_type) - - if args.format == "json": - print(json.dumps(items, indent=2)) - else: - fmt = summarize_label if args.labels else summarize_symptom - print(("\n\n".join(fmt(i) for i in items)) if items else "No results.") - print("\nTotal: %d" % len(items)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/list-symptoms/test_list_symptoms.py b/plugins/ci/skills/list-symptoms/test_list_symptoms.py deleted file mode 100644 index add89e1c7..000000000 --- a/plugins/ci/skills/list-symptoms/test_list_symptoms.py +++ /dev/null @@ -1,53 +0,0 @@ -from list_symptoms import filter_labels, filter_symptoms, item_url - -SYMPTOMS = [ - {"id": "AWSAuthFailure", "summary": "AWS could not validate credentials", - "matcher_type": "string", "file_pattern": "build-log.txt", - "match_string": "api error AuthFailure", "label_ids": ["InfraFailure"]}, - {"id": "DNSFlake", "summary": "Cluster DNS lookup flake", - "matcher_type": "regex", "file_pattern": "**/pods/*.log", - "match_string": "dns lookup .* timed out", "label_ids": ["ClusterDNSFlake"]}, -] - -def test_filter_by_search_matches_summary_case_insensitive(): - assert [s["id"] for s in filter_symptoms(SYMPTOMS, search="aws could")] == ["AWSAuthFailure"] - -def test_filter_by_search_matches_match_string_and_id(): - assert [s["id"] for s in filter_symptoms(SYMPTOMS, search="dns lookup")] == ["DNSFlake"] - assert [s["id"] for s in filter_symptoms(SYMPTOMS, search="dnsflake")] == ["DNSFlake"] - -def test_filter_by_label(): - assert [s["id"] for s in filter_symptoms(SYMPTOMS, label="InfraFailure")] == ["AWSAuthFailure"] - -def test_filter_by_matcher_type(): - assert [s["id"] for s in filter_symptoms(SYMPTOMS, matcher_type="regex")] == ["DNSFlake"] - -def test_no_filters_returns_all(): - assert filter_symptoms(SYMPTOMS) == SYMPTOMS - -LABELS = [ - {"id": "InfraFailure", "label_title": "Infrastructure Failure", - "explanation": "Cloud provider or infra problem"}, - {"id": "ClusterDNSFlake", "label_title": "Cluster DNS Flake", - "explanation": "DNS lookups intermittently time out"}, -] - -def test_filter_labels_matches_title(): - assert [label["id"] for label in filter_labels(LABELS, search="dns flake")] == ["ClusterDNSFlake"] - -def test_filter_labels_matches_id(): - assert [label["id"] for label in filter_labels(LABELS, search="infrafailure")] == ["InfraFailure"] - -def test_filter_labels_case_insensitive_explanation(): - assert [label["id"] for label in filter_labels(LABELS, search="CLOUD PROVIDER")] == ["InfraFailure"] - -def test_filter_labels_no_search_returns_all(): - assert filter_labels(LABELS) == LABELS - -def test_item_url_encodes_reserved_characters(): - assert item_url("symptoms", "a/b c") == ( - "https://sippy.dptools.openshift.org/api/jobs/symptoms/a%2Fb%20c") - -def test_item_url_plain_id_unchanged(): - assert item_url("labels", "InfraFailure") == ( - "https://sippy.dptools.openshift.org/api/jobs/labels/InfraFailure") diff --git a/plugins/ci/skills/manage-labels/SKILL.md b/plugins/ci/skills/manage-labels/SKILL.md deleted file mode 100644 index 0aaa7036a..000000000 --- a/plugins/ci/skills/manage-labels/SKILL.md +++ /dev/null @@ -1,150 +0,0 @@ ---- -name: manage-labels -description: Create, update, or delete Sippy job run Labels (the human-readable tags applied by Symptoms) via the authenticated Sippy API ---- - -# Manage Labels - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -Labels must exist before a symptom can reference them, so create labels first when building a new symptom. - -## When to Use This Skill - -Use this skill when you need to: - -- Create a new label before creating a symptom that applies it (see `manage-symptoms`) -- Fix a label's title or explanation -- Hide a label from certain UI contexts (Spyglass, metrics, jaq-options) -- Remove an obsolete label - -## Prerequisites - -1. **OpenShift CLI Authentication**: Required for authenticating to the sippy-auth API - - Must be logged into the DPCR cluster via `oc login` - - Cluster API: `https://api.cr.j7t7.p1.openshiftapps.com:6443` - - Use the `oc-auth` skill to obtain the Bearer token - -2. **Python 3**: Python 3.6 or later - - Check: `python3 --version` - - Uses only standard library (no external dependencies) - -## Implementation Steps - -### Step 1: Obtain Authentication Token - -Use the `oc-auth` skill to obtain a Bearer token from the DPCR cluster: - -```bash -# Get token from the DPCR cluster context -# The oc-auth skill's curl_with_token.sh uses this cluster for sippy-auth -DPCR_CLUSTER="https://api.cr.j7t7.p1.openshiftapps.com:6443" - -# Find the oc context for the DPCR cluster and get the token -CONTEXT=$(oc config get-contexts -o name 2>/dev/null | while read -r ctx; do - server=$(oc config view -o jsonpath="{.clusters[?(@.name=='$(oc config view -o jsonpath="{.contexts[?(@.name=='$ctx')].context.cluster}" 2>/dev/null)')].cluster.server}" 2>/dev/null || echo "") - server_clean=$(echo "$server" | sed -E 's|^https?://||') - if [ "$server_clean" = "api.cr.j7t7.p1.openshiftapps.com:6443" ]; then - echo "$ctx" - break - fi -done) - -if [ -z "$CONTEXT" ]; then - echo "Error: Not logged into DPCR cluster. Please run: oc login $DPCR_CLUSTER" - exit 1 -fi - -export SIPPY_TOKEN=$(oc whoami -t --context="$CONTEXT" 2>/dev/null) -if [ -z "$SIPPY_TOKEN" ]; then - echo "Error: Failed to get token. Please re-authenticate to DPCR cluster." - exit 1 -fi -``` - -Prefer exporting `SIPPY_TOKEN` as above rather than passing `--token` on the command line — command-line arguments are visible in process listings. `--token` still works and takes precedence over the environment variable. - -### Step 2: Create a Label - -```bash -python3 plugins/ci/skills/manage-labels/manage_labels.py create \ - --title "Cluster DNS Flake" \ - --explanation "DNS lookups inside the cluster intermittently time out." -``` - -The label `id` is generated from the title by the server if you omit `--id`. Pass `--id` on create only if you need a specific identifier (max 80 characters). - -### Step 3: Update a Label - -Only pass the flags you want to change — the script fetches the existing label and merges, because the API's PUT is a full replacement: - -```bash -python3 plugins/ci/skills/manage-labels/manage_labels.py update \ - --id ClusterDNSFlake \ - --explanation "DNS lookups inside the cluster intermittently time out. Usually caused by node-local DNS cache restarts." -``` - -To hide a label from certain UI contexts: - -```bash -python3 plugins/ci/skills/manage-labels/manage_labels.py update \ - --id ClusterDNSFlake \ - --hide-display-contexts "spyglass,metrics" -``` - -### Step 4: Delete a Label - -**Before deleting, you MUST show the label to the user (fetch it with the `list-symptoms` skill: `python3 plugins/ci/skills/list-symptoms/list_symptoms.py --labels --id --format summary`) and get their explicit confirmation. Never run delete without the user confirming the specific label.** - -```bash -python3 plugins/ci/skills/manage-labels/manage_labels.py delete \ - --id ClusterDNSFlake -``` - -**Arguments**: -- `action`: `create`, `update`, or `delete` (positional, required) - -**Options**: -- `--token `: Bearer token from the oc-auth skill (optional if the `SIPPY_TOKEN` environment variable is set, which is preferred — argv is visible in process listings; `--token` takes precedence) -- `--id `: Label ID (required for update/delete; optional for create) -- `--title `: Human-readable label title (required for create) -- `--explanation `: Markdown explanation of the label -- `--hide-display-contexts `: Comma-separated subset of `spyglass,metrics,jaq-options` -- `--format json|summary`: Output format (default: json) - -## API Details - -**Base URL (writes)**: `https://sippy-auth.dptools.openshift.org/api/jobs/labels` - -- Create: `POST /api/jobs/labels` -- Update: `PUT /api/jobs/labels/{id}` (full replacement — the script fetches the existing label and merges your changes, so only pass flags you want to change) -- Delete: `DELETE /api/jobs/labels/{id}` - -**Authentication**: `Authorization: Bearer ` from the DPCR cluster. - -**Label fields**: - -| Field | Description | -|-------|-------------| -| `id` | Immutable identifier, max 80 characters; generated from the title if omitted on create | -| `label_title` | Human-readable title; must be unique | -| `explanation` | Markdown explanation of what the label means | -| `hide_display_contexts` | Optional subset of `spyglass`, `metrics`, `jaq-options` — UI contexts where the label is hidden | - -## Error Handling - -- **401/403**: Token missing or expired — refresh it via the `oc-auth` skill. -- **501**: You hit the read-only Sippy instance with a write; make sure the sippy-auth base URL is used (the script already does). -- **400**: Server-side validation failure — the server's message is shown in the `detail` field of the output. -- **Client-side validation**: Missing title, over-long ID, or invalid `hide_display_contexts` values are caught locally and reported before any request is sent (exit 1). -- **Concurrent edits**: The update flow is read-merge-replace with no server-side concurrency control, so near-simultaneous edits can overwrite each other — re-check the label after updating if others may be editing. - -**Exit Codes**: -- `0`: Success -- `1`: Validation error, API error, or network error - -## See Also - -- Related Skill: `oc-auth` (provides authentication tokens for sippy-auth) -- Related Skill: `list-symptoms` (list/inspect labels and symptoms, no auth needed) -- Related Skill: `manage-symptoms` (create/update/delete symptoms that apply labels) diff --git a/plugins/ci/skills/manage-labels/manage_labels.py b/plugins/ci/skills/manage-labels/manage_labels.py deleted file mode 100644 index 4ff57a664..000000000 --- a/plugins/ci/skills/manage-labels/manage_labels.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Create, update, or delete Sippy job run Labels (requires auth token).""" -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request - -WRITE_URL = "https://sippy-auth.dptools.openshift.org/api/jobs/labels" -READ_URL = "https://sippy.dptools.openshift.org/api/jobs/labels" -VALID_HIDE_CONTEXTS = ("spyglass", "metrics", "jaq-options") - - -def resolve_token(arg_token, env=None): - """Return the Bearer token from --token or the SIPPY_TOKEN env var. - - --token takes precedence over the environment variable. Prefer the env - var: command-line arguments are visible in process listings. - """ - env = os.environ if env is None else env - return arg_token or env.get("SIPPY_TOKEN") or None - - -def validate_label(payload): - errs = [] - if not payload.get("label_title"): - errs.append("label_title is required") - if payload.get("id") and len(payload["id"]) > 80: - errs.append("id must be at most 80 characters") - for ctx in payload.get("hide_display_contexts") or []: - if ctx not in VALID_HIDE_CONTEXTS: - errs.append("invalid hide_display_contexts value %r (valid: %s)" % (ctx, ", ".join(VALID_HIDE_CONTEXTS))) - return errs - - -def request(method, url, token, payload=None): - headers = {"Content-Type": "application/json", "Authorization": "Bearer %s" % token} - data = json.dumps(payload).encode("utf-8") if payload is not None else None - req = urllib.request.Request(url, data=data, headers=headers, method=method) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - body = resp.read().decode("utf-8") - return {"success": True, "result": json.loads(body) if body.strip() else {}} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode("utf-8") - except Exception: - pass - hint = "" - if e.code == 501: - hint = " (write endpoints are disabled on this instance; make sure you are using sippy-auth)" - elif e.code in (401, 403): - hint = " (token missing/expired; use the oc-auth skill to obtain a fresh token)" - return {"success": False, "error": "HTTP %d: %s%s" % (e.code, e.reason, hint), "detail": detail} - except urllib.error.URLError as e: - return {"success": False, "error": "Failed to connect: %s" % e.reason} - - -def build_update_payload(existing, title=None, explanation=None, hide_display_contexts=None): - """Merge changed fields into the existing label for a full-replacement PUT. - - Pass None to preserve the existing value; an explicit empty string for - explanation clears it. hide_display_contexts is a list when overriding. - """ - return {"id": existing.get("id"), - "label_title": title or existing.get("label_title"), - "explanation": explanation if explanation is not None else existing.get("explanation", ""), - "hide_display_contexts": hide_display_contexts if hide_display_contexts is not None - else existing.get("hide_display_contexts")} - - -def label_url(label_id): - return "%s/%s" % (WRITE_URL, urllib.parse.quote(label_id, safe="")) - - -def fetch_existing(label_id): - try: - with urllib.request.urlopen("%s/%s" % (READ_URL, urllib.parse.quote(label_id, safe="")), timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) - except (urllib.error.HTTPError, urllib.error.URLError) as e: - print("Error: cannot fetch existing label %s: %s" % (label_id, e), file=sys.stderr) - return None - - -def main(): - p = argparse.ArgumentParser(description="Create/update/delete Sippy labels") - p.add_argument("action", choices=["create", "update", "delete"]) - p.add_argument("--token", help="Bearer token (or set SIPPY_TOKEN env var, preferred; use oc-auth skill)") - p.add_argument("--id", help="Label ID (required for update/delete; optional for create)") - p.add_argument("--title", help="Human-readable label title (required for create)") - p.add_argument("--explanation", help="Markdown explanation of the label") - p.add_argument("--hide-display-contexts", help="Comma-separated: spyglass,metrics,jaq-options") - p.add_argument("--format", choices=["json", "summary"], default="json") - args = p.parse_args() - - token = resolve_token(args.token) - if not token: - print("Error: no token provided — pass --token or set the SIPPY_TOKEN " - "environment variable (preferred; use the oc-auth skill to obtain " - "one)", file=sys.stderr) - return 1 - - if args.action in ("update", "delete") and not args.id: - print("Error: --id is required for %s" % args.action, file=sys.stderr) - return 1 - - hide_contexts = None - if args.hide_display_contexts is not None: - hide_contexts = [c.strip() for c in args.hide_display_contexts.split(",") if c.strip()] - - if args.action == "delete": - out = request("DELETE", label_url(args.id), token) - else: - if args.action == "create": - payload = {"label_title": args.title, "explanation": args.explanation or ""} - if args.id: - payload["id"] = args.id - if hide_contexts is not None: - payload["hide_display_contexts"] = hide_contexts - else: - existing = fetch_existing(args.id) - if existing is None: - return 1 - existing["id"] = args.id - payload = build_update_payload(existing, title=args.title, - explanation=args.explanation, - hide_display_contexts=hide_contexts) - payload = {k: v for k, v in payload.items() if v is not None} - errs = validate_label(payload) - if errs: - for e in errs: - print("Validation error: %s" % e, file=sys.stderr) - return 1 - if args.action == "create": - out = request("POST", WRITE_URL, token, payload) - else: - out = request("PUT", label_url(args.id), token, payload) - - if args.format == "json": - print(json.dumps(out, indent=2)) - else: - if out["success"]: - print("Label %s - SUCCESS" % args.action) - print(json.dumps(out.get("result", {}), indent=2)) - else: - print("Label %s - FAILED: %s" % (args.action, out.get("error"))) - if out.get("detail"): - print("Detail: %s" % out["detail"]) - return 0 if out["success"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/manage-labels/test_manage_labels.py b/plugins/ci/skills/manage-labels/test_manage_labels.py deleted file mode 100644 index f173bc918..000000000 --- a/plugins/ci/skills/manage-labels/test_manage_labels.py +++ /dev/null @@ -1,57 +0,0 @@ -import manage_labels -from manage_labels import validate_label, build_update_payload - -def test_valid_label_no_errors(): - assert validate_label({"label_title": "Infra Failure", "explanation": "x"}) == [] - -def test_missing_title(): - errs = validate_label({"explanation": "x"}) - assert any("label_title" in e for e in errs) - -def test_id_too_long(): - errs = validate_label({"id": "a" * 81, "label_title": "t"}) - assert any("80" in e for e in errs) - -def test_bad_hide_context(): - errs = validate_label({"label_title": "t", "hide_display_contexts": ["bogus"]}) - assert any("hide_display_contexts" in e for e in errs) - -def test_valid_hide_contexts(): - assert validate_label({"label_title": "t", - "hide_display_contexts": ["spyglass", "metrics", "jaq-options"]}) == [] - - -EXISTING = {"id": "ClusterDNSFlake", "label_title": "Cluster DNS Flake", - "explanation": "old text", "hide_display_contexts": ["spyglass"]} - - -def test_update_overrides_title(): - out = build_update_payload(EXISTING, title="New Title") - assert out["label_title"] == "New Title" - assert out["id"] == "ClusterDNSFlake" - - -def test_update_preserves_explanation_when_not_passed(): - out = build_update_payload(EXISTING) - assert out["explanation"] == "old text" - - -def test_update_empty_string_clears_explanation(): - out = build_update_payload(EXISTING, explanation="") - assert out["explanation"] == "" - - -def test_update_hide_contexts_preserved_and_overridden(): - assert build_update_payload(EXISTING)["hide_display_contexts"] == ["spyglass"] - out = build_update_payload(EXISTING, hide_display_contexts=["metrics"]) - assert out["hide_display_contexts"] == ["metrics"] - - -def test_resolve_token_arg_wins_over_env(): - assert manage_labels.resolve_token("argtok", {"SIPPY_TOKEN": "envtok"}) == "argtok" - -def test_resolve_token_falls_back_to_env(): - assert manage_labels.resolve_token(None, {"SIPPY_TOKEN": "envtok"}) == "envtok" - -def test_resolve_token_none_when_unset(): - assert manage_labels.resolve_token(None, {}) is None diff --git a/plugins/ci/skills/manage-symptoms/SKILL.md b/plugins/ci/skills/manage-symptoms/SKILL.md deleted file mode 100644 index 3d29c7ee8..000000000 --- a/plugins/ci/skills/manage-symptoms/SKILL.md +++ /dev/null @@ -1,192 +0,0 @@ ---- -name: manage-symptoms -description: Create, update, or delete Sippy Symptoms — known CI failure signatures that automatically label job runs — via the authenticated Sippy API ---- - -# Manage Symptoms - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -## When to Use This Skill - -Use this skill when you need to: - -- Auto-label a recurring failure pattern you found in CI logs, so future runs are recognized without re-debugging -- Correct an existing symptom's matcher, file pattern, or match string -- Retire an obsolete symptom that no longer applies - -## Prerequisites - -1. **OpenShift CLI Authentication**: Required for authenticating to the sippy-auth API - - Must be logged into the DPCR cluster via `oc login` - - Cluster API: `https://api.cr.j7t7.p1.openshiftapps.com:6443` - - Use the `oc-auth` skill to obtain the Bearer token - -2. **Python 3**: Python 3.6 or later - - Check: `python3 --version` - - Uses only standard library (no external dependencies) - -## Implementation Steps - -### Step 1: Check for Duplicates - -Before creating a symptom, search the existing catalog to avoid duplicates: - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --search "AuthFailure" --format summary -``` - -If an equivalent symptom already exists, prefer updating it instead of creating a new one. - -### Step 2: Verify the Target Labels Exist - -Symptoms can only reference labels that already exist: - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --labels --format summary -``` - -If a label is missing, create it first with the `manage-labels` skill. (The script also verifies label IDs against the labels API before submitting; use `--skip-label-check` only if the labels API is unreachable.) - -### Step 3: Obtain Authentication Token - -Use the `oc-auth` skill to obtain a Bearer token from the DPCR cluster: - -```bash -# Get token from the DPCR cluster context -# The oc-auth skill's curl_with_token.sh uses this cluster for sippy-auth -DPCR_CLUSTER="https://api.cr.j7t7.p1.openshiftapps.com:6443" - -# Find the oc context for the DPCR cluster and get the token -CONTEXT=$(oc config get-contexts -o name 2>/dev/null | while read -r ctx; do - server=$(oc config view -o jsonpath="{.clusters[?(@.name=='$(oc config view -o jsonpath="{.contexts[?(@.name=='$ctx')].context.cluster}" 2>/dev/null)')].cluster.server}" 2>/dev/null || echo "") - server_clean=$(echo "$server" | sed -E 's|^https?://||') - if [ "$server_clean" = "api.cr.j7t7.p1.openshiftapps.com:6443" ]; then - echo "$ctx" - break - fi -done) - -if [ -z "$CONTEXT" ]; then - echo "Error: Not logged into DPCR cluster. Please run: oc login $DPCR_CLUSTER" - exit 1 -fi - -export SIPPY_TOKEN=$(oc whoami -t --context="$CONTEXT" 2>/dev/null) -if [ -z "$SIPPY_TOKEN" ]; then - echo "Error: Failed to get token. Please re-authenticate to DPCR cluster." - exit 1 -fi -``` - -Prefer exporting `SIPPY_TOKEN` as above rather than passing `--token` on the command line — command-line arguments are visible in process listings. `--token` still works and takes precedence over the environment variable. - -### Step 4: Confirm the Payload with the User - -**Before any create or update, show the user the full payload that will be sent (summary, matcher type, file pattern, match string, label IDs) and get their confirmation. Before delete, you MUST show the symptom (`list-symptoms --id --format summary`) and get explicit confirmation — never run delete without the user confirming the specific symptom.** - -### Step 5: Create a Symptom - -```bash -python3 plugins/ci/skills/manage-symptoms/manage_symptoms.py create \ - --summary "AWS could not validate access credentials" \ - --matcher-type string \ - --file-pattern "build-log.txt" \ - --match-string "api error AuthFailure: AWS was not able to validate the provided access credentials" \ - --label-ids InfraFailure -``` - -The symptom `id` is generated by the server from the summary — do not pass `--id` on create. - -### Step 6: Update a Symptom - -Only pass the flags you want to change — the script fetches the existing symptom and merges, because the API's PUT is a full replacement: - -```bash -python3 plugins/ci/skills/manage-symptoms/manage_symptoms.py update \ - --id AWSCouldNotValidateAccessCredentials \ - --match-string "api error AuthFailure" -``` - -To remove all labels from a symptom, pass an explicit empty value: `--label-ids ""` clears the label list on update (omitting the flag preserves the existing labels). - -### Step 7: Delete a Symptom - -Delete is a soft delete on the server side. Requires explicit user confirmation first (see Step 4): - -```bash -python3 plugins/ci/skills/manage-symptoms/manage_symptoms.py delete \ - --id AWSCouldNotValidateAccessCredentials -``` - -### Step 8: Verify - -After create/update, verify the result: - -```bash -python3 plugins/ci/skills/list-symptoms/list_symptoms.py --id --format summary -``` - -Optionally test the symptom against a known-affected run with the `reevaluate-job-runs` skill using `--dry-run` to preview matches without writing anything. - -**Arguments**: -- `action`: `create`, `update`, or `delete` (positional, required) - -**Options**: -- `--token `: Bearer token from the oc-auth skill (optional if the `SIPPY_TOKEN` environment variable is set, which is preferred — argv is visible in process listings; `--token` takes precedence) -- `--id `: Symptom ID (required for update/delete; server-generated on create) -- `--summary `: Short unique description (required for create, max 200 characters) -- `--matcher-type string|regex|none|cel`: How the match string is interpreted -- `--file-pattern `: Artifact glob, e.g. `**/build-log.txt` (required for non-CEL matchers) -- `--match-string `: Substring, regex, or CEL expression -- `--label-ids `: Comma-separated label IDs to apply on match; on update, `--label-ids ""` clears all labels -- `--skip-label-check`: Skip verifying label IDs against the labels API -- `--format json|summary`: Output format (default: json) - -## API Details - -**Base URL (writes)**: `https://sippy-auth.dptools.openshift.org/api/jobs/symptoms` - -- Create: `POST /api/jobs/symptoms` -- Update: `PUT /api/jobs/symptoms/{id}` (full replacement — the script fetches the existing symptom and merges your changes, so only pass flags you want to change) -- Delete: `DELETE /api/jobs/symptoms/{id}` (soft delete) - -**Authentication**: `Authorization: Bearer ` from the DPCR cluster. - -**Symptom fields**: - -| Field | Description | -|-------|-------------| -| `id` | Immutable identifier, generated from the summary on create | -| `summary` | Required, unique, max 200 characters | -| `matcher_type` | One of `string`, `regex`, `none`, `cel` | -| `file_pattern` | Artifact glob; required for all matcher types except `cel` | -| `match_string` | Required for `string`/`regex`/`cel`; not used by `none` (file merely exists) | -| `label_ids` | Label IDs applied on match; must reference existing labels | -| `created_by`, `updated_by`, timestamps | Metadata set by the server | - -**Matcher-type rules**: -- `string` / `regex`: require both `file_pattern` and `match_string` -- `none`: requires only `file_pattern` (matches when the file exists) -- `cel`: requires only `match_string` (a CEL expression over other label names) - -## Error Handling - -- **Client-side validation**: Missing summary, over-long summary, invalid matcher type, or missing file_pattern/match_string for the chosen matcher are caught locally before any request (exit 1). -- **Label not found**: If a `--label-ids` value does not exist, validation fails and points you to the `manage-labels` skill to create it first. -- **401/403**: Token missing or expired — refresh it via the `oc-auth` skill. -- **501**: You hit the read-only Sippy instance with a write; make sure the sippy-auth base URL is used (the script already does). -- **400**: Server-side validation failure — the server's message is shown in the `detail` field of the output. -- **Concurrent edits**: The update flow is read-merge-replace with no server-side concurrency control, so near-simultaneous edits can overwrite each other — re-check the symptom after updating if others may be editing. - -**Exit Codes**: -- `0`: Success -- `1`: Validation error, API error, or network error - -## See Also - -- Related Skill: `oc-auth` (provides authentication tokens for sippy-auth) -- Related Skill: `list-symptoms` (search/inspect symptoms and labels, no auth needed) -- Related Skill: `manage-labels` (create labels before symptoms reference them) -- Related Skill: `reevaluate-job-runs` (apply or preview symptoms on past runs) -- Related Skill: `diagnose-job-run-symptoms` (explain which symptoms matched a run) diff --git a/plugins/ci/skills/manage-symptoms/manage_symptoms.py b/plugins/ci/skills/manage-symptoms/manage_symptoms.py deleted file mode 100644 index 189f162b0..000000000 --- a/plugins/ci/skills/manage-symptoms/manage_symptoms.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Create, update, or delete Sippy Symptoms (requires auth token).""" -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request - -WRITE_URL = "https://sippy-auth.dptools.openshift.org/api/jobs/symptoms" -READ_SYMPTOMS_URL = "https://sippy.dptools.openshift.org/api/jobs/symptoms" -READ_LABELS_URL = "https://sippy.dptools.openshift.org/api/jobs/labels" -VALID_MATCHERS = ("string", "regex", "none", "cel") - - -def resolve_token(arg_token, env=None): - """Return the Bearer token from --token or the SIPPY_TOKEN env var. - - --token takes precedence over the environment variable. Prefer the env - var: command-line arguments are visible in process listings. - """ - env = os.environ if env is None else env - return arg_token or env.get("SIPPY_TOKEN") or None - - -def validate_symptom(payload): - errs = [] - summary = payload.get("summary") - if not summary: - errs.append("summary is required") - elif len(summary) > 200: - errs.append("summary must be at most 200 characters") - mt = payload.get("matcher_type") - if mt not in VALID_MATCHERS: - errs.append("matcher_type must be one of: %s" % ", ".join(VALID_MATCHERS)) - return errs - if mt != "cel" and not payload.get("file_pattern"): - errs.append("file_pattern is required for matcher_type %r" % mt) - if mt in ("string", "regex", "cel") and not payload.get("match_string"): - errs.append("match_string is required for matcher_type %r" % mt) - return errs - - -def build_update_payload(existing, summary=None, matcher_type=None, - file_pattern=None, match_string=None, label_ids=None): - """Merge changed fields into the existing symptom for a full-replacement PUT. - - Pass None to preserve the existing value; an explicit empty string for - file_pattern/match_string clears it. label_ids is a list when overriding. - """ - return {"id": existing.get("id"), - "summary": summary or existing.get("summary"), - "matcher_type": matcher_type or existing.get("matcher_type"), - "file_pattern": file_pattern if file_pattern is not None else existing.get("file_pattern"), - "match_string": match_string if match_string is not None else existing.get("match_string"), - "label_ids": label_ids if label_ids is not None else existing.get("label_ids", [])} - - -def parse_label_ids(value): - """Parse the --label-ids flag. None = flag not passed (preserve on update); - empty string = explicit clear (returns []); otherwise comma-separated list.""" - if value is None: - return None - return [x.strip() for x in value.split(",") if x.strip()] - - -def symptom_url(base, symptom_id): - return "%s/%s" % (base, urllib.parse.quote(symptom_id, safe="")) - - -def get_json(url): - try: - with urllib.request.urlopen(url, timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) - except (urllib.error.HTTPError, urllib.error.URLError) as e: - print("Error fetching %s: %s" % (url, e), file=sys.stderr) - return None - - -def check_labels_exist(label_ids): - labels = get_json(READ_LABELS_URL) - if not isinstance(labels, list) or not all(isinstance(label, dict) for label in labels): - return ["could not verify label IDs (labels API unreachable or returned unexpected data)"] - known = {label.get("id") for label in labels} - return ["label ID %r does not exist (create it first with the manage-labels skill)" % lid - for lid in label_ids if lid not in known] - - -def request(method, url, token, payload=None): - headers = {"Content-Type": "application/json", "Authorization": "Bearer %s" % token} - data = json.dumps(payload).encode("utf-8") if payload is not None else None - req = urllib.request.Request(url, data=data, headers=headers, method=method) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - body = resp.read().decode("utf-8") - return {"success": True, "result": json.loads(body) if body.strip() else {}} - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode("utf-8") - except Exception: - pass - hint = "" - if e.code == 501: - hint = " (write endpoints are disabled on this instance; make sure you are using sippy-auth)" - elif e.code in (401, 403): - hint = " (token missing/expired; use the oc-auth skill to obtain a fresh token)" - return {"success": False, "error": "HTTP %d: %s%s" % (e.code, e.reason, hint), "detail": detail} - except urllib.error.URLError as e: - return {"success": False, "error": "Failed to connect: %s" % e.reason} - - -def main(): - p = argparse.ArgumentParser(description="Create/update/delete Sippy symptoms") - p.add_argument("action", choices=["create", "update", "delete"]) - p.add_argument("--token", help="Bearer token (or set SIPPY_TOKEN env var, preferred; use oc-auth skill)") - p.add_argument("--id", help="Symptom ID (required for update/delete; server-generated on create)") - p.add_argument("--summary", help="Short unique description (required for create, max 200 chars)") - p.add_argument("--matcher-type", choices=list(VALID_MATCHERS), - help="How match_string is interpreted: string=substring, regex, " - "none=file exists, cel=CEL expression over label names") - p.add_argument("--file-pattern", help="Artifact glob, e.g. '**/build-log.txt'") - p.add_argument("--match-string", help="Substring, regex, or CEL expression") - p.add_argument("--label-ids", help="Comma-separated label IDs to apply on match") - p.add_argument("--skip-label-check", action="store_true", - help="Skip verifying label IDs against the labels API") - p.add_argument("--format", choices=["json", "summary"], default="json") - args = p.parse_args() - - token = resolve_token(args.token) - if not token: - print("Error: no token provided — pass --token or set the SIPPY_TOKEN " - "environment variable (preferred; use the oc-auth skill to obtain " - "one)", file=sys.stderr) - return 1 - - if args.action in ("update", "delete") and not args.id: - print("Error: --id is required for %s" % args.action, file=sys.stderr) - return 1 - - if args.action == "delete": - out = request("DELETE", symptom_url(WRITE_URL, args.id), token) - else: - label_ids = parse_label_ids(args.label_ids) - if args.action == "create": - payload = {"summary": args.summary, "matcher_type": args.matcher_type, - "file_pattern": args.file_pattern, "match_string": args.match_string, - "label_ids": label_ids or []} - else: - existing = get_json(symptom_url(READ_SYMPTOMS_URL, args.id)) - if existing is None: - print("Error: cannot fetch existing symptom %s (see error above; " - "check the ID with the list-symptoms skill)" % args.id, file=sys.stderr) - return 1 - existing["id"] = args.id - payload = build_update_payload(existing, summary=args.summary, - matcher_type=args.matcher_type, - file_pattern=args.file_pattern, - match_string=args.match_string, - label_ids=label_ids) - payload = {k: v for k, v in payload.items() if v is not None} - errs = validate_symptom(payload) - if not args.skip_label_check and payload.get("label_ids"): - errs += check_labels_exist(payload["label_ids"]) - if errs: - for e in errs: - print("Validation error: %s" % e, file=sys.stderr) - return 1 - if args.action == "create": - out = request("POST", WRITE_URL, token, payload) - else: - out = request("PUT", symptom_url(WRITE_URL, args.id), token, payload) - - if args.format == "json": - print(json.dumps(out, indent=2)) - else: - if out["success"]: - print("Symptom %s - SUCCESS" % args.action) - print(json.dumps(out.get("result", {}), indent=2)) - else: - print("Symptom %s - FAILED: %s" % (args.action, out.get("error"))) - if out.get("detail"): - print("Detail: %s" % out["detail"]) - return 0 if out["success"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/manage-symptoms/test_manage_symptoms.py b/plugins/ci/skills/manage-symptoms/test_manage_symptoms.py deleted file mode 100644 index 3c7d86664..000000000 --- a/plugins/ci/skills/manage-symptoms/test_manage_symptoms.py +++ /dev/null @@ -1,101 +0,0 @@ -import manage_symptoms -from manage_symptoms import (validate_symptom, build_update_payload, - parse_label_ids, check_labels_exist) - -def test_valid_string_symptom(): - assert validate_symptom({"summary": "AWS auth failure", "matcher_type": "string", - "file_pattern": "build-log.txt", "match_string": "AuthFailure", - "label_ids": ["InfraFailure"]}) == [] - -def test_missing_summary(): - errs = validate_symptom({"matcher_type": "string", "file_pattern": "f", "match_string": "x"}) - assert any("summary" in e for e in errs) - -def test_bad_matcher_type(): - errs = validate_symptom({"summary": "s", "matcher_type": "glob", "file_pattern": "f"}) - assert any("matcher_type" in e for e in errs) - -def test_non_cel_requires_file_pattern(): - errs = validate_symptom({"summary": "s", "matcher_type": "none"}) - assert any("file_pattern" in e for e in errs) - -def test_cel_requires_match_string_but_not_file_pattern(): - assert validate_symptom({"summary": "s", "matcher_type": "cel", - "match_string": "'A' in labels"}) == [] - errs = validate_symptom({"summary": "s", "matcher_type": "cel"}) - assert any("match_string" in e for e in errs) - -def test_string_and_regex_require_match_string(): - errs = validate_symptom({"summary": "s", "matcher_type": "regex", "file_pattern": "f"}) - assert any("match_string" in e for e in errs) - -def test_none_matcher_needs_no_match_string(): - assert validate_symptom({"summary": "s", "matcher_type": "none", "file_pattern": "f"}) == [] - -def test_summary_too_long(): - errs = validate_symptom({"summary": "a" * 201, "matcher_type": "none", "file_pattern": "f"}) - assert any("200" in e for e in errs) - - -EXISTING = {"id": "AWSAuthFailure", "summary": "AWS could not validate credentials", - "matcher_type": "string", "file_pattern": "build-log.txt", - "match_string": "api error AuthFailure", "label_ids": ["InfraFailure"]} - - -def test_update_overrides_match_string(): - out = build_update_payload(EXISTING, match_string="new pattern") - assert out["match_string"] == "new pattern" - assert out["id"] == "AWSAuthFailure" - assert out["summary"] == "AWS could not validate credentials" - - -def test_update_preserves_fields_when_not_passed(): - out = build_update_payload(EXISTING) - assert out == EXISTING - - -def test_update_empty_string_clears_match_string(): - out = build_update_payload(EXISTING, match_string="") - assert out["match_string"] == "" - - -def test_update_label_ids_preserved_and_overridden(): - assert build_update_payload(EXISTING)["label_ids"] == ["InfraFailure"] - out = build_update_payload(EXISTING, label_ids=["ClusterDNSFlake"]) - assert out["label_ids"] == ["ClusterDNSFlake"] - - -def test_update_empty_list_clears_label_ids(): - assert build_update_payload(EXISTING, label_ids=[])["label_ids"] == [] - - -def test_parse_label_ids(): - assert parse_label_ids(None) is None - assert parse_label_ids("") == [] - assert parse_label_ids("A, B,,C ") == ["A", "B", "C"] - - -def test_check_labels_exist_handles_bad_api_response(monkeypatch): - import manage_symptoms - for bad in (None, {"error": "x"}, "oops", [1, 2], ["str"]): - monkeypatch.setattr(manage_symptoms, "get_json", lambda url, _b=bad: _b) - errs = check_labels_exist(["InfraFailure"]) - assert errs and "could not verify label IDs" in errs[0] - - -def test_check_labels_exist_valid_response(monkeypatch): - import manage_symptoms - monkeypatch.setattr(manage_symptoms, "get_json", - lambda url: [{"id": "InfraFailure"}]) - assert check_labels_exist(["InfraFailure"]) == [] - assert check_labels_exist(["Nope"]) - - -def test_resolve_token_arg_wins_over_env(): - assert manage_symptoms.resolve_token("argtok", {"SIPPY_TOKEN": "envtok"}) == "argtok" - -def test_resolve_token_falls_back_to_env(): - assert manage_symptoms.resolve_token(None, {"SIPPY_TOKEN": "envtok"}) == "envtok" - -def test_resolve_token_none_when_unset(): - assert manage_symptoms.resolve_token(None, {}) is None diff --git a/plugins/ci/skills/payload-analysis/SKILL.md b/plugins/ci/skills/payload-analysis/SKILL.md index dc1254f2c..ec0353ee4 100644 --- a/plugins/ci/skills/payload-analysis/SKILL.md +++ b/plugins/ci/skills/payload-analysis/SKILL.md @@ -56,14 +56,6 @@ Before starting, you **MUST** load the following skills (they define output sche ### Step 1: Parse Arguments -**Anchor the output directory before anything else.** Capture the current working directory up front so all output files land in one stable, predictable location even if a later step changes directories: - -```bash -OUTPUT_DIR="$(pwd)" -``` - -All three output files — the payload results YAML (Step 6.5), the HTML report (Step 7), and the autodl JSON (Step 8) — MUST be written under `$OUTPUT_DIR`, never into a snapshot subdirectory or a path a later `cd` may have changed. The Step 10 self-check verifies them at `$OUTPUT_DIR`. - The first argument is a **full payload tag** (e.g., `4.22.0-0.nightly-2026-02-25-152806`). Parse from it: - `tag`: The specific payload tag to analyze - `version`: Extract from the tag (e.g., `4.22` from `4.22.0-0.nightly-...`) @@ -94,13 +86,6 @@ python3 "$SNAPSHOT_SCRIPT" After locating `summary.json`, set `SNAPSHOT_DIR` to the directory containing it. All relative paths in `summary.json` (e.g., `job_json`, `junit_results`, `build_log`, PR paths) resolve from this directory. -The snapshot script automatically prefers release-controller data and falls -back to Sippy for payloads that have been garbage collected. Do not truncate -the analysis chain merely because an originating tag is absent from the live -release controller; use the Sippy-backed `payloads[]` entry and its PR data. -Check each entry's `source` and `changelog_source` fields when provenance or -field completeness matters. - ### Step 3: Extract Failure Data from Snapshot Read `summary.json` to extract all data needed for analysis. The snapshot has already done the work of fetching payloads, building the chain, tracking streaks, and collecting PR data. @@ -108,27 +93,9 @@ Read `summary.json` to extract all data needed for analysis. The snapshot has al #### 3.1: Payload Metadata From `summary.json` top-level fields: -- `payload_tag`, `phase`, `release_url`, `source`, `architecture`, `stream`, `version` +- `payload_tag`, `phase`, `release_url`, `architecture`, `stream`, `version` - `chain_length`, `baseline_tag`, `hours_since_baseline` -**Record `phase` verbatim** from the `summary.json` metadata (`Accepted`, `Rejected`, or `Ready`). Never infer the phase from the job results or from whether failures exist — a payload can be `Accepted` *with* blocking failures (force-accepted) or `Ready` while jobs are still running. The stored phase drives the force-accept decision (Step 6.4) and the executive summary (Step 7.1), so an inferred phase silently corrupts both. - -#### 3.1b: If the Snapshot Is Incomplete, Collect the Data Yourself - -Check `summary.json` → `data_complete`. An absent `test_failure_count` means -*unknown*, not zero — never conclude a job had no test failures, and therefore -failed for some other reason, from missing data. - -When data is missing, collect it yourself from the job's `gcs_url` artifacts -rather than analyzing around the gap. Do the same for any payload in the chain -whose per-test data is missing. Report a gap as a limitation only when the -artifacts themselves are unreachable. - -An aggregated job with no per-test results at all is **unclassified**, not part -of a regression streak — aggregation also fails when too few child runs -completed or infrastructure killed them. Check the child runs: one that died -before the test phase cannot have failed a test. - #### 3.2: Failed Blocking Jobs From `summary.json` → `blocking_jobs.failed_jobs[]`, each entry contains: @@ -146,39 +113,10 @@ For each failed job's `streak.originating_payload`, find the matching entry in ` - `url`, `component`, `number`, `description` - Paths to local artifacts: `diff`, `comments`, `jobs` -Treat this as a **preliminary** list only. The job-level streak merges unrelated failure modes, so its originating payload is frequently earlier than the regression being investigated — and candidates gathered from it can omit the causal PR entirely. Before scoring, re-derive the originating payload **per failure mode** from `test_failures.blocking[].first_failed_in` (Step 5) and collect the candidates from *that* payload. - -For a Sippy-backed originating payload, the PR list remains usable for -candidate scoring and the normal GitHub diff/comment/job artifacts are still -collected. Sippy does not provide release-controller-only -`nodeImageStreams`, async jobs, or `previousAttemptURLs`; treat those fields as -unavailable rather than empty evidence. +These PRs are the **candidates** for failures that started in that originating payload. #### 3.4: Test Failure Details -Only `test_failures.blocking[]` contains failures that can reject the payload. **`test_failures.informing[]` and `test_failures.flakes[]` cannot fail a job or reject a payload** — never score them as candidate causes, never use them to derive a failure mode's originating payload, and never propose a revert for them. - -**"Informing job" ≠ "informing test."** These are two completely different -concepts that share a name: - -- **Informing job** (`informing_jobs.failed_jobs[]`): a CI *job* that runs - for visibility but does not gate the payload. Its pass/fail status is - job-level. An informing job can still contain blocking tests. -- **Informing test** (`test_failures.informing[]`): an individual *test case* - with `lifecycle="informing"`. It can appear inside any job — blocking or - informing. Its results never count toward `test_failure_count`. - -Never combine informing-job counts with informing-test lists. When -reporting informing/flake tests, list individual test names from -`test_failures.informing[]` / `test_failures.flakes[]` — do NOT report -informing *job* failure counts in the same section. - -Report informing and flake tests in their own section of the report, under a heading that says so, with this caveat: - -> These tests do not by themselves cause job failures or payload rejections. The only potential impact is if the test itself affects cluster health — for example if it breaks an operator or does something otherwise catastrophic. - -Keep them visible: informing tests are new tests being stabilized, and a badly-behaved test can occasionally damage the cluster it runs on. Investigate one only when there is evidence of that, and say plainly that it is not a rejection cause. - From `summary.json` → `test_failures.blocking[]`: - `test_name`, `jobs`, `first_failed_in`, `payloads_failing` - `failure_message`, `failure_text` (full, not truncated) @@ -229,8 +167,6 @@ If this finds nothing, also check steps that run earlier in the workflow and set **Scoring CI infrastructure candidates.** If a commit/PR modified a step that the failing job executes (or a shared dependency of that step), flag it as a **CI infrastructure candidate** — include it in Step 6.1 scoring alongside component PR candidates. When the failure's error messages reference URLs, domains, binaries, or configurations that were changed by the PR, the error message match signal (+40) should fire strongly. The key test: does the PR's diff introduce, modify, or remove something that appears in the error output? -**A causal CI-infrastructure change MUST appear as a scored entry in the `candidates[]` output**, exactly like a component PR — even when the overall `failure_type` is `infra`. Classifying a failure as infrastructure does not exempt its cause from structured output. Unlike a self-resolving lease/quota blip, a CI-config change is a persistent issue (Step 6.4) that needs a human fix or a revert, so it must be visible to the downstream revert/experiment commands, not buried in prose. - This step catches failures caused by CI tooling changes (mirror URL migrations, proxy configuration updates, script refactors) that are invisible to the snapshot's PR tracking. #### 3.7: RHCOS RPM Changes @@ -252,7 +188,7 @@ For each failed job, identify the matching RHCOS variant's RPM changes (if any) For each failed blocking job in the **target payload**, launch a **parallel subagent** to investigate the failure. Pass the subagent the Prow URL and all previous attempt URLs from Step 3.2. -Almost all blocking jobs install a cluster and then run tests, so the job name alone does not tell you the failure type. Each subagent therefore runs the `ci:prow-job-analysis` skill, which classifies the failure and routes to the correct specialized reference internally. +Each subagent should determine whether the failure is an install failure or a test failure by checking the JUnit results (e.g., look for `install should succeed*` test failures), then use the appropriate analysis skill. Almost all blocking jobs install a cluster and then run tests, so the job name alone does not tell you the failure type. You MUST use the following prompt verbatim (substituting the placeholder values) when launching each subagent. Do NOT paraphrase, shorten, or write your own prompt — the specific instructions below are critical for analysis quality: @@ -266,7 +202,11 @@ You MUST use the following prompt verbatim (substituting the placeholder values) > > **RHCOS RPM changes**: Read `` and find the entry in `payloads[]` whose `tag` equals ``. If that entry has an `rhcos_changes[]` array, look up the RHCOS variant matching this job's `rhcos_version` using the tag mapping: `rhel-coreos` → `rhcos9`/`rhcos9-default`, `rhel-coreos-10` → `rhcos10`/`rhcos10-default`, both apply to `rhcos9_10`. Check whether any changed, added, or removed RPM packages overlap with the failure's root cause. If the failure involves OS-level components (kernel, bootloader, systemd, SELinux, rpm-ostree, cri-o, crun, runc, networking) and matching packages changed, note the potential correlation in your ANALYSIS_RESULT. > -> Use the `ci:prow-job-analysis` skill for this investigation. It is the single entry point for every failed job: it identifies the job type, classifies the failure, and routes to the correct specialized reference — install, metal/bare-metal, test, upgrade, and more — internally. Do NOT pre-classify the failure yourself. Perform the full analysis, including downloading and analyzing must-gather when it is available. +> First, check the JUnit results or build log to determine whether this is an install failure (look for `install should succeed: overall` or similar install-related test failures) or a test failure (install passed, specific tests failed). +> +> Based on the failure type, use the appropriate skill: +> - **Install failure**: Use the `ci:prow-job-analyze-install-failure` skill. For metal/bare-metal jobs (job name contains "metal"), also perform analysis using the `ci:prow-job-analyze-metal-install-failure` skill for dev-scripts, Metal3/Ironic, and BareMetalHost-specific diagnostics. +> - **Test failure**: Use the `ci:prow-job-analyze-test-failure` skill. Do NOT use `--fast` — always perform the full analysis including must-gather extraction and analysis. > > **IMPORTANT** — Trace every failure to its specific root cause by examining actual logs. Never stop at high-level symptoms like "0 nodes ready", "operator degraded", or "containers are crash-looping". Download and read the actual log bundles, pod logs, and container previous logs. Cite specific error messages. The root cause must be actionable, not a restatement of the symptom. > @@ -343,24 +283,6 @@ Compare the subagent's root cause analysis for the target payload against previo If a job fails in two consecutive payloads but for **different reasons**, treat each as a separate streak=1 failure with its own originating payload and candidate PRs. Re-split the streak and re-assign originating payloads before proceeding to scoring. -**The job-level streak is not a failure mode's originating payload.** `streak.originating_payload` tracks when the *job* started failing, which merges unrelated modes — an infrastructure blip, a flake, and a real regression all read as one streak. Scoring candidates from a payload that predates the actual regression guarantees misattribution: the causal PR is not even in the candidate set. - -For each failure mode, take the originating payload from the matching `test_failures.blocking[]` entry's **`first_failed_in`**. When that is later than the job-level streak's, the job's earlier failures are a different mode — score from `first_failed_in`. Confirm the test passed in the preceding payload; where that payload has no per-test data, check its child runs rather than assuming it was failing. - -Establish this before enumerating candidate PRs (Step 6.1). When the two onsets differ, record both and state which drove scoring. - -### Step 5b: Adjudicate Conflicting Root Causes - -When two or more investigations reach **contradictory root causes for the same failure signature** (same test, same operation, or same error class — across jobs, across retries, or between a subagent and a previous analysis), the analysis is **UNRESOLVED**. It is *not* a tie to be broken by whichever explanation feels more plausible. Resolve it only with discriminating evidence, applying these rules: - -- **Discriminating evidence must come from the exact failing operation or phase** — the specific subcommand, step, or reconcile loop that actually errored, not from adjacent activity. -- **"Cleared" requires positive evidence from the failing code path.** A candidate is exonerated only by positive evidence that its code path executed and completed without error *during the failing operation itself*. A candidate succeeding in a *different* subcommand, phase, or job does **not** clear it. -- **Absence of a log line is not evidence when the log is truncated.** If the relevant log was truncated, rotated, or never captured, treat the missing line as *unknown*, never as proof that a code path did not execute. -- **A causal chain must be shown to execute, not merely shown to be possible.** Demonstrate that the proposed mechanism actually ran during the failing operation (via timestamps, ordering, or an emitted log/metric). "This change *could* cause this" is a hypothesis, not a root cause. -- **When you override a subagent's conclusion, update the stored per-job root cause** so the streak data, YAML, JSON, and HTML all reflect the adjudicated cause. Divergent per-job root causes across outputs are a defect (checked in Step 10). - -**Tenacity booster:** Finding a plausible mechanism is the *midpoint* of the investigation, not the end. When rival explanations exist, your job is to *discriminate between them* with evidence from the failing operation — not to stop at the first mechanism that could work. If the evidence cannot discriminate, record the failure mode as UNRESOLVED with its competing hypotheses rather than committing to a guess (a wrong-PR attribution is far more damaging than an honest "unresolved"). - ### Step 6: Collect Investigation Results and Identify Revert Candidates Wait for all subagents to complete and collect their analysis results. For each failed job, you now have: @@ -376,46 +298,20 @@ For each failed job, cross-reference the failure analysis from the subagent with If a subagent traced the root cause to a PR outside the payload (e.g., an `openshift/release` PR that modified a CI step registry script), include that PR as a candidate. -**Before scoring, mechanically enumerate every distinct failure mode for each job — do not score only the dominant one.** A single job can fail for more than one reason (e.g., an install timeout *and* an unrelated test regression). For each failed job, first write out each distinct failure mode the subagent identified as an explicit list, then run every candidate PR through the rubric **once per failure mode** — a PR that explains failure mode A does not automatically explain failure mode B. Do not collapse a job down to its loudest symptom and score only that. Any failure mode you dismiss as a flake (or as pre-existing) MUST cite the specific evidence for that dismissal — a passing retry with no code change, the same test failing on the *accepted* baseline, or a known-flaky test ID — never an unsupported "intermittent" label. - -Score each (failed job, failure mode, candidate PR) tuple using the following weighted rubric: +Score each (failed job, candidate PR) pair using the following weighted rubric: | Signal | Weight | Criteria | |--------|--------|----------| -| New failure mode | +30 | This failure mode was not present in previous payloads **and** is plausibly attributable to code that changed (some PR touches the implicated code path). A brand-new symptom with no changed code behind it does not earn this signal (see infrastructure exclusion below). | -| Component exclusivity | +10 to +30 | The failure involves a component modified by this PR. **Sole modifier of the affected component = +30** — this tier already covers the "only one candidate PR touches the component" case, so do not also count it separately. 2-3 PRs modify the component = +20; 4+ PRs modify it = +10. | -| Error message match | +10 to +40 | Tiered by how directly the failure output links to the PR's diff. **Direct match = +40**: an error string, symbol, function name, or identifier from the failure appears verbatim in the PR's diff. **Same code path = +20-30**: the PR modifies the function or execution flow that produced the error, but the exact message is not in the diff. **Same subsystem only = +10**: the PR touches the same subsystem/component but not the specific failing code path. | -| Multi-job correlation | +10 | The same PR is a candidate for this failure mode in multiple independent jobs | +| New failure mode | +30 | The specific failure mode was not present in previous payloads | +| Component exclusivity | +10 to +30 | The failure involves a component modified by this PR. Sole modifier = +30, 2-3 PRs = +20, 4+ PRs = +10 | +| Error message match | +40 | Error messages or stack traces directly reference code, packages, or functionality changed by this PR | +| Multi-job correlation | +10 | The same PR is a candidate for failures in multiple independent jobs | | Presubmit coverage gap | +10 | The failing job tests a scenario not covered by the PR's presubmit tests | +| Single candidate | +10 | Only one PR landed in the originating payload that touches the affected component | -Maximum possible score is 120, capped at 100. Record the numeric score alongside qualitative rationale. - -**Every candidate's rationale MUST itemize the score** — one line per signal that fired — so the number is auditable rather than asserted: - -``` -signal_name: +points — one line of concrete evidence -``` - -For example: - -``` -error_message_match: +40 — panic "nil pointer in reconcileNode" from build-log appears verbatim in the PR diff (controller.go:214) -component_exclusivity: +30 — sole PR modifying machine-config-operator in the originating payload -new_failure_mode: +30 — job passed the 6 prior payloads; first failed in the originating payload -total: 100 -``` - -Record this breakdown in the candidate's `rationale` field in the YAML/JSON output. A bare score with no itemized breakdown is not acceptable. - -The recorded confidence score MUST equal `min(100, sum of itemized signals)`, each signal at exactly its defined weight, one line of evidence per claimed signal. No unclaimed points, no unlisted signals. - -**Apply the rubric mechanically, then verify the top-tier claims.** Sum the weights for each signal that fires on concrete evidence. Do NOT adjust the score downward based on speculative counter-arguments like "if this were the sole cause, other jobs would also fail" or "this could be a coincidence" — if the error messages reference the PR's changes, that's a match, and the fact that some other jobs didn't fail doesn't negate it. **But when the raw sum exceeds the cap** (you claimed a maximum tier on more than one signal at once), re-verify each maximum-tier claim before recording: is the error-message match a true verbatim string/symbol match (+40), or really only same-subsystem (+10)? Is this genuinely the *sole* modifier of the component (+30)? Downgrade any tier that does not survive this check. This self-skepticism pass removes tier inflation without weakening genuinely strong matches. Trust the rubric — it exists to prevent both over- and under-attribution. - -**Infrastructure exclusion — do not let unrelated PRs accumulate points.** The rubric measures *product-code causation*. When the root cause is affirmatively infrastructure (Step 6.4 definition) or an affirmatively-identified CI-config change (Step 3.6), payload component PRs with **no error-message and no code-path correlation** to the failure must score **at or near zero**. Do not award "new failure mode" or bare "component exclusivity" points to a PR that merely happens to be present in the payload — "new failure mode" fires only when the failure is plausibly attributable to code that changed. A new symptom whose actual cause is a lease timeout, a quota block, or a step-registry edit is not evidence against an unrelated component PR. - -**"Intermittent" and "flake" are conclusions requiring evidence, not default labels.** Before dismissing a failure as a flake, confirm affirmative evidence for it (e.g., the same job passed on retry with no code change, or it is a known-flaky test that also fails on *accepted* payloads). First check whether any candidate PR touches the failing code path: a reproducible failure in code that changed is a regression, not a flake, even if it does not reproduce on every run. +Maximum possible score is 130, capped at 100. Record the numeric score alongside qualitative rationale. -**When a failed job ends up with zero causally-linked candidates, state why — explicitly, per job.** An empty candidate list is itself a claim: that no payload PR and no CI-infrastructure change (Step 3.6) is causally linked to the failure. Justify it rather than leaving it blank. For each such job, record a one-line rationale explaining why no payload PR explains the failure (e.g., "root cause is a Boskos lease timeout — no component PR touches the failing path"; "failure also reproduces on the accepted baseline payload, so it predates every candidate PR in this originating payload"). State the cross-job correlation explicitly: note whether the same failure mode appears in other failing jobs (pointing to shared infrastructure or a common dependency) or is isolated to this one. A silent empty candidate list is indistinguishable from an un-investigated job and is not acceptable. +**Apply the rubric mechanically.** Calculate the score by summing the weights for each signal that fires based on concrete evidence. Do NOT adjust the score downward based on speculative counter-arguments like "if this were the sole cause, other jobs would also fail" or "this could be a coincidence." If the error messages reference the PR's changes, that's +40 — the fact that some other jobs didn't fail doesn't negate the match. If the failure is new (streak=1) and the PR is the only one touching the component, those signals fire regardless of theoretical alternatives. Trust the rubric — it exists to prevent both over- and under-attribution. #### 6.1b: RHCOS RPM Change Correlation @@ -452,14 +348,6 @@ For each revert candidate, record: PR URL, description, component, confidence sc **Do NOT propose reverts for**: Infrastructure failures, flaky tests that also fail on accepted payloads, jobs where analysis is inconclusive. -**Special case — Kubernetes rebase version skew.** When the candidate PR is a **Kubernetes rebase** (a PR in the `openshift/kubernetes` repo, typically a rebase/version-bump onto a new upstream Kubernetes release) **AND** the failures show **kubelet version skew** — the kubelet reporting an older Kubernetes version than the kube-apiserver (e.g., kube-apiserver at `1.36.2` while nodes still run kubelet `1.35.3`) — do **NOT** mark the rebase as a revert candidate, even when its rubric score is >= 85. - -The kubelet binary is built **from** the `openshift/kubernetes` source. After a rebase merges, there is an expected lag of hours while the kubelet is rebuilt against the new source and delivered via an updated RHCOS image. During this window the skew is **transient build lag**, not a regression: - -- Reverting the rebase would only prevent the kubelet from ever picking up the new version — it does not fix anything. -- In the structured output, `failure_type` MUST stay one of the fixed enum values (`install`/`test`/`upgrade`/`infra`) — set it to **`test`**, since kubelet version skew produces real test failures. Do **not** put "transient build lag" in `failure_type`. Record the **"transient build lag"** classification in the free-text `root_cause_summary` (e.g., `"kubelet version skew — transient build lag pending kubelet rebuild"`), and treat it as build lag rather than a revert candidate. -- In the report, record the failure as pending kubelet rebuild: the kubelet must be rebuilt from the rebased source and delivered via an updated RHCOS image. Recommend **monitoring for the rebuilt kubelet (updated RHCOS) to land** so the skew resolves on its own. - #### 6.3: Check if Revert Candidates Were Already Reverted For each revert candidate: @@ -475,37 +363,21 @@ If a revert PR is found: #### 6.4: Determine Force-Accept Recommendation -Force-accepting is only meaningful for a payload that has **not** already been accepted. **If the snapshot's `phase` (Step 3.1) is already `Accepted`, `force_accept_recommended` MUST be `false`** — the question is moot, so do not recommend it regardless of the failures present. - -Otherwise, recommend force-accepting when **all** of the following are true: +Recommend force-accepting when **all** of the following are true: -1. All failures are **temporary** infrastructure issues (`failure_type: "infra"`) — see the definition below +1. All failures are temporary infrastructure issues (`failure_type: "infra"`) 2. No more than 2 blocking jobs failed 3. `hours_since_baseline` from `summary.json` is >= 18 (or null) -**What counts as a "temporary infrastructure issue".** The decisive test: *will the failure self-resolve on the next run WITHOUT human action?* - -- **Yes → temporary (force-accept eligible):** Boskos/lease acquisition failures, cloud quota exhaustion, transient cloud-provider API errors or throttling, a one-off network timeout to a cloud endpoint, a CI control-plane blip. These clear themselves on retry. -- **No → persistent (NOT force-accept eligible):** stale or expired credentials, a broken or misconfigured CI step/workflow, a bad mirror/registry URL, a persistent misconfiguration, or any product regression. These fail again on the next run until a human intervenes — force-accepting only defers the problem. Do not classify these as a temporary infra pass; a causal CI-config change is scored as a candidate (Steps 3.6 and 6.1) instead. - -**Guard — kubelet version skew from a Kubernetes rebase.** When the blocking failures are caused by kubelet version skew following a Kubernetes rebase (the "transient build lag" case in Step 6.2), recommend **neither force-accept nor force-reject**: - -- **Force-accepting is NOT appropriate.** Kubelet version skew produces real test failures, not temporary infrastructure flakes, so it does not satisfy the `failure_type: "infra"` criterion above. Set `force_accept_recommended` to `false`. -- **Force-rejecting is also NOT appropriate.** Rejecting the payload only makes the next payload assemble sooner, which will hit the **same** skew unless the RHCOS carrying the rebuilt kubelet is ready by then — so it accomplishes nothing except churn. - -Instead, recommend the correct action: **wait for the RHCOS with the rebuilt kubelet to land** (i.e., for the transient build lag from Step 6.2 to resolve). Once the updated kubelet is delivered, the skew clears and the payload passes on its own. - #### 6.5: Write Payload Results YAML -Use the `payload-results-yaml` skill to create `$OUTPUT_DIR/payload-results-{tag}.yaml` (the `$OUTPUT_DIR` captured in Step 1) +Use the `payload-results-yaml` skill to create: `payload-results-{tag}.yaml` This file contains ALL scored candidates across all confidence tiers (HIGH, MEDIUM, LOW), enabling downstream commands to filter by their own criteria. If RHCOS RPM suspects were identified in Step 6.1b, include them in the `rhcos_suspects[]` array (see the `payload-results-yaml` skill for the schema). -**Every affirmatively-identified root cause must be represented as a scored `candidates[]` entry** — including causal CI-infrastructure / step-registry changes (Step 3.6), even when the failure's `failure_type` is `infra`. A failure whose cause is known must not leave `candidates[]` empty; each entry carries its itemized rubric breakdown (Step 6.1) in its `rationale`. - ### Step 7: Generate HTML Report -Create a self-contained HTML file named `payload-analysis--summary.html` in `$OUTPUT_DIR` (the directory captured in Step 1). The tag should be sanitized for use as a filename. +Create a self-contained HTML file named `payload-analysis--summary.html` in the current working directory. The tag should be sanitized for use as a filename. The report must include the following sections: @@ -521,18 +393,12 @@ The report must include the following sections:

Executive Summary

-

Phase: {phase}

{total_blocking} blocking jobs: {succeeded} passed, {failed} failed

{new_failures} new failure(s), {persistent_failures} persistent failure(s)

Chain: {chain_length} payloads, {hours_since_baseline}h since baseline

-

Consecutive rejections in this stream: {consecutive_rejection_count}

-

Last accepted: {baseline_tag} ({hours_since_baseline}h ago)

-

Per-job persistence: {for each failed job — "job_name: failing N consecutive payloads"}

``` -**Payload-chain context** surfaces the streak at a glance — include all of the fields above. Derive them from the snapshot: `consecutive_rejection_count` is the number of consecutive non-`Accepted` payloads in the chain up to and including this one (the chain runs from the last accepted baseline forward — see `chain_length` and the `payloads[]` phases); the last accepted payload is `baseline_tag`, cut `hours_since_baseline` hours ago; per-job persistence is each failed job's `streak.streak_length` (how many consecutive payloads that specific job has been failing). Render `phase` verbatim from Step 3.1. - #### 7.2: Blocking Jobs Summary Table A table showing ALL blocking jobs with columns: @@ -709,7 +575,7 @@ Include these RHCOS-specific styles: ### Step 8: Generate JSON Data File -Use the `payload-autodl-json` skill to produce `$OUTPUT_DIR/payload-analysis--autodl.json` (the `$OUTPUT_DIR` captured in Step 1). +Use the `payload-autodl-json` skill to produce `payload-analysis--autodl.json`. See the `payload-autodl-json` skill for the complete schema, row cardinality rules, and field rules. @@ -739,13 +605,13 @@ Use this prompt for the reviewer: > > Check for these specific problems: > -> 1. **Missing skill invocations**: Was the `prow-job-analysis` skill actually loaded and used? A subagent that improvises without loading the appropriate skill produces shallow analysis. +> 1. **Missing skill invocations**: Were `prow-job-analyze-install-failure` and `prow-job-analyze-test-failure` skills actually loaded and used? A subagent that improvises without loading the appropriate skill produces shallow analysis. > > 2. **Shallow root causes**: Do root cause summaries cite specific error messages, code paths, or log excerpts? Or do they just restate test names and job status? "Test X failed" is not a root cause. "Test X failed because pod Y OOMKilled at 512Mi limit after PR Z increased memory usage in function F" is a root cause. > > 3. **Incomplete coverage**: Are there failed jobs with no subagent analysis or with only a one-line summary? Every failed blocking job deserves a thorough investigation. > -> 4. **Wrong reference for failure type**: Did the analysis route to the correct reference — install (and metal for metal jobs) for install failures, and the test/flaky-test reference for test failures? Using the wrong reference produces misdirected analysis. +> 4. **Wrong skill for failure type**: Was an install failure analyzed with the test failure skill or vice versa? > > 5. **Missing RHCOS RPM correlation**: If RHCOS RPM changes exist in the originating payload and failures are variant-isolated or involve OS-level components, was the correlation checked? Were relevant packages surfaced as suspects? > @@ -768,21 +634,14 @@ After receiving the reviewer's response: - **Never lower rubric-based confidence scores** based on the reviewer's response. The rubric is mechanical — if the signals fired, the score stands. - Populate the "Adversarial Review" section (Step 7.6) in the HTML report with the reviewer's findings and any actions taken. -### Step 10: Final Self-Check, Save, and Present - -Before presenting, confirm that **all Step 4 investigation subagents and the Step 9 reviewer have completed and returned their results** — never assemble the report while an investigation is still outstanding. Then run a **mechanical self-check** and fix any gap it finds — do not present a partial report: - -1. **All three output files exist at `$OUTPUT_DIR`** (the directory captured in Step 1) and are non-empty. Verify these three exact spec'd filenames — do NOT glob, so that a stray file from a previous run cannot satisfy the check: - - HTML report: `$OUTPUT_DIR/payload-analysis--summary.html` - - JSON data file: `$OUTPUT_DIR/payload-analysis--autodl.json` - - Payload results YAML: `$OUTPUT_DIR/payload-results-.yaml` -2. **The HTML contains every required section** from Step 7: header + executive summary (including the payload-chain context from Step 7.1), recommended reverts (or the "No Recommended Reverts" verdict), the force-accept verdict when applicable, the blocking-jobs summary table, a collapsible details block for **every** failed job, the RHCOS Changes section when any payload has RHCOS changes, and the Adversarial Review section. -3. **Cross-output consistency**: phase, failure counts, per-job root causes (including any adjudicated in Step 5b), and scored candidates agree across the HTML, YAML, and JSON. -4. **Every affirmative root cause appears as a scored `candidates[]` entry** — including causal CI-infrastructure changes, even when `failure_type: infra`. +### Step 10: Save and Present -If any check fails, fix it before presenting. +1. Save all output files to the current working directory: + - HTML report: `payload-analysis--summary.html` + - JSON data file: `payload-analysis--autodl.json` + - Payload results YAML: `payload-results-.yaml` -Then tell the user: +2. Tell the user: - Path to each saved file - Brief text summary (number of failures, new vs persistent, key candidate PRs) - Whether the adversarial review changed any conclusions @@ -824,6 +683,7 @@ Note: PR diff data not available in snapshot. Scoring based on component match a - Related Skill: `payload-snapshot` — creates the snapshot data this skill consumes - Related Skill: `payload-results-yaml` — schema for the results YAML - Related Skill: `payload-autodl-json` — schema for the autodl JSON data file -- Related Skill: `prow-job-analysis` — deep test/install failure investigation (used by subagents) +- Related Skill: `prow-job-analyze-test-failure` — deep test failure investigation (used by subagents) +- Related Skill: `prow-job-analyze-install-failure` — deep install failure investigation (used by subagents) - Related Command: `/ci:payload-revert` — stages reverts for high-confidence candidates - Related Command: `/ci:payload-experiment` — tests medium-confidence candidates experimentally diff --git a/plugins/ci/skills/payload-autodl-json/SKILL.md b/plugins/ci/skills/payload-autodl-json/SKILL.md index bfbe08f56..70ce752c0 100644 --- a/plugins/ci/skills/payload-autodl-json/SKILL.md +++ b/plugins/ci/skills/payload-autodl-json/SKILL.md @@ -150,7 +150,7 @@ The filename **must** end with `-autodl.json`. Sanitize the tag for filename saf |-------|------|-------------| | `payload_tag` | string | Full payload tag | | `version` | string | OCP version (e.g., `"4.22"`) | -| `stream` | string | Canonical stream type: `"nightly"` or `"ci"`. For OKD payloads (e.g. `okd-scos-nightly`, `okd-scos`), map to the base type: `okd-scos-nightly` → `"nightly"`, `okd-scos` → `"ci"`. | +| `stream` | string | `"nightly"` or `"ci"` | | `architecture` | string | `"amd64"`, `"arm64"`, `"multi"`, etc. | | `phase` | string | Payload phase: `"Rejected"`, `"Accepted"`, `"Ready"` | | `release_controller_url` | string | URL to the payload on the release controller | diff --git a/plugins/ci/skills/payload-snapshot/SKILL.md b/plugins/ci/skills/payload-snapshot/SKILL.md index d425d6b3d..3ddc20245 100644 --- a/plugins/ci/skills/payload-snapshot/SKILL.md +++ b/plugins/ci/skills/payload-snapshot/SKILL.md @@ -28,10 +28,8 @@ Use this skill when you need to: 3. **Google Cloud SDK (`gcloud`)** — for JUnit test result download - Install: `brew install google-cloud-sdk` (macOS) or see https://cloud.google.com/sdk - - **Authentication is not required** — the CI artifact buckets are public and - are read anonymously when no account is configured - - Without `gcloud` entirely, JUnit data is skipped and the snapshot is - reported as incomplete (see [Data completeness](#data-completeness)) + - Authenticate: `gcloud auth login` + - Without `gcloud`, JUnit data is skipped; job directories still created 4. **Container runtime (`podman`)** — for RHCOS RPMDB extraction - Install: available in most Linux distributions; `brew install podman` on macOS @@ -40,7 +38,6 @@ Use this skill when you need to: 5. **Network access** to: - `*.ocp.releases.ci.openshift.org` (release controller) - - `sippy.dptools.openshift.org` (historical payload fallback) - `api.github.com` (via `gh` CLI) - `storage.googleapis.com` (via `gcloud` CLI) @@ -65,16 +62,13 @@ python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --no-junit # Skip RPMDB extraction python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --no-rpmdb - -# Force Sippy for all payload metadata (normally fallback is automatic) -python3 "$script_path" 4.22.0-0.nightly-2026-02-25-152806 --sippy ``` The script will: 1. Parse the payload tag to determine version, stream, and architecture 2. Probe all available streams for the version (nightly, ci, across architectures) -3. Chain backwards through Sippy's time-ordered release tag list until finding one where all blocking jobs passed, restoring tags garbage collected from the release controller -4. For each payload in the chain, prefer release controller data and changelogs; fall back per historical tag or cross-tag diff to Sippy payload, PR, and job data +3. Chain backwards through previous payloads until finding one where all blocking jobs passed +4. For each payload in the chain, download release controller data and the changelog (PR diff) 5. Split jobs into blocking/informing directories with metadata and GCS browser links 6. For each failed blocking job, download and parse JUnit XML test results 7. For each failed blocking job, download build-log.txt from GCS and extract error/warning lines + log tail @@ -171,9 +165,6 @@ Options: --workers N Parallel workers for API calls (default: 8) --no-junit Skip JUnit download and regression tracking --no-rpmdb Skip RHCOS RPMDB extraction - --sippy Force Sippy for all payload metadata instead of using - the automatic release-controller-first fallback - --fail-on-incomplete Exit 1 if any requested data could not be collected ``` ## Output Files @@ -185,97 +176,13 @@ Lists all available streams for the payload's version. ### `summary.json` Comprehensive stream-level triage data — start here. Contains: -- Payload metadata: `payload_tag`, `phase`, `release_url`, `source`, `architecture`, `stream`, `version` +- Payload metadata: `payload_tag`, `phase`, `release_url`, `architecture`, `stream`, `version` - Chain data: `chain_length`, `baseline_tag`, `hours_since_baseline` - `blocking_jobs.failed_jobs[]` — detailed objects with `name`, `state`, `prow_url`, `gcs_url`, and relative path `job_json`. May include: `rhcos_version`, `streak` (with `streak_length`, `originating_payload`, `is_new_failure`, `failure_pattern`), `build_log_errors`, `test_failure_count`, and relative paths `junit_results`, `build_log` - `informing_jobs.failed_jobs[]` — job name strings -- `test_failures.blocking[]` — **gating** failures only: `test_name`, `jobs`, `first_failed_in`, `payloads_failing`, `failure_message`, `failure_text` (full, not truncated). These are the failures that can fail a job and therefore reject the payload. -- `test_failures.informing[]` / `test_failures.flakes[]` — `test_name`, `jobs`. Neither can fail a job. No onset is tracked for them, because an onset implies there is a culprit to find. -- `payloads[]` — per-payload entries with `tag`, `phase`, `source`, `changelog_source`, relative file paths, `prs[]` with component/diff/comments paths, and `rhcos_changes[]` with RPM diffs per RHCOS variant +- `test_failures.blocking[]` — `test_name`, `jobs`, `first_failed_in`, `payloads_failing`, `failure_message`, `failure_text` (full, not truncated) +- `payloads[]` — per-payload entries with `tag`, `phase`, relative file paths, `prs[]` with component/diff/comments paths, and `rhcos_changes[]` with RPM diffs per RHCOS variant - `rhcos_rpms[]` — RPMDB metadata for the target payload's RHCOS variants: `tag`, `name`, `pullspec`, `rpmdb` (relative path to rpmdb.sqlite) -- `data_complete` — `true` when all requested data was ultimately collected, including via a fallback after an initial read failed. `false` means some requested data could not be read at all. -- `collection_errors[]` — every read failure encountered. Each entry has `reason`, `command`, and optionally `detail`, `stage`, `job`, `payload_tag`, `recovered`. Reasons: `auth`, `timeout`, `gcloud_missing`, `command_failed`, `junit_unavailable` (nothing readable), `junit_missing` (nothing discovered), `junit_unparseable` (corrupt XML), `junit_partial` (some files unread), `build_log_unavailable`. - - `recovered: true` means a fallback subsequently obtained the data. These entries are diagnostic only (useful for spotting a timeout that needs tuning) and do **not** make `data_complete` false. - - `data_complete` is `false` only when at least one error was **not** recovered. - - -#### Only gating results count as failures - -A test result falls into exactly one of three categories, and only the last -can fail a job or reject a payload: - -| Category | Rule | Gates? | -|---|---|---| -| flake | the same test, in the same suite, both failed and passed | no | -| informing | the testcase carries `lifecycle="informing"` | no | -| failure | failed everywhere, no `informing` lifecycle | **yes** | - -Informing tests are run to stabilize them and are not expected to gate. A -missing `lifecycle` attribute means the test **does** gate — the attribute -exists only to opt a test out. - -`results.json` records all three so nothing is hidden, each entry carrying -`status` (`failed`, `error`, `flake`) and `test_lifecycle` (`blocking`, -`informing`). - -`test_failure_count` on a failed job counts **only gating** results — it is -the failure count, and flakes and informing tests are not added to it. -Those two are listed by name under `test_failures.flakes[]` and -`test_failures.informing[]`. Regression onset (`first_failed_in`) is derived -from gating failures alone. - -#### "Informing job" vs "informing test" — two unrelated concepts - -The word "informing" appears in two places with **completely different -meanings**. Confusing them produces wrong analysis: - -| Concept | Where it lives | What it means | -|---|---|---| -| **Informing job** | `informing_jobs.failed_jobs[]` in summary.json | A CI *job* that runs for visibility but does **not** gate the payload. Job-level pass/fail. | -| **Informing test** | `test_failures.informing[]` in summary.json | An individual *test case* whose `lifecycle="informing"` attribute opts it out of gating. Can appear inside **any** job — blocking or informing. | - -An informing *test* can run inside a *blocking* job. -An informing *job* can contain *blocking* tests. -They are orthogonal. Never combine them in the same section or count. - -#### gcloud credentials are not required - -The CI artifact buckets are public. When gcloud has no active account it is -run in anonymous mode automatically, so an unauthenticated environment still -produces a complete snapshot. Authenticate only if you also need private -buckets. - -#### Missing data is absent, never empty - -When a collection step fails, the affected file is **not written** and the -corresponding summary field is **omitted** — it is never emitted as an empty -list or a zero count. Specifically: - -- If JUnit could not be read for a job, `results.json` is not created, the - job entry has **no** `test_failure_count` and `junit_results`, and instead - carries `junit_collection_failed: true`. -- An absent `test_failure_count` therefore means *unknown*, whereas `0` means - *verified clean*. - -- If **some** JUnit files were read and others were not, `results.json` holds - the real results that were obtained and the job entry adds - `junit_collection_partial: true`. `test_failure_count` is then a **lower - bound**, not a total. -- A failed job with **no** readable JUnit at all — none discovered - (`junit_missing`), or nothing that parsed (`junit_unparseable`) — is left - without a count entirely. A job whose tests may never have run is unknown, - not clean. A parse failure alongside other readable files is the partial - case above, not this one. - -Consumers **must** distinguish these. Treating unreadable data as "no test -failures" makes a broken job look like it failed for some other reason, which -misdirects root-cause analysis. Check `data_complete` before drawing any -conclusion from an absence of test failures. Use `--fail-on-incomplete` in -automation to exit non-zero rather than emit a partial snapshot. - -State is persisted to `collection_errors.json` beside `summary.json`, so a -later process can tell that an existing `results.json` came from an incomplete -read. ### `AGENTS.md` / `CLAUDE.md` @@ -347,18 +254,6 @@ PR artifacts from GitHub (unchanged from previous version). The script chains backwards from the target payload until it finds a payload where **all blocking jobs succeeded**. This is stricter than the `Accepted` phase — a payload can be force-accepted with failed blocking jobs, which does not count as a stop point. -Sippy's release tag list, sorted by `release_time`, is used to identify every -preceding assembled payload. If a tag is still retained, its payload details -and changelog come from the release controller. If it has been garbage -collected, the script constructs compatible `payload.json` and -`changelog.json` files from Sippy's release tags, pull requests, and job runs -APIs. A changelog that crosses a garbage-collected tag also comes from Sippy -because the release controller can no longer compute that diff. - -The generated `source` and `changelog_source` fields expose this provenance. -Sippy-backed data is intentionally partial: RHCOS `nodeImageStreams`, async -jobs, and `previousAttemptURLs` are unavailable. - For terminal payloads (Accepted/Rejected), jobs showing `Pending` on the release controller are cross-checked against the actual Prow `prowjob.json` artifact to get their real state. ## Aggregated Jobs @@ -372,16 +267,10 @@ Aggregated jobs run the same underlying test multiple times with statistical ana - **Tag not found**: Exits with code 2 and a descriptive error - **Release controller unreachable**: Exits with code 1 -- **Historical tag missing from release controller**: Automatically uses Sippy - **`gh` not authenticated**: Prints a warning and continues without PR data -- **`gcloud` not available**: Warns, skips JUnit download, and records - `gcloud_missing` so the snapshot is reported incomplete -- **`gcloud` not authenticated**: Reads the public buckets anonymously -- **Individual job/PR fetch failure**: Logs a warning, records a collection - error, and continues -- **Idempotent**: Re-running skips files that already exist — except JUnit - output a previous run recorded as incomplete, which is discarded and - re-collected so a partial snapshot cannot be inherited as complete +- **`gcloud` not available**: Prints a warning and skips JUnit download +- **Individual job/PR fetch failure**: Logs a warning and continues +- **Idempotent**: Re-running skips files that already exist ## Notes diff --git a/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py b/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py index 549ba55f9..42941a460 100755 --- a/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py +++ b/plugins/ci/skills/payload-snapshot/scripts/payload_snapshot.py @@ -17,7 +17,6 @@ import subprocess import sys import tempfile -import threading import time import urllib.error import urllib.parse @@ -48,14 +47,6 @@ "error": "Failed", } -FALLBACK_FETCH_ERRORS = ( - urllib.error.HTTPError, - urllib.error.URLError, - json.JSONDecodeError, - TimeoutError, - SystemExit, -) - # --------------------------------------------------------------------------- # PayloadTag — immutable value object for parsed payload tags @@ -256,47 +247,17 @@ def resolve_prow_state(self, prow_url: str) -> Optional[str]: # --------------------------------------------------------------------------- class SippyClient: - """Client for Sippy's release-controller mirror APIs.""" + """Client for Sippy APIs, used when release controller data is unavailable.""" SIPPY_BASE = "https://sippy.dptools.openshift.org/api" - SIPPY_UI = "https://sippy.dptools.openshift.org/sippy-ng" - def __init__( - self, release: str, architecture: str = "amd64", stream: str = "nightly" - ): + def __init__(self, release: str): self.release = release - self.architecture = architecture - self.stream = stream self._tags_cache: Optional[list] = None - self._job_runs_cache: dict[str, list[dict]] = {} - - @staticmethod - def _filter(column: str, value: str) -> dict: - return { - "columnField": column, - "operatorValue": "equals", - "value": value, - } - - @classmethod - def _encoded_filter(cls, *items: tuple[str, str]) -> str: - filter_json = { - "items": [cls._filter(column, value) for column, value in items] - } - return urllib.parse.quote(json.dumps(filter_json)) def _get_tags(self) -> list[dict]: if self._tags_cache is None: - filter_value = self._encoded_filter( - ("architecture", self.architecture), - ("stream", self.stream), - ) - url = ( - f"{self.SIPPY_BASE}/releases/tags" - f"?filter={filter_value}" - f"&release={urllib.parse.quote(self.release)}" - "&sortField=release_time&sort=desc" - ) + url = f"{self.SIPPY_BASE}/releases/tags?release={urllib.parse.quote(self.release)}" self._tags_cache = fetch_json(url, timeout=60) return self._tags_cache @@ -306,55 +267,29 @@ def find_tag(self, tag_name: str) -> Optional[dict]: return t return None - def fetch_tags(self) -> list[dict]: - """Fetch release tags newest first for this release/arch/stream.""" - return self._get_tags() - def fetch_job_runs(self, tag_name: str) -> list[dict]: - if tag_name in self._job_runs_cache: - return self._job_runs_cache[tag_name] - filter_value = self._encoded_filter(("release_tag", tag_name)) + filter_json = json.dumps({"items": [ + {"columnField": "release_tag", "operatorValue": "equals", + "value": tag_name} + ]}) url = (f"{self.SIPPY_BASE}/releases/job_runs" - f"?filter={filter_value}" - f"&sortField=kind&sort=asc&limit=1000") - runs = fetch_json(url, timeout=60) - self._job_runs_cache[tag_name] = runs - return runs - - def fetch_pull_requests(self, tag_name: str) -> list[dict]: - """Fetch PRs introduced by a payload from Sippy.""" - filter_value = self._encoded_filter(("release_tag", tag_name)) - url = ( - f"{self.SIPPY_BASE}/releases/pull_requests" - f"?filter={filter_value}" - "&sortField=pull_request_id&sort=asc&limit=1000" - ) + f"?filter={urllib.parse.quote(filter_json)}" + f"&sortField=kind&sort=asc&limit=200") return fetch_json(url, timeout=60) - def fetch_changelog( - self, tag_name: str, from_tag: Optional[str] = None - ) -> list[dict]: - """Fetch the incremental PR diff, with per-payload PR fallback.""" + def fetch_changelog(self, tag_name: str, from_tag: Optional[str] = None) -> list[dict]: + params = f"toPayload={urllib.parse.quote(tag_name)}" + if not from_tag: + tag_meta = self.find_tag(tag_name) + if tag_meta: + from_tag = tag_meta.get("previous_release_tag", "") if from_tag: - url = ( - f"{self.SIPPY_BASE}/payloads/diff" - f"?toPayload={urllib.parse.quote(tag_name)}" - f"&fromPayload={urllib.parse.quote(from_tag)}" - ) - try: - return fetch_json(url, timeout=60) - except FALLBACK_FETCH_ERRORS as exc: - _log( - " Sippy incremental payload diff unavailable; " - f"using per-payload PR data: {exc}" - ) - return self.fetch_pull_requests(tag_name) - - def release_url(self, tag_name: str) -> str: - return ( - f"{self.SIPPY_UI}/release/{urllib.parse.quote(self.release)}" - f"/tags/{urllib.parse.quote(tag_name)}" - ) + params += f"&fromPayload={urllib.parse.quote(from_tag)}" + url = f"{self.SIPPY_BASE}/payloads/diff?{params}" + try: + return fetch_json(url, timeout=60) + except Exception: + return [] def build_synthetic_payload(self, tag_name: str, tag: "PayloadTag") -> dict: tag_meta = self.find_tag(tag_name) @@ -379,19 +314,18 @@ def build_synthetic_payload(self, tag_name: str, tag: "PayloadTag") -> dict: else: informing_jobs[name] = entry + rc = ReleaseController(tag.architecture, stream=tag.stream) return { "phase": phase, "results": { "blockingJobs": blocking_jobs, "informingJobs": informing_jobs, }, - "_release_url": self.release_url(tag_name), + "_release_url": rc.release_url(tag.stream_name, tag_name), "_source": "sippy", } - def build_synthetic_changelog( - self, tag_name: str, from_tag: Optional[str] = None - ) -> dict: + def build_synthetic_changelog(self, tag_name: str, from_tag: Optional[str] = None) -> dict: prs = self.fetch_changelog(tag_name, from_tag=from_tag) if not isinstance(prs, list): return {"changeLogJson": {"updatedImages": []}, "_source": "sippy"} @@ -440,10 +374,10 @@ def build(self, start_tag: str) -> list[str]: try: start_idx = tag_names.index(start_tag) - except ValueError as exc: + except ValueError: raise ValueError( f"Tag {start_tag} not found in stream {self.stream_name}" - ) from exc + ) chain = [] for i in range(start_idx, min(start_idx + self.max_depth, len(tag_names))): @@ -475,148 +409,41 @@ def _all_blocking_passed(self, details: dict) -> bool: class SippyPayloadChain: - """Walks backwards through Sippy's time-ordered payload tags.""" + """Walks backwards through payloads using Sippy tag data.""" def __init__(self, sippy: SippyClient, max_depth: int = 20): self.sippy = sippy self.max_depth = max_depth - def _all_blocking_passed(self, tag_name: str) -> bool: - """Check whether every blocking job in a payload succeeded.""" + def _has_blocking_failures(self, tag_name: str) -> bool: + """Check whether a payload has any failed blocking jobs.""" runs = self.sippy.fetch_job_runs(tag_name) - blocking = [r for r in runs if r.get("kind") == "Blocking"] - if not blocking: - tag_meta = self.sippy.find_tag(tag_name) - return bool( - tag_meta - and tag_meta.get("phase") == "Accepted" - and not tag_meta.get("forced", False) - and tag_meta.get("failed_job_names") == [] - ) - return all( - r.get("state") == "Succeeded" for r in blocking + return any( + r.get("kind") == "Blocking" and r.get("state") == "Failed" + for r in runs ) def build(self, start_tag: str) -> list[str]: - tag_names = [ - tag["release_tag"] for tag in self.sippy.fetch_tags() - if tag.get("release_tag") - ] - try: - start_index = tag_names.index(start_tag) - except ValueError as exc: - raise ValueError(f"Tag {start_tag} not found in Sippy") from exc - - chain = [] - for tag_name in tag_names[ - start_index:start_index + self.max_depth - ]: - chain.append(tag_name) - if self._all_blocking_passed(tag_name): - break - return chain - - -class HybridPayloadChain: - """Build a complete payload chain from RC data plus Sippy history. - - Release-controller details remain authoritative for retained tags. Sippy's - time-ordered tag list identifies payloads that were garbage collected - between retained release-controller entries. - """ - - def __init__( - self, - rc: ReleaseController, - sippy: SippyClient, - stream_name: str, - max_depth: int = 20, - ): - self.rc = rc - self.sippy = sippy - self.stream_name = stream_name - self.max_depth = max_depth - self.sources: dict[str, str] = {} - self._rc_tags: list[str] = [] - self._sippy_tags: list[str] = [] - self._sippy_available = True - self._warned_sippy = False - - def _load_sippy_tags(self) -> list[str]: - if not self._sippy_available: - return [] - try: - self._sippy_tags = [ - tag["release_tag"] for tag in self.sippy.fetch_tags() - if tag.get("release_tag") - ] - return self._sippy_tags - except FALLBACK_FETCH_ERRORS as exc: - self._sippy_available = False - if not self._warned_sippy: - _log( - "Warning: Sippy release ancestry is unavailable; " - f"falling back to release-controller ordering: {exc}" - ) - self._warned_sippy = True - return [] - - @staticmethod - def _previous(tag_names: list[str], tag_name: str) -> str: - try: - index = tag_names.index(tag_name) - except ValueError: - return "" - if index + 1 >= len(tag_names): - return "" - return tag_names[index + 1] - - def _fetch_details(self, tag_name: str, rc_tag_names: set[str]) -> dict: - if tag_name in rc_tag_names: - self.sources[tag_name] = "release-controller" - return self.rc.fetch_release(self.stream_name, tag_name) - - tag = PayloadTag.parse(tag_name) - self.sources[tag_name] = "sippy" - return self.sippy.build_synthetic_payload(tag_name, tag) - - def build(self, start_tag: str) -> list[str]: - rc_tags = self.rc.fetch_tags(self.stream_name) - self._rc_tags = [t["name"] for t in rc_tags] - rc_tag_names = set(self._rc_tags) - sippy_tag_names = set(self._load_sippy_tags()) - - if start_tag not in rc_tag_names and start_tag not in sippy_tag_names: - raise ValueError( - f"Tag {start_tag} not found in release controller or Sippy" - ) - - chain: list[str] = [] + chain = [start_tag] current = start_tag - for _ in range(self.max_depth): - if current in chain: - _log(f"Warning: payload ancestry cycle detected at {current}") + for _ in range(self.max_depth - 1): + tag_meta = self.sippy.find_tag(current) + if not tag_meta: break - chain.append(current) - - details = self._fetch_details(current, rc_tag_names) - rc_chain = PayloadChain(self.rc, self.stream_name) - if rc_chain._all_blocking_passed(details): + prev = tag_meta.get("previous_release_tag", "") + if not prev: break - - sippy_previous = self._previous(self._sippy_tags, current) - rc_previous = self._previous(self._rc_tags, current) - previous = sippy_previous or rc_previous - if not previous: + chain.append(prev) + if not self._has_blocking_failures(prev): + # First payload with all blocking jobs green — include + # one more predecessor so this payload gets a changelog. + prev_meta = self.sippy.find_tag(prev) + if prev_meta: + anchor = prev_meta.get("previous_release_tag", "") + if anchor: + chain.append(anchor) break - - if sippy_previous and sippy_previous not in rc_tag_names: - _log( - " Sippy history restored garbage-collected payload " - f"{sippy_previous} before {current}" - ) - current = previous - + current = prev return chain @@ -706,7 +533,6 @@ def _fetch(self) -> dict: _log(f" Fetching payload details: {self.tag}") details = self.rc.fetch_release(self.stream_name, self.tag) details["_release_url"] = self.rc.release_url(self.stream_name, self.tag) - details["_source"] = "release-controller" return details @@ -723,11 +549,7 @@ def __init__(self, output_path: str, rc: ReleaseController, def _fetch(self) -> dict: _log(f" Fetching changelog: {self.tag} from {self.from_tag}") - changelog = self.rc.fetch_changelog( - self.stream_name, self.tag, self.from_tag - ) - changelog["_source"] = "release-controller" - return changelog + return self.rc.fetch_changelog(self.stream_name, self.tag, self.from_tag) class PullRequestCollector(Collector): @@ -894,48 +716,7 @@ def collect(self) -> bool: os.makedirs(self.output_dir, exist_ok=True) all_results: list[_TestResult] = [] - with _error_scope() as own_errors: - junit_files = self._list_junit_files() - downloaded, failed_downloads = self._download_and_parse( - junit_files, all_results - ) - gcloud_failed = any( - not e.get("recovered") for e in own_errors - ) - - # Nothing was discovered at all. These collectors only run for - # jobs that FAILED, so "no JUnit anywhere" is not evidence of a - # clean run — it is an unknown. Publishing [] here would report a - # verified zero for a job whose tests may never have run. - if not junit_files: - _record_collection_error( - "junit_missing", - ["gcloud", "storage", "ls", ""], - detail=( - "no JUnit artifacts discovered for a failed job; " - "test results are unknown, not zero" - ), - stage="junit", - job=self.job.name, - payload_tag=self.payload_tag, - ) - _log( - f" ERROR: no JUnit discovered for {self.job.name} — " - f"leaving results.json absent (unknown, not zero)" - ) - return False - - return self._publish( - junit_files, all_results, downloaded, failed_downloads, - gcloud_failed, - ) - - def _download_and_parse( - self, junit_files: list[str], all_results: list - ) -> tuple[int, int]: - """Fetch each JUnit file and parse it; count successes/failures.""" - downloaded = 0 - failed_downloads = 0 + junit_files = self._list_junit_files() for gcs_uri in junit_files: filename = os.path.basename(gcs_uri) local_path = os.path.join(self.output_dir, filename) @@ -949,72 +730,7 @@ def _download_and_parse( if os.path.exists(local_path): results = _parse_junit_xml(local_path, source_name=filename) - if results is None: - # Downloaded but unparseable. Treated as unread, not as - # "no failures" — corrupt XML must never become a zero. - failed_downloads += 1 - _record_collection_error( - "junit_unparseable", - ["parse", filename], - detail=f"{filename} is not valid JUnit XML", - stage="junit", - job=self.job.name, - payload_tag=self.payload_tag, - ) - continue - downloaded += 1 all_results.extend(results) - else: - failed_downloads += 1 - return downloaded, failed_downloads - - def _publish( - self, junit_files: list[str], all_results: list, - downloaded: int, failed_downloads: int, gcloud_failed: bool, - ) -> bool: - """Write results.json unless doing so would misreport the data.""" - # Refuse to publish an empty result set that was caused by a read - # failure. Writing `[]` here makes the job look clean, and - # downstream consumers report "0 test failures" for a job that - # actually failed. Leaving results.json absent means "unknown", - # which the summary and the analysis skill treat as such. - if gcloud_failed and downloaded == 0: - _record_collection_error( - "junit_unavailable", - ["gcloud", "storage", "cat", ""], - detail=( - "no JUnit XML could be read; results.json intentionally " - "not written so this is not mistaken for zero failures" - ), - stage="junit", - job=self.job.name, - payload_tag=self.payload_tag, - ) - _log( - f" ERROR: could not read JUnit for {self.job.name} — " - f"leaving results.json absent (unknown, not zero)" - ) - return False - - # Some files read, others not. The results are real but partial, so - # publish them and mark them partial — a count derived from half the - # JUnit is not authoritative and must not read as one that is. - if failed_downloads: - _record_collection_error( - "junit_partial", - ["gcloud", "storage", "cat", ""], - detail=( - f"{failed_downloads} of {len(junit_files)} JUnit file(s) " - f"could not be read; test_failure_count is a lower bound" - ), - stage="junit", - job=self.job.name, - payload_tag=self.payload_tag, - ) - _log( - f" WARNING: partial JUnit for {self.job.name} — " - f"{failed_downloads}/{len(junit_files)} file(s) unread" - ) failures = _test_results_to_json(all_results) _write_json(self.output_path, failures) @@ -1032,41 +748,19 @@ def _publish( return True def _list_junit_files(self) -> list[str]: - """Discover JUnit XML files in GCS for this job. - - Tries a recursive glob first, then falls back to bounded-depth - probes. Jobs that emit very large artifact trees (Hypershift e2e - can produce 10,000+ cluster resource dumps) make the ``**`` glob - slow enough to time out, and a timeout here previously produced a - silently empty result set. - """ + """Discover JUnit XML files in GCS for this job.""" bucket_path = self.job.gcs_bucket_path base = f"gs://{bucket_path}" - with _error_scope() as glob_errors: - output = _run_gcloud( - ["gcloud", "storage", "ls", - f"{base}/artifacts/**/junit*.xml"], - timeout=120, - ) - - files = [line.strip() for line in (output or "").strip().splitlines() - if line.strip()] - - if not files: - _log( - f" warn: recursive glob found no JUnit for {self.job.name} " - f"— trying bounded-depth fallback" - ) - files = self._list_junit_files_fallback(base) - if files: - # The glob's failure is recovered; any probe failures the - # fallback itself hit are NOT — they may be missing data. - _mark_errors_recovered(glob_errors) - - if not files: + output = _run_gcloud( + ["gcloud", "storage", "ls", f"{base}/artifacts/**/junit*.xml"], + timeout=30, + ) + if not output: return [] + files = [l.strip() for l in output.strip().splitlines() if l.strip()] + if self.job.is_aggregated: # For aggregated jobs, keep junit_operator.xml and the # junit-aggregated.xml. Filter out other junit files that are @@ -1089,71 +783,6 @@ def _list_junit_files(self) -> list[str]: keep = operator + (preferred if preferred else others[:1]) return keep if keep else files - def _list_junit_files_fallback(self, base: str) -> list[str]: - """Targeted JUnit discovery that avoids a full recursive glob. - - Lists the top-level step directories under ``artifacts/`` and - probes each at the depths JUnit files actually appear, rather - than enumerating every object in the tree: - - - ``{step}/junit*.xml`` - - ``{step}/artifacts/junit*.xml`` - - ``{step}/*/artifacts/junit*.xml`` - - Aggregated jobs keep ``junit-aggregated.xml`` far deeper than - that, so the aggregator subtree gets its own scoped recursive - probe — scoped to one small directory, it stays fast. - """ - dir_output = _run_gcloud( - ["gcloud", "storage", "ls", f"{base}/artifacts/"], - timeout=60, - ) - if not dir_output: - return [] - - # Each line is a directory like gs://bucket/path/artifacts/step-name/ - step_dirs = [ - d.strip().rstrip("/").split("/")[-1] - for d in dir_output.strip().splitlines() - if d.strip() - ] - - # junit_operator.xml sits directly in artifacts/, not in a step dir. - found: list[str] = [] - top = _run_gcloud( - ["gcloud", "storage", "ls", f"{base}/artifacts/junit*.xml"], - timeout=60, - ) - if top: - found.extend(line.strip() for line in top.strip().splitlines() - if line.strip()) - - for step in step_dirs: - patterns = [ - f"{base}/artifacts/{step}/junit*.xml", - f"{base}/artifacts/{step}/artifacts/junit*.xml", - f"{base}/artifacts/{step}/*/artifacts/junit*.xml", - ] - # The aggregated report lives several levels below the - # aggregator step; a scoped ** over that one subtree is cheap. - if "aggregator" in step: - patterns.append(f"{base}/artifacts/{step}/**/junit*.xml") - - for pattern in patterns: - probe = _run_gcloud( - ["gcloud", "storage", "ls", pattern], timeout=60 - ) - if probe: - found.extend( - line.strip() for line in probe.strip().splitlines() - if line.strip() - ) - - found = sorted(set(found)) - if found: - _log(f" fallback found {len(found)} JUnit file(s)") - return found - def _detect_underlying_job_name(self) -> Optional[str]: """For aggregated jobs, extract the underlying job name from GCS paths.""" if not self.job.is_aggregated: @@ -1197,19 +826,10 @@ def _fetch(self) -> Optional[dict]: return None gcs_uri = f"gs://{self.job.gcs_bucket_path}/build-log.txt" - with _error_scope() as own_errors: - raw = _run_gcloud_bytes( - ["gcloud", "storage", "cat", gcs_uri], timeout=120 - ) + raw = _run_gcloud_bytes( + ["gcloud", "storage", "cat", gcs_uri], timeout=120 + ) if not raw: - if any(not e.get("recovered") for e in own_errors): - _record_collection_error( - "build_log_unavailable", - ["gcloud", "storage", "cat", gcs_uri], - detail="build-log.txt could not be read", - stage="build_log", - job=self.job.name, - ) return None try: @@ -1406,11 +1026,6 @@ def __init__(self, base_dir: str, chain: list[str], target_tag: str): self.base_dir = base_dir self.chain = chain self.target_tag = target_tag - self._skipped = {"flake": 0, "informing": 0} - - def non_gating_counts(self) -> dict[str, int]: - """Counts of target-payload results excluded from onset tracking.""" - return dict(self._skipped) def track(self) -> list[dict]: """Analyze failures in the target payload and trace their origins.""" @@ -1466,17 +1081,6 @@ def _load_all_failures(self, tag_dir: str) -> dict[str, dict]: name = test.get("name", "") if not name: continue - # Only gating results establish a regression onset. A - # flake or an informing test cannot reject a payload, so - # letting one set first_failed_in sends the analysis - # hunting for a culprit that never existed. - if not _is_gating(test): - if tag_dir.endswith(self.target_tag): - if test.get("status") == "flake": - self._skipped["flake"] += 1 - else: - self._skipped["informing"] += 1 - continue if name in failures: if job_name not in failures[name]["jobs"]: failures[name]["jobs"].append(job_name) @@ -1600,8 +1204,6 @@ def generate(self) -> None: phase = payload_data.get("phase", "Unknown") if payload_data else "Unknown" release_url = (payload_data.get("_release_url", "") if payload_data else "") - source = (payload_data.get("_source", "release-controller") - if payload_data else "unknown") results = payload_data.get("results", {}) if payload_data else {} blocking = results.get("blockingJobs", {}) or {} @@ -1629,7 +1231,6 @@ def generate(self) -> None: "payload_tag": self.target_tag, "phase": phase, "release_url": release_url, - "source": source, "architecture": self.tag.architecture, "stream": self.tag.stream, "version": self.tag.version, @@ -1649,15 +1250,9 @@ def generate(self) -> None: "failed_jobs": failed_informing_names, }, "test_failures": { - # Gating failures only — these are what can fail a job and - # therefore reject the payload. "blocking": [ r for r in regressions if r.get("lifecycle") == "blocking" ], - # Neither of the following can fail a job or reject a - # payload. They are surfaced for visibility, not for blame. - "informing": self._collect_non_gating("informing"), - "flakes": self._collect_non_gating("flake"), }, "payloads": payloads, } @@ -1669,14 +1264,7 @@ def generate(self) -> None: for entry in target_rpms ] - # Surface any data that could not be collected. An empty snapshot - # section must never be silently indistinguishable from a clean one. - if _COLLECTION_ERRORS: - summary["collection_errors"] = list(_COLLECTION_ERRORS) - summary["data_complete"] = not _unrecovered_errors() - _write_json(os.path.join(self.base_dir, "summary.json"), summary) - _write_collection_state(self.base_dir) self._write_agents_md(summary) _log(" Generated summary.json, AGENTS.md, and CLAUDE.md") @@ -1728,25 +1316,6 @@ def _write_agents_md(self, summary: dict) -> None: "", ] - if not summary.get("data_complete", True): - errs = [e for e in summary.get("collection_errors", []) - if not e.get("recovered")] - reasons = sorted({e.get("reason", "unknown") for e in errs}) - lines.extend([ - "## ⚠️ INCOMPLETE SNAPSHOT", - "", - f"{len(errs)} collection error(s) occurred: " - f"{', '.join(reasons)}.", - "See `collection_errors` in `summary.json`.", - "", - "Data that could not be read is **absent, not empty**. A job", - "with `junit_collection_failed: true` and no", - "`test_failure_count` has an *unknown* number of test", - "failures — do NOT report it as zero, and do not conclude a", - "job failed for reasons other than its tests on this basis.", - "", - ]) - if failed_names: lines.append("### Failed blocking jobs") lines.append("") @@ -1798,8 +1367,6 @@ def _write_agents_md(self, summary: dict) -> None: " payload acceptance. Informing jobs are tracked but don't block.", "- **Chain**: The sequence of payloads walking backwards from the", " target until one where all blocking jobs passed (the baseline).", - " Sippy's time-ordered tag list restores garbage-collected tags;", - " retained payload data comes from the release controller.", "- **Streaks**: Per-job consecutive failure count from the target", " backwards. `failure_pattern` shows the full history (F=fail,", " S=succeed) across the chain.", @@ -1812,8 +1379,8 @@ def _write_agents_md(self, summary: dict) -> None: "## summary.json Schema", "", "Top-level fields:", - "- `payload_tag`, `phase`, `release_url`, `source`,", - " `architecture`, `stream`, `version`", + "- `payload_tag`, `phase`, `release_url`, `architecture`,", + " `stream`, `version`", "- `chain_length`, `baseline_tag`, `hours_since_baseline`", "- `blocking_jobs.failed_jobs[]` — each entry has: `name`,", " `state`, `prow_url`, `gcs_url`, `streak` (with", @@ -1825,8 +1392,7 @@ def _write_agents_md(self, summary: dict) -> None: " `first_failed_in`, `payloads_failing`, `failure_message`,", " `failure_text`", "- `payloads[]` — per-payload entries with `tag`, `phase`,", - " `source`, `changelog_source`, relative paths, `prs[]` with", - " component/diff/comments paths,", + " relative paths, `prs[]` with component/diff/comments paths,", " `rhcos_changes[]` with RPM diffs per RHCOS variant, and", " `rhcos_rpms[]` with rpmdb.sqlite per variant", "- `rhcos_rpms[]` — RPMDB per RHCOS variant for the target", @@ -1902,27 +1468,7 @@ def _build_failed_job_details(self, lifecycle: str) -> list[dict]: f"{tag_rel}/jobs/{lifecycle}/{job_name}/junit/results.json" ) results_data = _read_json(results_path) or [] - # Counts only results that could fail the job. Flakes and - # informing tests are listed by name under `test_failures` - # but are deliberately not counted here — this number is the - # failure count that drives triage. - entry["test_failure_count"] = sum( - 1 for t in results_data if _is_gating(t) - ) - if _job_junit_state( - job_name, self.target_tag, "junit_partial" - ): - # Real results, but not all of them: the count is a - # lower bound, not an authoritative total. - entry["junit_collection_partial"] = True - elif any( - _job_junit_state(job_name, self.target_tag, r) - for r in ("junit_unavailable", "junit_missing", - "junit_unparseable") - ): - # Data could not be read. Do NOT emit test_failure_count — - # its absence means "unknown", where 0 would mean "clean". - entry["junit_collection_failed"] = True + entry["test_failure_count"] = len(results_data) build_log_path = os.path.join( lifecycle_dir, job_name, "build_log.json" @@ -1940,45 +1486,6 @@ def _build_failed_job_details(self, lifecycle: str) -> list[dict]: details.append(entry) return details - def _collect_non_gating(self, kind: str) -> list[dict]: - """Collect the target payload's non-gating test results. - - ``kind`` is "flake" (failed then passed) or "informing" (opted out - of gating via lifecycle). Neither can fail a job, so no regression - onset is computed for them — an onset implies a culprit to find. - """ - found: dict[str, dict] = {} - lifecycle_dir = os.path.join( - self.base_dir, self.target_tag, "jobs", "blocking" - ) - if not os.path.isdir(lifecycle_dir): - return [] - for job_name in sorted(os.listdir(lifecycle_dir)): - results = _read_json(os.path.join( - lifecycle_dir, job_name, "junit", "results.json" - )) - if not results: - continue - for test in results: - name = test.get("name", "") - if not name: - continue - if kind == "flake": - match = test.get("status") == "flake" - else: - match = ( - test.get("status") in ("failed", "error") - and test.get("test_lifecycle") == "informing" - ) - if not match: - continue - if name in found: - if job_name not in found[name]["jobs"]: - found[name]["jobs"].append(job_name) - else: - found[name] = {"test_name": name, "jobs": [job_name]} - return sorted(found.values(), key=lambda e: e["test_name"]) - def _build_payload_entries(self) -> list[dict]: """Build the payloads array with all path references.""" payloads = [] @@ -1994,7 +1501,6 @@ def _build_payload_entries(self) -> list[dict]: ) if pd: entry["phase"] = pd.get("phase", "") - entry["source"] = pd.get("_source", "release-controller") changelog_path = os.path.join( self.base_dir, tag_name, "changelog.json" @@ -2002,10 +1508,6 @@ def _build_payload_entries(self) -> list[dict]: if os.path.exists(changelog_path): entry["changelog"] = f"{tag_rel}/changelog.json" changelog = _read_json(changelog_path) - if changelog: - entry["changelog_source"] = changelog.get( - "_source", "release-controller" - ) prs = _extract_prs(changelog) if changelog else [] if prs: entry["prs"] = [ @@ -2108,10 +1610,9 @@ def __init__(self, tag: PayloadTag, output_dir: str = "payload", self.use_sippy = use_sippy self.collect_rpmdb = collect_rpmdb self.rc = ReleaseController(tag.architecture, stream=tag.stream) - self.sippy = SippyClient( - tag.version, architecture=tag.architecture, stream=tag.stream - ) - self._payload_sources: dict[str, str] = {} + self.sippy: Optional[SippyClient] = None + if use_sippy: + self.sippy = SippyClient(tag.version) def run(self) -> None: """Execute the full snapshot.""" @@ -2120,16 +1621,6 @@ def run(self) -> None: ) os.makedirs(base_dir, exist_ok=True) - # A previous run over this directory may have left JUnit output it - # recorded as incomplete. Drop it so it is re-collected rather than - # inherited as trustworthy — but only when we will actually - # re-collect, otherwise we would delete the only copy and then - # report the result as complete. - if self.collect_junit: - _invalidate_suspect_junit(base_dir) - else: - _carry_forward_junit_errors(base_dir) - self._collect_streams(base_dir) chain = self._build_chain() @@ -2160,7 +1651,7 @@ def run(self) -> None: def _collect_streams(self, base_dir: str) -> None: """Collect the streams list for this version.""" path = os.path.join(base_dir, "streams.json") - if self.use_sippy: + if self.sippy: if os.path.exists(path): return _log("Skipping streams collection in Sippy mode (RC-only feature)") @@ -2170,22 +1661,13 @@ def _collect_streams(self, base_dir: str) -> None: def _build_chain(self) -> list[str]: """Build the backward payload chain.""" - if self.use_sippy: + if self.sippy: chain_builder = SippyPayloadChain(self.sippy, self.max_chain) - chain = chain_builder.build(self.tag.raw) - self._payload_sources = {tag: "sippy" for tag in chain} - return chain - chain_builder = HybridPayloadChain( - self.rc, self.sippy, self.tag.stream_name, self.max_chain + return chain_builder.build(self.tag.raw) + chain_builder = PayloadChain( + self.rc, self.tag.stream_name, self.max_chain ) - chain = chain_builder.build(self.tag.raw) - self._payload_sources = chain_builder.sources - return chain - - def _payload_source(self, tag_name: str) -> str: - if self.use_sippy: - return "sippy" - return self._payload_sources.get(tag_name, "release-controller") + return chain_builder.build(self.tag.raw) def _collect_payloads( self, base_dir: str, chain: list[str] @@ -2199,8 +1681,7 @@ def _collect_payloads( _log(f"\nProcessing payload: {tag_name}") payload_path = os.path.join(tag_dir, "payload.json") - payload_source = self._payload_source(tag_name) - if payload_source == "sippy": + if self.sippy: if not os.path.exists(payload_path): _log(f" Fetching payload details from Sippy: {tag_name}") tag_obj = PayloadTag.parse(tag_name) @@ -2220,21 +1701,10 @@ def _collect_payloads( prev_tag = chain[i + 1] changelog_path = os.path.join(tag_dir, "changelog.json") - # The release controller cannot diff across a tag it has already - # garbage collected. Use Sippy's incremental diff (with its - # per-payload PR fallback) whenever either side is unavailable. - changelog_source = ( - "release-controller" - if payload_source == "release-controller" - and self._payload_source(prev_tag) == "release-controller" - else "sippy" - ) - if changelog_source == "sippy": + if self.sippy: if not os.path.exists(changelog_path): _log(f" Fetching changelog from Sippy: {tag_name}") - data = self.sippy.build_synthetic_changelog( - tag_name, from_tag=prev_tag - ) + data = self.sippy.build_synthetic_changelog(tag_name, from_tag=prev_tag) os.makedirs(os.path.dirname(changelog_path), exist_ok=True) _write_json(changelog_path, data) else: @@ -2414,14 +1884,8 @@ def _track_regressions(self, base_dir: str, chain: list[str]) -> None: new_count = sum(1 for r in regressions if r["payloads_failing"] == 1) persistent_count = len(regressions) - new_count - _log(f" Found {len(regressions)} gating test failures: " + _log(f" Found {len(regressions)} failing tests: " f"{new_count} new, {persistent_count} persistent") - skipped = tracker.non_gating_counts() - if skipped["flake"] or skipped["informing"]: - _log(f" Excluded from regression tracking: " - f"{skipped['flake']} flake(s), " - f"{skipped['informing']} informing failure(s) — neither can " - f"fail a job") def _collect_build_logs(self, base_dir: str, chain: list[str]) -> None: """Download and parse build-log.txt for failed jobs. @@ -2698,367 +2162,29 @@ def _check_gh_auth() -> bool: return False -# Records every gcloud failure that could hide real data. A "no match" -# result is normal (the object simply does not exist) and is NOT recorded. -# Anything else — a missing binary, a timeout, an auth failure — means the -# snapshot could not read data that may well exist, and callers must be able -# to tell that apart from "there was nothing to find". -_COLLECTION_ERRORS: list[dict] = [] -_COLLECTION_ERRORS_LOCK = threading.Lock() - -# Per-thread stack of "operation scopes". Every recorded error is appended -# to each active scope as well as the global ledger, so a caller can act on -# exactly the errors *its own* calls produced. Positional indexes into the -# global list are not safe here: collectors run concurrently, so another -# worker can append between one collector's start and end offsets. -_ERROR_SCOPES = threading.local() - -# stderr fragments that mean "the object does not exist", not "I failed". -# Keep this narrow: only the specific gcloud "URLs matched no objects" -# outcome is benign. Broad tokens like "not found" would swallow 404s -# and bucket-not-found errors that should be recorded. -_GCLOUD_NO_MATCH_PATTERNS = ( - "matched no objects", - "matched no such objects", -) - -# stderr fragments that indicate a credentials/permission problem. -_GCLOUD_AUTH_PATTERNS = ( - "does not have storage.objects", - "anonymous caller", - "reauthentication", - "credentials", - "unauthorized", - "forbidden", - "403", - "401", -) - - -def _error_scope(): - """Collect errors recorded by this thread inside the ``with`` body.""" - return _ErrorScope() - - -class _ErrorScope: - """Context manager yielding the list of errors recorded inside it.""" - - def __init__(self) -> None: - self.entries: list[dict] = [] - - def __enter__(self) -> list[dict]: - stack = getattr(_ERROR_SCOPES, "stack", None) - if stack is None: - stack = [] - _ERROR_SCOPES.stack = stack - stack.append(self.entries) - return self.entries - - def __exit__(self, *exc) -> None: - getattr(_ERROR_SCOPES, "stack", []).pop() - return None - - -def _sanitize_detail(text: str) -> str: - """Make tool stderr safe to persist in a shareable summary. - - Strips control characters (which can spoof terminal output when the - summary is read back) and collapses whitespace. Diagnostics are - truncated: they exist to identify a failure class, not to carry full - tool output. - """ - cleaned = "".join( - ch if ch.isprintable() else " " for ch in (text or "") - ) - # These diagnostics are persisted into a summary this skill exists to - # hand to agents and share. gcloud auth errors routinely name the - # active account, and URLs can carry signed-request parameters; neither - # should leave the local environment. - cleaned = re.sub( - r"[\w.+-]+@[\w-]+\.[\w.-]+", "", cleaned - ) - cleaned = re.sub(r"\?\S+", "?", cleaned) - return re.sub(r"\s+", " ", cleaned).strip()[:300] - - -def _classify_gcloud_stderr(stderr: str) -> str: - """Classify a failed gcloud invocation from its stderr.""" - low = (stderr or "").lower() - if any(p in low for p in _GCLOUD_AUTH_PATTERNS): - return "auth" - if any(p in low for p in _GCLOUD_NO_MATCH_PATTERNS): - return "no_match" - return "command_failed" - - -def _record_collection_error( - reason: str, - command: list[str], - detail: str = "", - stage: str = "", - job: str = "", - payload_tag: str = "", -) -> None: - """Record a data-collection failure so it can be surfaced, not swallowed. - - Silently returning empty data makes "I could not read this" look - identical to "there is nothing here", which invites downstream - consumers to report zero failures for a job that actually failed. - """ - entry: dict = {"reason": reason, "command": " ".join(command[:4])} - if detail: - entry["detail"] = _sanitize_detail(detail) - if stage: - entry["stage"] = stage - if job: - entry["job"] = job - if payload_tag: - entry["payload_tag"] = payload_tag - with _COLLECTION_ERRORS_LOCK: - _COLLECTION_ERRORS.append(entry) - for scope in getattr(_ERROR_SCOPES, "stack", []): - scope.append(entry) - - -def _collection_error_count() -> int: - """Number of collection errors recorded so far.""" - return len(_COLLECTION_ERRORS) - - -COLLECTION_STATE_FILE = "collection_errors.json" - -# Reasons that make a job's persisted JUnit output untrustworthy. -_JUNIT_SUSPECT_REASONS = ( - "junit_unavailable", "junit_partial", "junit_missing", - "junit_unparseable", -) - - -def _is_within(base: str, target: str) -> bool: - """True when ``target`` resolves inside ``base``.""" - base_r = os.path.realpath(base) - target_r = os.path.realpath(target) - return target_r == base_r or target_r.startswith(base_r + os.sep) - - -def _safe_component(value: str) -> bool: - """True when ``value`` is usable as a single path component.""" - return bool(value) and value not in (".", "..") and not any( - sep in value for sep in (os.sep, "/", "\\") - ) - - -def _invalidate_suspect_junit(base_dir: str) -> int: - """Discard JUnit output that a previous run recorded as incomplete. - - Collection errors live in process memory but ``results.json`` persists, - and collectors skip any job whose output already exists. Without this, - re-running over the same directory regenerates the summary from an empty - ledger and promotes a partial snapshot to "complete". Dropping the - suspect output forces it to be re-collected and re-judged. - """ - state_path = os.path.join(base_dir, COLLECTION_STATE_FILE) - previous = _read_json(state_path) - if not previous: - return 0 - - invalidated = 0 - invalidated_tags: set[str] = set() - for entry in previous: - if entry.get("recovered"): - continue - if entry.get("reason") not in _JUNIT_SUSPECT_REASONS: - continue - tag, job = entry.get("payload_tag"), entry.get("job") - # The ledger is data read back from disk; never let it aim a - # recursive delete at a path outside the snapshot directory. - if not _safe_component(tag or "") or not _safe_component(job or ""): - _log(f" warn: ignoring unsafe collection-state entry " - f"(tag={tag!r}, job={job!r})") - continue - for lifecycle in ("blocking", "informing"): - junit_dir = os.path.join( - base_dir, tag, "jobs", lifecycle, job, "junit" - ) - if not _is_within(base_dir, junit_dir): - continue - if os.path.isdir(junit_dir): - shutil.rmtree(junit_dir, ignore_errors=True) - invalidated += 1 - invalidated_tags.add(tag) - for tag in invalidated_tags: - reg_path = os.path.join(base_dir, tag, "regressions.json") - if _is_within(base_dir, reg_path) and os.path.exists(reg_path): - os.remove(reg_path) - _log(f" removed stale regressions.json for {tag}") - if invalidated: - _log(f" re-collecting {invalidated} job(s) whose previous JUnit " - f"collection was incomplete") - return invalidated - - -def _carry_forward_junit_errors(base_dir: str) -> int: - """Re-record persisted JUnit failures when JUnit is not re-collected. - - With ``--no-junit`` nothing re-reads the artifacts, so a previous run's - partial or unreadable JUnit is still exactly as incomplete as it was. - Without carrying those errors forward, the ledger would be rewritten - from an empty in-memory list and the stale output would be reported as - a complete snapshot. - """ - previous = _read_json( - os.path.join(base_dir, COLLECTION_STATE_FILE) - ) or [] - carried = 0 - for entry in previous: - if entry.get("recovered"): - continue - if entry.get("reason") not in _JUNIT_SUSPECT_REASONS: - continue - forwarded = dict(entry) - forwarded["carried_forward"] = True - with _COLLECTION_ERRORS_LOCK: - _COLLECTION_ERRORS.append(forwarded) - carried += 1 - if carried: - _log(f" carrying forward {carried} unresolved JUnit collection " - f"error(s) from a previous run (JUnit collection is disabled)") - return carried - - -def _write_collection_state(base_dir: str) -> None: - """Persist the error ledger so a later process can see it.""" - state_path = os.path.join(base_dir, COLLECTION_STATE_FILE) - if _COLLECTION_ERRORS: - _write_json(state_path, list(_COLLECTION_ERRORS)) - elif os.path.exists(state_path): - # Everything was collected this time; drop the stale ledger. - os.remove(state_path) - - -def _mark_errors_recovered(entries: list[dict]) -> None: - """Mark the given error entries as recovered. - - A first-choice read can fail (e.g. a glob times out) and a fallback - can then succeed. The failure is still worth reporting — it is how - you learn a timeout needs tuning — but it must not make the snapshot - look incomplete when the data was in fact obtained. - - Entries are identified by object, not by position: only the specific - failures handed in are recovered, never a concurrent collector's. - """ - for entry in entries: - entry["recovered"] = True - - -def _unrecovered_errors() -> list[dict]: - """Collection errors that were not resolved by a fallback.""" - return [e for e in _COLLECTION_ERRORS if not e.get("recovered")] - - -def _unrecovered_error_count() -> int: - """Number of unrecovered collection errors.""" - return len(_unrecovered_errors()) - - -def _job_junit_state(job_name: str, payload_tag: str, reason: str) -> bool: - """True when this payload's collection of this job hit ``reason``. - - Scoped by payload tag: the same job name recurs in every payload of - the chain, so an unscoped match would stamp one payload's job entry - with another payload's failure. - """ - return any( - e.get("job") == job_name - and e.get("payload_tag") == payload_tag - and e.get("reason") == reason - and not e.get("recovered") - for e in _COLLECTION_ERRORS - ) - - -# Whether gcloud must be run in anonymous mode. Resolved once, on first -# use, under a lock — collectors run in a thread pool. -_GCLOUD_ANONYMOUS: Optional[bool] = None -_GCLOUD_ANONYMOUS_LOCK = threading.Lock() - - -def _gcloud_env() -> dict: - """Environment for gcloud calls, enabling anonymous access if needed. - - The CI artifact buckets are public, but ``gcloud storage`` refuses - client-side when no account is configured ("You do not currently have - an active account selected") — it never even issues the request. - Setting ``CLOUDSDK_AUTH_DISABLE_CREDENTIALS`` makes it read public - objects anonymously, so an unauthenticated environment still gets a - complete snapshot instead of an empty one. - """ - global _GCLOUD_ANONYMOUS - if _GCLOUD_ANONYMOUS is None: - with _GCLOUD_ANONYMOUS_LOCK: - if _GCLOUD_ANONYMOUS is None: - anonymous = not _check_gcloud_credentials() - if anonymous: - _log(" note: no gcloud credentials found — reading " - "public artifact buckets anonymously") - _GCLOUD_ANONYMOUS = anonymous - env = os.environ.copy() - if _GCLOUD_ANONYMOUS: - env["CLOUDSDK_AUTH_DISABLE_CREDENTIALS"] = "true" - return env - - def _run_gcloud(args: list[str], timeout: int = 120) -> Optional[str]: - """Run a gcloud CLI command, returning stdout or None on error. - - Failures other than "no such object" are recorded in - ``_COLLECTION_ERRORS`` so callers can distinguish an unreadable - source from an empty one. - """ + """Run a gcloud CLI command, returning stdout or None on error.""" try: result = subprocess.run( - args, capture_output=True, text=True, timeout=timeout, - env=_gcloud_env(), + args, capture_output=True, text=True, timeout=timeout ) if result.returncode != 0: - reason = _classify_gcloud_stderr(result.stderr) - if reason != "no_match": - _record_collection_error(reason, args, detail=result.stderr) return None return result.stdout - except subprocess.TimeoutExpired: - _record_collection_error( - "timeout", args, detail=f"exceeded {timeout}s" - ) - return None - except FileNotFoundError: - _record_collection_error("gcloud_missing", args) + except (subprocess.TimeoutExpired, FileNotFoundError): return None def _run_gcloud_bytes(args: list[str], timeout: int = 120) -> Optional[bytes]: - """Run a gcloud CLI command, returning raw stdout bytes or None. - - Records failures the same way as :func:`_run_gcloud`. - """ + """Run a gcloud CLI command, returning raw stdout bytes or None.""" try: result = subprocess.run( - args, capture_output=True, timeout=timeout, env=_gcloud_env() + args, capture_output=True, timeout=timeout ) if result.returncode != 0: - stderr = (result.stderr or b"").decode("utf-8", errors="replace") - reason = _classify_gcloud_stderr(stderr) - if reason != "no_match": - _record_collection_error(reason, args, detail=stderr) return None return result.stdout - except subprocess.TimeoutExpired: - _record_collection_error( - "timeout", args, detail=f"exceeded {timeout}s" - ) - return None - except FileNotFoundError: - _record_collection_error("gcloud_missing", args) + except (subprocess.TimeoutExpired, FileNotFoundError): return None @@ -3074,62 +2200,21 @@ def _check_gcloud() -> bool: return False -def _check_gcloud_credentials() -> bool: - """Report whether gcloud has an active credential. - - ``gcloud --version`` only proves the binary exists. An installed but - unauthenticated gcloud passes that check and then fails every read, - which previously produced a snapshot full of silently empty JUnit - data. Public buckets can still be read anonymously, so a negative - result here is a warning rather than a hard failure. - """ - try: - result = subprocess.run( - ["gcloud", "auth", "list", - "--filter=status:ACTIVE", "--format=value(account)"], - capture_output=True, text=True, timeout=15, - ) - return result.returncode == 0 and bool(result.stdout.strip()) - except (FileNotFoundError, subprocess.TimeoutExpired): - return False - - def _check_podman() -> bool: """Verify that podman is available and functional. Goes beyond --version: runs 'podman info' which exercises the runtime, storage, and namespace setup. This catches cases where podman is installed but cannot operate (e.g. nested containers without - appropriate privileges). On failure, stdout/stderr from the probe are - logged so the underlying runtime error (e.g. a namespace or storage - driver failure under nested podman) is visible instead of a bare - "not available" message. + appropriate privileges). """ try: result = subprocess.run( ["podman", "info"], capture_output=True, text=True, timeout=30, ) - if result.returncode != 0: - _log(f"'podman info' exited {result.returncode}.") - if result.stdout.strip(): - _log(f"stdout:\n{result.stdout.strip()}") - if result.stderr.strip(): - _log(f"stderr:\n{result.stderr.strip()}") return result.returncode == 0 - except subprocess.TimeoutExpired as e: - _log("'podman info' timed out after 30s.") - if e.stdout: - stdout = e.stdout if isinstance(e.stdout, str) else e.stdout.decode(errors="replace") - if stdout.strip(): - _log(f"stdout:\n{stdout.strip()}") - if e.stderr: - stderr = e.stderr if isinstance(e.stderr, str) else e.stderr.decode(errors="replace") - if stderr.strip(): - _log(f"stderr:\n{stderr.strip()}") - return False - except OSError as e: - _log(f"'podman info' could not be run: {e}") + except (OSError, subprocess.TimeoutExpired): return False @@ -3222,11 +2307,8 @@ class _TestResult: """Parsed test result from JUnit XML.""" name: str - status: str # passed, failed, error, skipped, flake + status: str # passed, failed, error, skipped suite_name: str = "" - # The testcase's own lifecycle attribute, NOT the job's. Absent means - # the test gates the job; "informing" means it cannot. - test_lifecycle: str = "" failure_message: str = "" failure_text: str = "" error_message: str = "" @@ -3273,17 +2355,12 @@ def _parse_system_out_yaml(text: str) -> dict: return result -def _parse_junit_xml( - source, source_name: str = "" -) -> Optional[list[_TestResult]]: +def _parse_junit_xml(source, source_name: str = "") -> list[_TestResult]: """Parse JUnit XML, returning a list of _TestResult.""" try: tree = ET.parse(source) except (ET.ParseError, OSError): - # Return None, not []: an unreadable or corrupt file is not the same - # as a file that legitimately contains no failures. Callers must be - # able to tell those apart or corruption becomes a verified zero. - return None + return [] root = tree.getroot() if root.tag == "testsuites": @@ -3300,7 +2377,6 @@ def _parse_junit_xml( suite_name = suite.get("name", "") for tc in suite.findall("testcase"): name = tc.get("name", "") - test_lifecycle = tc.get("lifecycle", "") failure_el = tc.find("failure") error_el = tc.find("error") skipped_el = tc.find("skipped") @@ -3338,7 +2414,6 @@ def _parse_junit_xml( name=name, status=status, suite_name=suite_name, - test_lifecycle=test_lifecycle, failure_message=failure_message, failure_text=failure_text, error_message=error_message, @@ -3348,56 +2423,19 @@ def _parse_junit_xml( agg_failures=agg_failures, agg_skips=agg_skips, )) - return _mark_flakes(results) - - -def _mark_flakes(results: list[_TestResult]) -> list[_TestResult]: - """Re-label failures as flakes when the same test also passed. - - A test that failed and then passed on retry is a flake. Flakes do not - fail jobs, so counting them as failures overstates what could have - rejected a payload — and lets a flake drive regression onset. - - Grouped per (suite, name): the same name in two suites is two tests. - """ - passed: set[tuple[str, str]] = { - (r.suite_name, r.name) for r in results if r.status == "passed" - } - for r in results: - if r.status in ("failed", "error") and (r.suite_name, r.name) in passed: - r.status = "flake" return results -def _is_gating(entry: dict) -> bool: - """True when this recorded result could actually fail its job. - - Excludes flakes (a later pass cleared them) and tests explicitly opted - out via ``lifecycle="informing"`` — informing tests are run to - stabilize them and are not expected to gate. - """ - return ( - entry.get("status") in ("failed", "error") - and entry.get("test_lifecycle") != "informing" - ) - - def _test_results_to_json(results: list[_TestResult]) -> list[dict]: - """Convert _TestResult list to JSON dicts (failures, errors and flakes). - - Flakes are included so consumers can see them, but each entry carries - ``status`` and ``test_lifecycle`` so only genuinely gating results are - counted as failures. - """ + """Convert _TestResult list to JSON-serializable dicts (failures only).""" output = [] for r in results: - if r.status not in ("failed", "error", "flake"): + if r.status not in ("failed", "error"): continue d: dict = { "name": r.name, "status": r.status, "suite_name": r.suite_name, - "test_lifecycle": r.test_lifecycle or "blocking", } if r.failure_message: d["failure_message"] = r.failure_message @@ -3452,22 +2490,13 @@ def main() -> None: "--no-junit", action="store_true", help="Skip JUnit download, regression tracking", ) - parser.add_argument( - "--fail-on-incomplete", action="store_true", - help=("Exit non-zero if any data could not be collected. Use in " - "automation so an incomplete snapshot is not analyzed as " - "though it were complete."), - ) parser.add_argument( "--no-rpmdb", action="store_true", help="Skip RHCOS RPMDB extraction", ) parser.add_argument( "--sippy", action="store_true", - help=( - "Force Sippy for all payload metadata (default: prefer release " - "controller and automatically fall back for historical tags)" - ), + help="Use Sippy APIs instead of release controller (for historical payloads)", ) args = parser.parse_args() @@ -3493,20 +2522,6 @@ def main() -> None: _log("Warning: 'gcloud' CLI not found. JUnit data will not be fetched.") _log("Install gcloud SDK to enable JUnit download and regression tracking.\n") collect_junit = False - # JUnit was requested and cannot be collected, so the snapshot is - # incomplete by construction. Record it: otherwise the completeness - # gate sees an empty ledger and reports a complete snapshot. - _record_collection_error( - "gcloud_missing", - ["gcloud", "storage"], - detail=("gcloud CLI not available; JUnit, build logs and " - "regression tracking were skipped"), - stage="preflight", - ) - elif collect_junit and not _check_gcloud_credentials(): - _log("Note: 'gcloud' has no active credentials. CI artifact buckets") - _log("are public, so they will be read anonymously. Run") - _log("'gcloud auth login' if you need private buckets too.\n") collect_rpmdb = not args.no_rpmdb if collect_rpmdb and not _check_podman(): @@ -3515,7 +2530,7 @@ def main() -> None: collect_rpmdb = False if args.sippy: - _log("Forcing Sippy APIs for all release payload data.\n") + _log("Using Sippy APIs for release controller data.\n") try: snapshotter = Snapshotter( @@ -3528,39 +2543,6 @@ def main() -> None: collect_rpmdb=collect_rpmdb, ) snapshotter.run() - - unrecovered = _unrecovered_errors() - recovered = len(_COLLECTION_ERRORS) - len(unrecovered) - if recovered: - _log("") - status = ("data is complete" if not unrecovered - else "see the warning below for what is still missing") - _log(f"Note: {recovered} read failure(s) were recovered by a " - f"fallback; {status}. See 'collection_errors' " - f"(recovered: true) in summary.json.") - if unrecovered: - by_reason: dict[str, int] = {} - for e in unrecovered: - r = e.get("reason", "unknown") - by_reason[r] = by_reason.get(r, 0) + 1 - _log("") - _log("=" * 68) - _log(f"WARNING: SNAPSHOT IS INCOMPLETE " - f"({len(unrecovered)} unrecovered collection error(s))") - for reason, count in sorted(by_reason.items()): - _log(f" {reason}: {count}") - affected = sorted({ - e["job"] for e in unrecovered if e.get("job") - }) - if affected: - _log(f" affected jobs: {', '.join(affected)}") - _log("") - _log("Missing data is ABSENT, not empty. Do not interpret a job") - _log("without test_failure_count as having zero test failures.") - _log("Details: 'collection_errors' in summary.json") - _log("=" * 68) - if args.fail_on_incomplete: - sys.exit(1) except urllib.error.HTTPError as e: print(f"Error: HTTP {e.code}: {e.reason}", file=sys.stderr) sys.exit(1) diff --git a/plugins/ci/skills/payload-snapshot/scripts/test_collection_completeness.py b/plugins/ci/skills/payload-snapshot/scripts/test_collection_completeness.py deleted file mode 100644 index a359c76cd..000000000 --- a/plugins/ci/skills/payload-snapshot/scripts/test_collection_completeness.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for the snapshot completeness contract. - -The contract under test: data the snapshot could not read must never be -published as an authoritative zero. Each test below corresponds to a way -that guarantee was previously broken. - -Run with: pytest test_collection_completeness.py -""" - -import importlib.util -import json -import os -import sys -import threading - -import pytest - -_HERE = os.path.dirname(os.path.abspath(__file__)) - - -def _load_module(): - spec = importlib.util.spec_from_file_location( - "payload_snapshot", os.path.join(_HERE, "payload_snapshot.py") - ) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -ps = _load_module() - - -@pytest.fixture(autouse=True) -def clean_ledger(): - """Each test starts with an empty global error ledger.""" - ps._COLLECTION_ERRORS.clear() - ps._GCLOUD_ANONYMOUS = True # never probe real credentials in tests - yield - ps._COLLECTION_ERRORS.clear() - - -def _job(name="job-a", bucket="bucket/path"): - return ps.JobInfo( - name=name, state="Failed", lifecycle="blocking", url="", - retries=0, previous_attempt_urls=[], is_aggregated=False, - gcs_bucket_path=bucket, - ) - - -def _collector(tmp_path, job=None, tag="4.99.0-0.ci-2026-01-01-000000"): - return ps.JUnitCollector(str(tmp_path / "junit"), job or _job(), tag) - - -VALID_JUNIT = ( - '' - 'boom' - "" -) - - -# -------------------------------------------------------------------------- -# Error classification -# -------------------------------------------------------------------------- - -@pytest.mark.parametrize("stderr,expected", [ - ("ERROR: One or more URLs matched no objects.", "no_match"), - ("ERROR: 403 does not have storage.objects.list access", "auth"), - ("ERROR: Anonymous caller does not have access", "auth"), - ("ERROR: something unexpected", "command_failed"), -]) -def test_stderr_classification(stderr, expected): - assert ps._classify_gcloud_stderr(stderr) == expected - - -def test_no_match_is_not_recorded_as_error(monkeypatch): - """A missing object is absence, not failure — probes must stay quiet.""" - monkeypatch.setattr(ps.subprocess, "run", lambda *a, **k: type( - "R", (), {"returncode": 1, "stdout": "", - "stderr": "ERROR: matched no objects"})()) - assert ps._run_gcloud(["gcloud", "storage", "ls", "x"]) is None - assert ps._COLLECTION_ERRORS == [] - - -def test_auth_failure_is_recorded(monkeypatch): - monkeypatch.setattr(ps.subprocess, "run", lambda *a, **k: type( - "R", (), {"returncode": 1, "stdout": "", - "stderr": "ERROR: 403 forbidden"})()) - assert ps._run_gcloud(["gcloud", "storage", "ls", "x"]) is None - assert [e["reason"] for e in ps._COLLECTION_ERRORS] == ["auth"] - - -def test_detail_is_sanitized(): - """Persisted diagnostics must not carry control characters.""" - ps._record_collection_error( - "auth", ["gcloud"], detail="bad\x1b[31mred\x00\nthing" - ) - detail = ps._COLLECTION_ERRORS[0]["detail"] - assert "\x1b" not in detail and "\x00" not in detail - assert "\n" not in detail - - -# -------------------------------------------------------------------------- -# Recovery bookkeeping -# -------------------------------------------------------------------------- - -def test_recovery_is_scoped_to_its_own_operation(): - """A recovered fallback must not clear another operation's error.""" - with ps._error_scope() as mine: - ps._record_collection_error("timeout", ["gcloud"], job="mine") - # A concurrent collector records its own, unrelated failure. - ps._record_collection_error("auth", ["gcloud"], job="theirs") - - # Only the entry captured for *this* operation is recovered. - ps._mark_errors_recovered([e for e in mine if e.get("job") == "mine"]) - - by_job = {e["job"]: e for e in ps._COLLECTION_ERRORS} - assert by_job["mine"].get("recovered") is True - assert by_job["theirs"].get("recovered") is None - assert len(ps._unrecovered_errors()) == 1 - - -def test_concurrent_collectors_do_not_recover_each_others_errors(): - """Threads racing on the shared ledger must stay isolated.""" - started = threading.Barrier(2) - - def worker(tag): - with ps._error_scope() as own: - ps._record_collection_error("timeout", ["gcloud"], job=tag) - started.wait(timeout=5) # force interleaving - if tag == "recovers": - ps._mark_errors_recovered(own) - - threads = [threading.Thread(target=worker, args=(t,)) - for t in ("recovers", "keeps")] - for t in threads: - t.start() - for t in threads: - t.join(timeout=10) - - by_job = {e["job"]: e for e in ps._COLLECTION_ERRORS} - assert by_job["recovers"].get("recovered") is True - assert by_job["keeps"].get("recovered") is None - - -def test_data_complete_ignores_recovered_errors(): - with ps._error_scope() as own: - ps._record_collection_error("timeout", ["gcloud"]) - ps._mark_errors_recovered(own) - assert ps._unrecovered_errors() == [] - - -# -------------------------------------------------------------------------- -# Parsing -# -------------------------------------------------------------------------- - -def test_malformed_xml_returns_none_not_empty(tmp_path): - """Corrupt XML must be distinguishable from "no failures".""" - bad = tmp_path / "junit_bad.xml" - bad.write_text('trunc') - assert ps._parse_junit_xml(str(bad)) is None - - -def test_valid_xml_with_no_failures_returns_empty_list(tmp_path): - good = tmp_path / "junit_ok.xml" - good.write_text('' - "") - results = ps._parse_junit_xml(str(good)) - assert results is not None - assert ps._test_results_to_json(results) == [] - - -# -------------------------------------------------------------------------- -# Publishing decisions -# -------------------------------------------------------------------------- - -def _stub_discovery(monkeypatch, files): - monkeypatch.setattr( - ps.JUnitCollector, "_list_junit_files", lambda self: list(files) - ) - - -def test_unreadable_junit_leaves_results_absent(tmp_path, monkeypatch): - """Read failure => results.json absent (unknown), never [].""" - _stub_discovery(monkeypatch, ["gs://b/junit_operator.xml"]) - monkeypatch.setattr(ps, "_run_gcloud_bytes", lambda *a, **k: ( - ps._record_collection_error("auth", ["gcloud"]) or None)) - - c = _collector(tmp_path) - assert c.collect() is False - assert not os.path.exists(c.output_path) - assert any(e["reason"] == "junit_unavailable" - for e in ps._COLLECTION_ERRORS) - - -def test_no_junit_discovered_is_unknown_not_zero(tmp_path, monkeypatch): - """A failed job with no JUnit at all is unknown, not verified clean.""" - _stub_discovery(monkeypatch, []) - - c = _collector(tmp_path) - assert c.collect() is False - assert not os.path.exists(c.output_path) - assert any(e["reason"] == "junit_missing" - for e in ps._COLLECTION_ERRORS) - - -def test_partial_read_is_published_but_flagged(tmp_path, monkeypatch): - """Half the files read => real results, explicitly marked partial.""" - _stub_discovery(monkeypatch, ["gs://b/junit_a.xml", "gs://b/junit_b.xml"]) - - def fake_cat(args, **kwargs): - if args[-1].endswith("junit_a.xml"): - return VALID_JUNIT.encode() - ps._record_collection_error("auth", ["gcloud"]) - return None - - monkeypatch.setattr(ps, "_run_gcloud_bytes", fake_cat) - - c = _collector(tmp_path) - assert c.collect() is True - assert json.load(open(c.output_path)) # real failures published - assert any(e["reason"] == "junit_partial" - for e in ps._COLLECTION_ERRORS) - - -def test_unparseable_download_counts_as_unread(tmp_path, monkeypatch): - """Corrupt XML must not be published as zero failures.""" - _stub_discovery(monkeypatch, ["gs://b/junit_bad.xml"]) - monkeypatch.setattr( - ps, "_run_gcloud_bytes", lambda *a, **k: b" failed, then passed on retry -> does NOT gate - informing -> lifecycle="informing" (stabilizing) -> does NOT gate - failure -> failed everywhere, no lifecycle -> GATES - -Counting the other two as failures overstates what could have rejected a -payload, and lets a non-cause drive regression onset. - -Run with: pytest test_test_classification.py -""" - -import importlib.util -import os -import sys - -import pytest - -_HERE = os.path.dirname(os.path.abspath(__file__)) - - -def _load_module(): - spec = importlib.util.spec_from_file_location( - "payload_snapshot", os.path.join(_HERE, "payload_snapshot.py") - ) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -ps = _load_module() - - -def _xml(cases: str) -> str: - return f'{cases}' - - -def _case(name, *, failed=False, lifecycle=None, skipped=False): - attrs = f'name="{name}"' - if lifecycle: - attrs += f' lifecycle="{lifecycle}"' - body = "" - if failed: - body = "boom" - elif skipped: - body = "" - return f"{body}" - - -def _parse(tmp_path, xml): - path = tmp_path / "junit.xml" - path.write_text(xml) - return ps._parse_junit_xml(str(path)) - - -def _json(tmp_path, xml): - return ps._test_results_to_json(_parse(tmp_path, xml)) - - -# -------------------------------------------------------------------------- -# Flakes -# -------------------------------------------------------------------------- - -def test_fail_then_pass_is_a_flake(tmp_path): - entries = _json(tmp_path, _xml( - _case("t", failed=True) + _case("t") - )) - assert [e["status"] for e in entries] == ["flake"] - assert not ps._is_gating(entries[0]) - - -def test_fail_only_is_a_failure(tmp_path): - entries = _json(tmp_path, _xml(_case("t", failed=True))) - assert [e["status"] for e in entries] == ["failed"] - assert ps._is_gating(entries[0]) - - -def test_multiple_failures_with_one_pass_is_still_a_flake(tmp_path): - """A flake is one success among any number of failures.""" - entries = _json(tmp_path, _xml( - _case("t", failed=True) * 3 + _case("t") - )) - assert {e["status"] for e in entries} == {"flake"} - assert not any(ps._is_gating(e) for e in entries) - - -def test_pass_in_a_different_suite_does_not_clear_a_failure(tmp_path): - """Same name in two suites is two different tests.""" - xml = ( - '' - f'{_case("t", failed=True)}' - f'{_case("t")}' - '' - ) - entries = _json(tmp_path, xml) - assert [e["status"] for e in entries] == ["failed"] - assert ps._is_gating(entries[0]) - - -def test_skipped_does_not_clear_a_failure(tmp_path): - entries = _json(tmp_path, _xml( - _case("t", failed=True) + _case("t", skipped=True) - )) - assert [e["status"] for e in entries] == ["failed"] - - -# -------------------------------------------------------------------------- -# Informing -# -------------------------------------------------------------------------- - -def test_informing_failure_does_not_gate(tmp_path): - entries = _json(tmp_path, _xml( - _case("udn", failed=True, lifecycle="informing") - )) - assert entries[0]["test_lifecycle"] == "informing" - assert entries[0]["status"] == "failed" - assert not ps._is_gating(entries[0]) - - -def test_absent_lifecycle_gates(tmp_path): - """No lifecycle attribute means the test gates the job.""" - entries = _json(tmp_path, _xml(_case("t", failed=True))) - assert entries[0]["test_lifecycle"] == "blocking" - assert ps._is_gating(entries[0]) - - -def test_explicit_blocking_lifecycle_gates(tmp_path): - entries = _json(tmp_path, _xml( - _case("t", failed=True, lifecycle="blocking") - )) - assert ps._is_gating(entries[0]) - - -def test_informing_flake_is_neither(tmp_path): - entries = _json(tmp_path, _xml( - _case("t", failed=True, lifecycle="informing") + _case("t") - )) - assert entries[0]["status"] == "flake" - assert not ps._is_gating(entries[0]) - - -# -------------------------------------------------------------------------- -# Counting -# -------------------------------------------------------------------------- - -def test_only_gating_results_are_counted_as_failures(tmp_path): - """The mixed case: one real failure among flakes and informing noise.""" - entries = _json(tmp_path, _xml( - _case("real", failed=True) - + _case("flaky", failed=True) + _case("flaky") - + _case("udn1", failed=True, lifecycle="informing") - + _case("udn2", failed=True, lifecycle="informing") - )) - gating = [e for e in entries if ps._is_gating(e)] - assert [e["name"] for e in gating] == ["real"] - # One flake entry: the failing attempt, relabelled. The passing attempt - # is not a result worth recording. - assert sum(1 for e in entries if e["status"] == "flake") == 1 - assert sum( - 1 for e in entries - if e["status"] == "failed" and e["test_lifecycle"] == "informing" - ) == 2 - - -def test_flakes_are_still_recorded_for_visibility(tmp_path): - """Not gating is not the same as not worth reporting.""" - entries = _json(tmp_path, _xml( - _case("flaky", failed=True) + _case("flaky") - )) - assert entries, "flakes must remain visible in results.json" - - -def test_passing_tests_are_not_recorded(tmp_path): - assert _json(tmp_path, _xml(_case("t"))) == [] - - -# -------------------------------------------------------------------------- -# Against real aggregated JUnit shapes -# -------------------------------------------------------------------------- - -REAL_SHAPE = _xml( - # A real gating invariant, as emitted with no lifecycle attribute. - _case( - '[Monitor:legacy-cvo-invariants][bz-config-operator] ' - 'clusteroperator/config-operator must go Progressing=True', - failed=True, - ) - # A UDN test being stabilized, opted out of gating. - + _case( - '[Feature:NetworkSegmentation][ovn-kubernetes-ote][sig-network] ' - 'Network Segmentation UserDefinedNetwork CRD Controller', - failed=True, lifecycle="informing", - ) -) - - -def test_real_aggregated_shape_splits_correctly(tmp_path): - entries = _json(tmp_path, REAL_SHAPE) - gating = [e for e in entries if ps._is_gating(e)] - assert len(gating) == 1 - assert "config-operator" in gating[0]["name"] - informing = [e for e in entries if e["test_lifecycle"] == "informing"] - assert len(informing) == 1 - assert "NetworkSegmentation" in informing[0]["name"] - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/plugins/ci/skills/prow-job-analysis/SKILL.md b/plugins/ci/skills/prow-job-analysis/SKILL.md deleted file mode 100644 index faeb9a3e4..000000000 --- a/plugins/ci/skills/prow-job-analysis/SKILL.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -name: prow-job-analysis -description: Use this skill when debugging a failed Prow CI job. ---- - -# Prow Job Analysis - -Analyze failures in OpenShift Prow CI jobs. Identify the job type, inspect artifacts, classify -the failure, and route to the specialized reference for deep analysis. - -## Input Format - -The user will provide: - -1. **Prow job URL** (required) — Prow UI or gcsweb URL - - `https://prow.ci.openshift.org/view/gs/test-platform-results/logs//` - - `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/...` - -2. **Test name** (optional) — specific failed test to focus on - -3. **Flags** (optional): - - `--backends ` — focus disruption analysis on specific backends - -## Prerequisites - -- **Python 3.7+**: `which python3` -- **jq**: `which jq` -- **gcloud CLI** (recommended, not required): `which gcloud` — fastest access to the - public bucket (no auth required). Without it, every artifact operation works over - plain HTTPS: [prow_job_artifact_search.py](prow_job_artifact_search.py) - (stdlib-only `list`/`search`/`fetch`) or `curl` against - `https://storage.googleapis.com/test-platform-results/...`. - -## Investigation Workflow - -### Step 1: Parse URL and Extract Metadata - -1. Find `test-platform-results/` in the URL and extract the bucket path -2. Extract `build_id` — pattern `(\d{10,})` in the path -3. Construct GCS base: `gs://test-platform-results/{bucket-path}/` - -### Step 2: Fetch prowjob.json - -Use the `fetch-prowjob-json` skill to get job metadata. Extract: -- **Job name** from `.spec.job` -- **Target** from `--target=` in ci-operator args -- **Job state** from `.status.state` -- **Refs** (org, repo, PR number) from `.spec.refs` - -### Step 3: Classify Job Type from Name - -Parse the job name to determine the environment and expected failure modes: - -| Pattern in Name | Job Type | Key Implications | -|-----------------|----------|------------------| -| `upgrade` | Upgrade job | Installs first, then upgrades — see [upgrade reference](references/upgrade.md) | -| `metal`, `baremetal` | Bare metal | Uses dev-scripts + Metal3/Ironic — see [metal install reference](references/install/metal.md) | -| `hypershift` | HyperShift | Hosted control planes — see [hypershift reference](references/hypershift.md) | -| `fips` | FIPS-enabled | Watch for crypto/TLS errors | -| `ipv6`, `dualstack` | IPv6/dualstack | Often disconnected, uses mirror registry | -| `single-node`, `sno` | Single-node | Resource exhaustion more likely | -| `aggregated-` prefix | Aggregated | Statistical analysis of multiple runs — see [aggregated reference](references/aggregated.md) | -| `aws`, `gcp`, `azure` | Cloud platform | Platform-specific errors — see [cloud provider reference](references/cloud-provider-errors.md) | -| `techpreview` | Tech preview | Feature gates enabled, features may be unstable | -| `rhcos9`, `rhcos10`, `rhcos9_10`, `rt` | RHCOS variant / RT kernel | OS variant pinned or heterogeneous; OS-level differences (kernel/systemd/SELinux) — see [operating system changes reference](references/operating-system-changes.md) | - -### Step 4: Download Key Artifacts - -```bash -mkdir -p .work/prow-job-analysis/{build_id}/logs - -# Build log (always) -gcloud storage cp gs://test-platform-results/{bucket-path}/build-log.txt \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - -# JUnit XML (always — identifies failed tests/steps) -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/junit*.xml" 2>/dev/null - -# Node journals (always, when the job created a cluster) — required input for the -# Step 5 OS-layer check. Gzip-compressed WITHOUT a .gz extension: zcat/zgrep only. -gcloud storage cp -r \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/nodes" \ - .work/prow-job-analysis/{build_id}/ --no-user-output-enabled 2>/dev/null || true -``` - -### Step 5: Classify Failure and Route to Reference - -Examine the build log and JUnit results to classify the failure, then consult the -appropriate reference file for detailed analysis procedures. - -#### OS-layer evidence check (mandatory for every job, before routing) - -Operating-system (RHCOS) layer breakage frequently masquerades as an unrelated product -failure: a single RHCOS bump swaps the kernel, cri-o, systemd, NetworkManager, and SELinux -policy across the whole cluster at once, so the real cause surfaces as a symptom in some -other domain. Before selecting a row from the routing table, complete BOTH steps: - -**1. Compare runtime versions across boots in the node journals** (downloaded in -Step 4; gzip-compressed **without** a `.gz` extension — plain `grep` silently matches -nothing, use `zcat`/`zgrep`): - -```bash -# Runtime versions per boot. End-of-run snapshots (oc_cmds/nodes, nodes.json) -# show only the final version; changes within the run are visible only here. -zgrep -hE "Starting CRI-O, version|Container runtime initialized" \ - .work/prow-job-analysis/{build_id}/nodes/*/journal | sort | uniq -c -``` - -**2. Scan the build log, JUnit, `oc_cmds` (node / clusteroperator status), -MachineConfig data, and the journals for these signals:** - -- `NetworkPluginNotReady`, or a missing CNI config (`/etc/cni/net.d` empty / no CNI plugin) -- A `ContainerRuntimeVersion` change on nodes (cri-o version bump between runs) -- A MachineConfigDaemon (MCD) rendered-config diff touching `passwd`, `files`, or `units` -- Multiple nodes going `NotReady` after a reboot -- `CreateContainerError`, `RunContainerError`, or OCI runtime errors (`crun` / `runc`) -- Kernel `panic`, `BUG`, `Oops`, or `soft lockup` in node journals or the serial console -- `avc: denied` / SELinux denials -- The same failure spanning multiple unrelated jobs at a payload boundary - -If step 1 shows more than one runtime version on any node, or any step-2 signal is -present, the RHCOS layer is implicated: still route via the table below using whichever -reference matches the surface symptom, but **also** read -[operating-system-changes.md](references/operating-system-changes.md) alongside it. -Never clear the OS layer from end-of-run snapshots alone. - -## Failure Routing Table - -| Failure Signal | Reference | When to Use | -|----------------|-----------|-------------| -| `install should succeed` fails in JUnit | [Install — General](references/install/general.md) | Install failed at config/infra/bootstrap/cluster-creation/operator-stability stage | -| Metal/baremetal job + install failure | [Install — Metal](references/install/metal.md) | Bare-metal install (dev-scripts, Metal3/Ironic, libvirt) — use alongside Install — General | -| A test failed (start here) | [Flaky Test Identification](references/flaky-test-identification.md) | Triage entry for any failing test: classify infra vs product regression vs flake, then route onward | -| Confirmed regression in a plain e2e test | [Test Failure Root-Cause](references/test-failure.md) | Root-cause a real product regression in a plain (non-extension/install/upgrade) e2e test — e.g. `[sig-network] ... should serve endpoints`: test source, cluster state, originating error | -| `*-tests-ext` extension binary error | [Test Extension Binaries](references/test-extension-binaries.md) | OTE extension-binary extraction/discovery/version-skew failures — not core `openshift-tests` | -| Disruption events in intervals | [Disruption](references/disruption.md) | API backends stopped responding; interpret interval/timeline data (cause vs symptom vs noise) | -| Upgrade-phase failure or regression | [Upgrade](references/upgrade.md) | CVO stuck, operators degraded, MCO drain/reboot stalls, or version skew during upgrade | -| HyperShift / HCP job failure | [HyperShift](references/hypershift.md) | Hosted control planes — correlate management and hosted clusters | -| `aggregated-` job failure | [Aggregated Jobs](references/aggregated.md) | Statistical regression analysis across parallel child runs | -| Cloud API errors, quota, throttling | [Cloud Provider Errors](references/cloud-provider-errors.md) | AWS/GCP/Azure API/quota/provisioning failures before/during cluster creation | -| Node NotReady, OOM, disk pressure | [Resource Exhaustion](references/resource-exhaustion.md) | CPU/memory/disk/PID/etcd exhaustion, eviction, unschedulable pods | -| DNS, OVN, registry/pull, ingress errors | [Networking](references/networking.md) | OVN-Kubernetes/SDN, DNS, image pull/registry, load balancer/ingress, network policy | -| Container-start (cri-o), kernel panic, NetworkManager, RHCOS variant-isolated failure | [Operating System Changes](references/operating-system-changes.md) | Node OS (RHCOS) layer — cri-o/crun, kernel, systemd, NetworkManager, SELinux, or an RHCOS bump in the payload | -| Lease/quota, ci-operator, Prow infra | [CI Infrastructure](references/ci-infrastructure-changes.md) | Distinguish "product broke" from "CI config changed"; ci-operator, step registry, leases | -| Need a specific artifact file | [Artifacts](references/artifacts.md) | Artifact directory structure, paths, and gcloud fetch commands | - -## Common Artifact Paths - -These are the most frequently needed artifacts. See [artifacts reference](references/artifacts.md) for the complete directory structure. - -| Path | Description | -|------|-------------| -| `build-log.txt` | Top-level ci-operator log | -| `artifacts/{target}/openshift-e2e-test/build-log.txt` | E2E test console log | -| `artifacts/{target}/openshift-e2e-test/artifacts/junit/` | JUnit XML results | -| `artifacts/{target}/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_*.json` | Disruption timeline data | -| `artifacts/{target}/gather-extra/artifacts/oc_cmds/` | Cluster state snapshots | -| `artifacts/{target}/gather-extra/artifacts/pods/` | Pod logs by namespace | -| `artifacts/{target}/gather-extra/artifacts/audit_logs/` | API server audit logs | -| `artifacts/{target}/gather-must-gather/artifacts/must-gather.tar` | Must-gather archive | -| `prowjob.json` | Job metadata and timing | - -## URL Formats - -Both formats are accepted and interchangeable: - -```text -# Prow UI -https://prow.ci.openshift.org/view/gs/test-platform-results/logs/{job}/{build_id} - -# gcsweb (direct GCS browser) -https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/{job}/{build_id} -``` - -The GCS bucket is always `test-platform-results`, publicly accessible, no auth required. - -## Tips - -- **Start with build-log.txt** — it shows the ci-operator orchestration and which steps failed -- **JUnit XML is the source of truth** for test pass/fail status -- **Job name encodes environment** — always parse it before diving into logs -- **Check `prowjob.json`** for timing, payload tag, and whether the job timed out -- **Upgrade jobs install first** — an "upgrade" job failing at install is an install failure, not an upgrade failure -- **Aggregated jobs** need statistical analysis, not individual test debugging -- **Use `.work/prow-job-analysis/{build_id}/`** as the working directory for downloads diff --git a/plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py b/plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py deleted file mode 100755 index 33a56d828..000000000 --- a/plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py +++ /dev/null @@ -1,605 +0,0 @@ -#!/usr/bin/env python3 -""" -Search and retrieve artifacts from Prow CI job runs stored in GCS. - -Provides list, search, and fetch operations against Prow's public GCS artifact -buckets. Both the ``test-platform-results`` and ``prow-artifact-archive`` -buckets are supported; the correct one is detected automatically from the Prow -URL. The buckets are PUBLIC (no authentication required), so this script works -two ways: - - 1. If the `gcloud` CLI is installed, it is used (fast, native globbing). - 2. Otherwise it falls back to the public GCS JSON/download API over plain - HTTPS via the Python standard library (urllib) — no external tools or - credentials needed. This keeps the skill usable in minimal containers - (e.g. CI eval images) that do not ship the Cloud SDK. - -Both backends produce identical JSON output, so callers do not need to know or -care which one ran. Set PROW_ARTIFACT_SEARCH_NO_GCLOUD=1 to force the HTTP -fallback even when gcloud is present. - -Usage: - prow_job_artifact_search.py list [subpath] - prow_job_artifact_search.py search [subpath] - prow_job_artifact_search.py fetch [--max-bytes N] - -Examples: - # List top-level artifacts - prow_job_artifact_search.py list - - # List a specific subdirectory - prow_job_artifact_search.py list artifacts/e2e-test/openshift-e2e-test - - # Search for files matching a glob pattern (recursive) - prow_job_artifact_search.py search "**/*intervals*.json" - - # Search within a subdirectory - prow_job_artifact_search.py search "**/nodes" artifacts/e2e-test/gather-extra - - # Fetch a specific file - prow_job_artifact_search.py fetch artifacts/e2e-test/build-log.txt - - # Fetch with size limit (default 512KB) - prow_job_artifact_search.py fetch artifacts/e2e-test/build-log.txt --max-bytes 1048576 -""" - -import argparse -import json -import os -import re -import subprocess -import sys -import urllib.error -import urllib.parse -import urllib.request - - -# Prow stores CI artifacts across two public (world-readable) GCS buckets. -# The correct bucket for a given job is detected from the Prow URL by -# parse_prow_url(), and every operation is scoped to that detected bucket. -SUPPORTED_BUCKETS = ("test-platform-results", "prow-artifact-archive") -DEFAULT_MAX_BYTES = 512 * 1024 # 512KB - -# Public GCS endpoints (no auth — the buckets are world-readable). -GCS_API_ROOT = "https://storage.googleapis.com/storage/v1/b" -GCS_DOWNLOAD_ROOT = "https://storage.googleapis.com" -HTTP_TIMEOUT = 60 -HTTP_HEADERS = {"User-Agent": "prow-job-artifact-search/1.0"} -# Upper bound for a single fetch when the object size is not advertised via a -# Content-Length header (GCS uses decompressive transcoding for gzip-stored -# text and streams it without a length). 64 MiB is far larger than any text -# artifact this tool is used to read, while still bounding memory use. -FETCH_READ_CAP = 64 * 1024 * 1024 -# Safety cap for paginated listings (1000 pages * 1000 objects = 1M objects). -# Real jobs have at most a few thousand objects, so this is effectively -# unreachable; it only guards against a runaway pagination loop. -MAX_LIST_PAGES = 1000 - -_GCLOUD_AVAILABLE = None - - -def gcloud_available(): - """Return True if the gcloud CLI is usable, caching the result. - - Honors PROW_ARTIFACT_SEARCH_NO_GCLOUD=1 to force the HTTP fallback. - """ - global _GCLOUD_AVAILABLE - if _GCLOUD_AVAILABLE is not None: - return _GCLOUD_AVAILABLE - - if os.environ.get("PROW_ARTIFACT_SEARCH_NO_GCLOUD"): - _GCLOUD_AVAILABLE = False - return _GCLOUD_AVAILABLE - - try: - result = subprocess.run( - ["gcloud", "version"], - capture_output=True, - timeout=10, - ) - _GCLOUD_AVAILABLE = result.returncode == 0 - except (FileNotFoundError, subprocess.TimeoutExpired, OSError): - _GCLOUD_AVAILABLE = False - return _GCLOUD_AVAILABLE - - -def parse_prow_url(url): - """Extract the GCS bucket and path prefix from a Prow job URL. - - Detects which supported bucket the URL refers to - (``test-platform-results`` or ``prow-artifact-archive``) and works with - either host format: - - https://prow.ci.openshift.org/view/gs//logs// - - https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs//logs// - - Returns a ``(bucket, path)`` tuple where ``path`` is the portion after the - bucket name, e.g.: - ("test-platform-results", "logs//") - """ - # Alternation of the supported bucket names for use inside the URL patterns. - bucket_alt = "|".join(re.escape(b) for b in SUPPORTED_BUCKETS) - patterns = [ - rf"prow\.ci\.openshift\.org/view/gs/(?P{bucket_alt})/(?P.+?)/?$", - rf"gcsweb-ci\.apps\.ci\.l2s4\.p1\.openshiftapps\.com/gcs/(?P{bucket_alt})/(?P.+?)/?$", - ] - for pat in patterns: - m = re.search(pat, url) - if m: - return m.group("bucket"), m.group("path").rstrip("/") - - # Fallback: look for a supported bucket name anywhere in the URL (e.g. a - # raw storage.googleapis.com or gs:// style reference). - for bucket in SUPPORTED_BUCKETS: - marker = f"{bucket}/" - if marker in url: - return bucket, url.split(marker, 1)[1].rstrip("/") - - raise ValueError( - f"Cannot parse Prow URL: {url}\n" - "Expected a URL referencing one of the supported GCS buckets " - f"({' or '.join(SUPPORTED_BUCKETS)}), e.g.:\n" - " https://prow.ci.openshift.org/view/gs/test-platform-results/logs//\n" - " https://prow.ci.openshift.org/view/gs/prow-artifact-archive/logs//" - ) - - -def gcs_path(bucket, prefix, subpath=None): - """Build a gs:// URI within the given bucket.""" - base = f"gs://{bucket}/{prefix}" - if subpath: - subpath = subpath.strip("/") - base = f"{base}/{subpath}" - return base - - -def object_path(prefix, subpath=None): - """Build a bucket-relative object path (no gs:// or bucket name).""" - path = prefix.strip("/") - if subpath: - path = f"{path}/{subpath.strip('/')}" - return path - - -def run_gcloud(args, timeout=60): - """Run a gcloud command and return (stdout, stderr, returncode).""" - cmd = ["gcloud"] + args - try: - result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - ) - return result.stdout, result.stderr, result.returncode - except subprocess.TimeoutExpired: - return "", f"Command timed out after {timeout}s", 1 - - -# --------------------------------------------------------------------------- -# Public GCS HTTP backend (used when gcloud is unavailable) -# --------------------------------------------------------------------------- - - -def _http_get_json(url): - """GET a URL and parse the JSON body.""" - req = urllib.request.Request(url, headers=HTTP_HEADERS) - with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: - return json.loads(resp.read().decode("utf-8")) - - -def gcs_api_list(bucket, obj_prefix, delimiter=None): - """List objects under obj_prefix in ``bucket`` via the public GCS JSON API. - - Follows pagination. Returns (items, prefixes) where: - - items is a list of full object-name strings under obj_prefix - - prefixes is a list of immediate sub-directory prefixes (only populated - when delimiter="/" is passed) - """ - items = [] - prefixes = [] - page_token = None - pages = 0 - base = f"{GCS_API_ROOT}/{bucket}/o" - while True: - params = {"prefix": obj_prefix, "maxResults": "1000"} - if delimiter: - params["delimiter"] = delimiter - if page_token: - params["pageToken"] = page_token - data = _http_get_json(f"{base}?{urllib.parse.urlencode(params)}") - for item in data.get("items", []): - name = item.get("name") - if name is not None: - items.append(name) - prefixes.extend(data.get("prefixes", [])) - page_token = data.get("nextPageToken") - pages += 1 - if not page_token or pages >= MAX_LIST_PAGES: - break - return items, prefixes - - -def glob_to_regex(pattern): - """Translate a gcloud/gsutil-style glob into an anchored regex. - - Mirrors gsutil wildcard semantics: - - ``*`` matches any run of characters within a single path segment - (does not cross ``/``) - - ``**`` matches across path segments (any depth) - - ``**/`` matches zero or more complete path segments - - ``?`` matches a single non-``/`` character - """ - out = ["^"] - i, n = 0, len(pattern) - while i < n: - c = pattern[i] - if c == "*": - if i + 1 < n and pattern[i + 1] == "*": - if i + 2 < n and pattern[i + 2] == "/": - out.append("(?:.*/)?") # **/ -> zero or more segments - i += 3 - else: - out.append(".*") # ** -> across segments - i += 2 - else: - out.append("[^/]*") # * -> within a segment - i += 1 - elif c == "?": - out.append("[^/]") - i += 1 - else: - out.append(re.escape(c)) - i += 1 - out.append("$") - return "".join(out) - - -def _http_list_lines(bucket, prefix, subpath=None): - """Return gcloud-ls-style ``gs://`` lines for a directory listing.""" - obj_prefix = object_path(prefix, subpath) - if not obj_prefix.endswith("/"): - obj_prefix += "/" - - items, prefixes = gcs_api_list(bucket, obj_prefix, delimiter="/") - - lines = [] - for pfx in prefixes: # sub-directories already carry a trailing slash - lines.append(f"gs://{bucket}/{pfx}") - for name in items: - if name == obj_prefix: - continue # skip the directory placeholder object, if any - lines.append(f"gs://{bucket}/{name}") - return sorted(set(lines)) - - -def _http_search_lines(bucket, prefix, pattern, subpath=None): - """Return gcloud-ls-style ``gs://`` lines for a recursive glob search. - - Matches leaf objects (files) whose path relative to the search root matches - the glob, mirroring ``gcloud storage ls "/"``. Directory - placeholder objects are skipped: gcloud's wildcard ``ls`` does not return - bare directories for these patterns, and callers use ``list`` to browse - directories. - """ - search_root = object_path(prefix, subpath) - if not search_root.endswith("/"): - search_root += "/" - - items, _ = gcs_api_list(bucket, search_root, delimiter=None) - regex = re.compile(glob_to_regex(pattern)) - - lines = set() - for name in items: - if not name.startswith(search_root): - continue - rel = name[len(search_root):] - if not rel or rel.endswith("/"): - continue # skip the search root itself and directory placeholders - if regex.match(rel): - lines.add(f"gs://{bucket}/{name}") - return sorted(lines) - - -def _http_fetch(bucket, obj_path, max_bytes): - """Download an object's content from ``bucket`` via the public download API. - - Returns (size_bytes, truncated, content). ``size_bytes`` is the object's - full size (from Content-Length when available); ``content`` holds at most - ``max_bytes`` decoded characters. - """ - url = f"{GCS_DOWNLOAD_ROOT}/{bucket}/{urllib.parse.quote(obj_path)}" - req = urllib.request.Request(url, headers=HTTP_HEADERS) - with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: - content_length = resp.headers.get("Content-Length") - size_bytes = None - if content_length is not None: - try: - size_bytes = int(content_length) - except ValueError: - size_bytes = None - - if size_bytes is not None: - # Size is known up front — only pull what we need for the content. - data = resp.read(max_bytes) - else: - # No Content-Length (e.g. gzip decompressive transcoding streams the - # decoded body without a length). Read the full body — bounded by a - # safety cap — so size_bytes and truncation are accurate. This mirrors - # gcloud, which downloads the whole object before reading it. - data = resp.read(FETCH_READ_CAP + 1) - size_bytes = len(data) - - truncated = size_bytes > max_bytes - content = data[:max_bytes].decode("utf-8", errors="replace") - return size_bytes, truncated, content - - -# --------------------------------------------------------------------------- -# Commands (backend-agnostic: identical JSON output either way) -# --------------------------------------------------------------------------- - - -def cmd_list(bucket, prefix, subpath=None): - """List contents of a GCS directory.""" - target = gcs_path(bucket, prefix, subpath) - if not target.endswith("/"): - target += "/" - - if gcloud_available(): - stdout, stderr, rc = run_gcloud(["storage", "ls", target], timeout=30) - if rc != 0: - return { - "success": False, - "error": f"gcloud storage ls failed: {stderr.strip()}", - "path": target, - } - lines = [ln.strip() for ln in stdout.strip().splitlines() if ln.strip()] - else: - try: - lines = _http_list_lines(bucket, prefix, subpath) - except urllib.error.HTTPError as e: - return { - "success": False, - "error": f"GCS API list failed: HTTP {e.code} {e.reason}", - "path": target, - } - except (urllib.error.URLError, OSError, ValueError) as e: - return { - "success": False, - "error": f"GCS API list failed: {e}", - "path": target, - } - - entries = [] - full_prefix = f"gs://{bucket}/{prefix}/" - for line in lines: - # Strip the gs://bucket/ prefix for readability - relative = line.replace(full_prefix, "") if line.startswith(full_prefix) else line - is_dir = line.endswith("/") - entries.append({ - "name": relative.rstrip("/").split("/")[-1] + ("/" if is_dir else ""), - "path": relative.rstrip("/"), - "type": "directory" if is_dir else "file", - "gcs_uri": line.rstrip("/") + ("/" if is_dir else ""), - }) - - return { - "success": True, - "path": target, - "count": len(entries), - "entries": entries, - } - - -def cmd_search(bucket, prefix, pattern, subpath=None): - """Search for files matching a glob pattern under a GCS path.""" - target = gcs_path(bucket, prefix, subpath) - if not target.endswith("/"): - target += "/" - - search_pattern = f"{target}{pattern}" - - if gcloud_available(): - stdout, stderr, rc = run_gcloud(["storage", "ls", search_pattern], timeout=120) - if rc != 0: - # gcloud returns non-zero when no matches found - if "CommandException" in stderr or "One or more URLs matched no objects" in stderr or "matched no objects" in stderr.lower(): - return { - "success": True, - "pattern": search_pattern, - "count": 0, - "matches": [], - } - return { - "success": False, - "error": f"gcloud storage ls failed: {stderr.strip()}", - "pattern": search_pattern, - } - lines = [ln.strip() for ln in stdout.strip().splitlines() if ln.strip()] - else: - try: - lines = _http_search_lines(bucket, prefix, pattern, subpath) - except urllib.error.HTTPError as e: - if e.code == 404: - return { - "success": True, - "pattern": search_pattern, - "count": 0, - "matches": [], - } - return { - "success": False, - "error": f"GCS API search failed: HTTP {e.code} {e.reason}", - "pattern": search_pattern, - } - except (urllib.error.URLError, OSError, ValueError) as e: - return { - "success": False, - "error": f"GCS API search failed: {e}", - "pattern": search_pattern, - } - - matches = [] - full_prefix = f"gs://{bucket}/{prefix}/" - for line in lines: - relative = line.replace(full_prefix, "") if line.startswith(full_prefix) else line - is_dir = line.endswith("/") - matches.append({ - "name": relative.rstrip("/").split("/")[-1] + ("/" if is_dir else ""), - "path": relative.rstrip("/"), - "type": "directory" if is_dir else "file", - "gcs_uri": line, - }) - - return { - "success": True, - "pattern": search_pattern, - "count": len(matches), - "matches": matches, - } - - -def cmd_fetch(bucket, prefix, filepath, max_bytes=DEFAULT_MAX_BYTES): - """Fetch contents of a specific file from GCS.""" - target = gcs_path(bucket, prefix, filepath) - - if gcloud_available(): - # Download to a temp file, then read - import tempfile - - with tempfile.NamedTemporaryFile(delete=False, suffix=".artifact") as tmp: - tmp_path = tmp.name - - try: - _stdout, stderr, rc = run_gcloud( - ["storage", "cp", target, tmp_path, "--no-user-output-enabled"], - timeout=60, - ) - - if rc != 0: - return { - "success": False, - "error": f"gcloud storage cp failed: {stderr.strip()}", - "path": target, - } - - file_size = os.path.getsize(tmp_path) - truncated = file_size > max_bytes - - with open(tmp_path, "r", errors="replace") as f: - content = f.read(max_bytes) - - return { - "success": True, - "path": target, - "size_bytes": file_size, - "truncated": truncated, - "max_bytes": max_bytes, - "content": content, - } - finally: - if os.path.exists(tmp_path): - os.unlink(tmp_path) - else: - try: - size_bytes, truncated, content = _http_fetch(bucket, object_path(prefix, filepath), max_bytes) - except urllib.error.HTTPError as e: - return { - "success": False, - "error": f"GCS download failed: HTTP {e.code} {e.reason}", - "path": target, - } - except (urllib.error.URLError, OSError, ValueError) as e: - return { - "success": False, - "error": f"GCS download failed: {e}", - "path": target, - } - - return { - "success": True, - "path": target, - "size_bytes": size_bytes, - "truncated": truncated, - "max_bytes": max_bytes, - "content": content, - } - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Search and retrieve artifacts from Prow CI job runs in GCS. " - "Supports both the test-platform-results and prow-artifact-archive " - "buckets (detected from the URL). Uses the gcloud CLI when " - "available, otherwise falls back to the public GCS HTTP API (no " - "auth or extra tooling required)." - ), - ) - parser.add_argument( - "prow_url", - help="Prow job URL (https://prow.ci.openshift.org/view/gs/...)", - ) - - subparsers = parser.add_subparsers(dest="command", required=True) - - # list - list_parser = subparsers.add_parser("list", help="List directory contents") - list_parser.add_argument( - "subpath", - nargs="?", - default=None, - help="Subdirectory path relative to the job root (optional)", - ) - - # search - search_parser = subparsers.add_parser("search", help="Search for files matching a glob pattern") - search_parser.add_argument( - "pattern", - help='Glob pattern to match (e.g., "**/*intervals*.json", "**/nodes")', - ) - search_parser.add_argument( - "subpath", - nargs="?", - default=None, - help="Subdirectory to search within (optional, searches from job root by default)", - ) - - # fetch - fetch_parser = subparsers.add_parser("fetch", help="Fetch a specific file's contents") - fetch_parser.add_argument( - "filepath", - help="Path to the file relative to the job root", - ) - fetch_parser.add_argument( - "--max-bytes", - type=int, - default=DEFAULT_MAX_BYTES, - help=f"Maximum bytes to read (default: {DEFAULT_MAX_BYTES})", - ) - - args = parser.parse_args() - - try: - bucket, prefix = parse_prow_url(args.prow_url) - except ValueError as e: - print(json.dumps({"success": False, "error": str(e)})) - sys.exit(1) - - if args.command == "list": - result = cmd_list(bucket, prefix, args.subpath) - elif args.command == "search": - result = cmd_search(bucket, prefix, args.pattern, args.subpath) - elif args.command == "fetch": - result = cmd_fetch(bucket, prefix, args.filepath, args.max_bytes) - else: - print(json.dumps({"success": False, "error": f"Unknown command: {args.command}"})) - sys.exit(1) - - print(json.dumps(result, indent=2)) - if not result.get("success"): - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/plugins/ci/skills/prow-job-analysis/references/aggregated.md b/plugins/ci/skills/prow-job-analysis/references/aggregated.md deleted file mode 100644 index 93490c619..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/aggregated.md +++ /dev/null @@ -1,647 +0,0 @@ -# Aggregated Jobs Reference - -Analyze **aggregated Prow CI jobs** — parent jobs that launch multiple child runs in -parallel and run statistical analysis to detect test regressions. Route here whenever a job -name starts with `aggregated-` or its artifacts contain `release-analysis-aggregator`. - ---- - -## What Are Aggregated Jobs? - -An aggregated job is a **parent orchestrator** that: - -1. Launches N copies of an **underlying (child) job** in parallel (typically 10 runs) -2. Waits for those child runs to complete -3. Collects JUnit XML results from every child run -4. Performs **statistical analysis** to compare pass rates against a historical baseline -5. Reports which tests have regressed with statistical significance - -The parent does **not** install a cluster, run tests, or produce test logs — all testing -happens in the child jobs. - ---- - -## How to Identify an Aggregated Job - -### 1. Job Name Prefix - -The most reliable signal: aggregated job names have an **`aggregated-`** prefix. - -Examples of aggregated job names: -```text -aggregated-aws-sdn-upgrade-4.19 -aggregated-azure-ovn-upgrade-4.19 -aggregated-gcp-ovn-upgrade-4.19 -aggregated-hypershift-ovn-conformance-4.22 -aggregated-metal-ovn-upgrade-4.19 -aggregated-aws-ovn-single-node-upgrade-4.19 -``` - -### 2. Aggregator Container/Step in prowjob.json - -Check `prowjob.json` for an `aggregator` container or step. The ci-operator config for -aggregated jobs references a `release-analysis-aggregator` step. - -### 3. Artifact Structure - -Aggregated jobs have a distinctive artifact layout: -```text -{build-id}/ -├── build-log.txt # ci-operator orchestration log (NOT test output) -├── prowjob.json # Job metadata -└── artifacts/ - ├── release-analysis-aggregator/ - │ └── openshift-release-analysis-aggregator/ - │ └── artifacts/ - │ └── release-analysis-aggregator/ - │ └── {underlying-job-name}/ - │ └── {payload-tag}/ - │ ├── junit-aggregated.xml # THE key artifact - │ └── ... - ├── ci-operator-step-graph.json - └── ci-operator.log -``` - -If you see `release-analysis-aggregator` in the artifact tree, the job is aggregated. - ---- - -## The Critical Mistake: Analyzing the Parent's build-log.txt - -> **⚠️ NEVER analyze the parent job's build-log.txt as if it contains test output.** - -The parent's `build-log.txt` is the ci-operator orchestration log. It contains: -- ci-operator startup and configuration -- Launching the aggregator step -- The aggregator's stdout (summary statistics) -- ci-operator teardown - -It does **NOT** contain test execution output, stack traces, E2E test logs, or cluster -state. - -On a parent failure, your FIRST action is to find the child job URLs from -`junit-aggregated.xml` and analyze those instead. The parent's build-log.txt may show only: -```text -Aggregation step failed: 3 of 10 tests failed statistical analysis -``` -This confirms the aggregation detected regressions but gives no diagnostic data — go to the -child runs. - ---- - -## Parent Job vs Child Job Artifact Structure - -### Parent Job Artifacts (Aggregated Orchestrator) - -```text -{parent-build-id}/ -├── build-log.txt # ci-operator log — NOT test output -├── prowjob.json # Parent job metadata -├── clone-log.txt -└── artifacts/ - ├── release-analysis-aggregator/ - │ └── openshift-release-analysis-aggregator/ - │ └── artifacts/ - │ └── release-analysis-aggregator/ - │ └── {underlying-job-name}/ - │ └── {payload-tag}/ - │ └── junit-aggregated.xml # Statistical results + child URLs - ├── ci-operator-step-graph.json - ├── ci-operator.log - └── junit_operator.xml # Phase pass/fail for the aggregator step itself -``` - -**What the parent has:** Statistical analysis results, links to child runs. -**What the parent lacks:** Test logs, stack traces, cluster state, must-gather, interval files. - -### Child Job Artifacts (Actual Test Run) - -Each child is a normal (non-aggregated) CI job with the full artifact set: - -```text -{child-build-id}/ -├── build-log.txt # Actual test output with logs and errors -├── prowjob.json -└── artifacts/ - ├── {target}/ - │ ├── openshift-e2e-test/ - │ │ ├── build-log.txt # E2E test console output - │ │ └── artifacts/ - │ │ ├── junit/ - │ │ │ ├── junit_e2e_*.xml - │ │ │ └── e2e-timelines_spyglass_*.json # Interval/disruption data - │ │ └── ... - │ ├── gather-extra/ # Cluster state snapshots - │ ├── gather-must-gather/ # Must-gather archive - │ ├── ipi-install-install/ # Installer artifacts - │ └── ... - ├── ci-operator-step-graph.json - └── ci-operator.log -``` - -**What the child has:** Everything — test logs, stack traces, cluster state, must-gather, -interval files, installer logs, pod logs, events. - ---- - -## Finding Child Job URLs from Parent Artifacts - -The `junit-aggregated.xml` is the key artifact — it links to every child run. - -### Locating the junit-aggregated.xml - -```bash -# Download from the parent job's artifacts -gcloud storage cp \ - "gs://test-platform-results/{parent-bucket-path}/artifacts/release-analysis-aggregator/openshift-release-analysis-aggregator/artifacts/release-analysis-aggregator/{underlying-job-name}/{payload-tag}/junit-aggregated.xml" \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled -``` - -If you don't know the `{underlying-job-name}` or `{payload-tag}`, use a wildcard search: - -```bash -# Find all junit-aggregated.xml files in the parent's artifacts -gcloud storage ls \ - "gs://test-platform-results/{parent-bucket-path}/artifacts/release-analysis-aggregator/**/junit-aggregated.xml" -``` - -### Extracting Child Job URLs from junit-aggregated.xml - -Each `` has a `` section with YAML listing passes, failures, and skips, -each with details about the individual child runs. - -Example structure inside ``: - -```yaml -passes: - - jobrunid: "1962527613477982001" - humanurl: "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn/1962527613477982001" - gcsartifacturl: "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn/1962527613477982001" - - jobrunid: "1962527613477982002" - humanurl: "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn/1962527613477982002" - gcsartifacturl: "..." -failures: - - jobrunid: "1962527613477982003" - humanurl: "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-release-main-ci-4.22-e2e-aws-ovn/1962527613477982003" - gcsartifacturl: "..." -skips: [] -``` - -**Key fields:** -- `jobrunid` — The build ID of the child run -- `humanurl` — Prow UI URL for the child run (use this for investigation) -- `gcsartifacturl` — Direct GCS browser link to the child's artifacts - -Investigate each `humanurl` with normal (non-aggregated) analysis workflows. - -### Extracting the Underlying Job Name - -The underlying (child) job name **cannot be reliably derived** from the aggregated job name; -extract it from the artifacts: - -1. **From the junit-aggregated.xml path** — the directory structure contains it: - ```text - artifacts/release-analysis-aggregator/.../release-analysis-aggregator/{underlying-job-name}/{payload-tag}/ - ``` - -2. **From the `humanurl` in ``** — the Prow URL path contains the child job name: - ```text - https://prow.ci.openshift.org/view/gs/test-platform-results/logs/{underlying-job-name}/{jobrunid} - ``` - -Examples of aggregated → underlying name mappings: - -| Aggregated Job | Underlying Job | -|----------------|----------------| -| `aggregated-aws-sdn-upgrade-4.19` | `periodic-ci-openshift-release-main-ci-4.19-e2e-aws-sdn-upgrade` | -| `aggregated-hypershift-ovn-conformance-4.22` | `periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-ovn-conformance` | -| `aggregated-metal-ovn-upgrade-4.19` | `periodic-ci-openshift-release-main-ci-4.19-e2e-metal-ipi-ovn-upgrade` | - -The naming transformation is **not mechanical** — the underlying job name can come from a -different repo (`openshift-hypershift` vs `openshift-release`), use different naming -conventions, and include segments absent from the aggregated name. Always extract from -artifacts. - ---- - -## JUnit Aggregation: How Child Results Appear in Parent Artifacts - -### The junit-aggregated.xml Structure - -The aggregated JUnit XML is not a concatenation of child JUnit files. The release analysis -aggregator: - -1. Collects JUnit results from all child runs -2. For each unique test name, counts passes, failures, and skips across all runs -3. Compares these counts against a historical baseline -4. Generates one `` per test with: - - Pass/fail/skip counts - - Statistical significance determination - - Links to each child run that contributed data - -### Example Testcase Entry - -```xml - - - Regression detected: pass rate 30% (3/10) vs historical 98% - - - passes: - - jobrunid: "1962527613477982001" - humanurl: "https://prow.ci.openshift.org/..." - gcsartifacturl: "..." - - jobrunid: "1962527613477982002" - humanurl: "https://prow.ci.openshift.org/..." - gcsartifacturl: "..." - - jobrunid: "1962527613477982005" - humanurl: "https://prow.ci.openshift.org/..." - gcsartifacturl: "..." - failures: - - jobrunid: "1962527613477982003" - humanurl: "https://prow.ci.openshift.org/..." - gcsartifacturl: "..." - ... - skips: [] - - -``` - -### What the Failure Message Tells You - -The `` element's `message` attribute contains a summary like: -```text -Passed 3 times, failed 7 times, skipped 0 times: we require at least 6 attempts to have -a chance at success with a flake rate of 10.00% and minimum pass rate of 90.00% -``` - -This tells you: -- How many times the test passed/failed/skipped across all child runs -- The statistical threshold being applied -- Whether the failure rate exceeds the acceptable flake rate - ---- - -## Failure Modes in Aggregated Jobs - -Aggregated jobs have three distinct failure modes. **Determine which mode applies before -diving into individual test analysis** — the investigation strategy differs dramatically -between modes. - -### Mode 1: Statistically Significant Test Failure - -The test itself fails frequently enough across runs to be flagged as a regression. - -- **Signal:** A specific test fails in a majority of runs (e.g., 7 of 10). The failure - message in junit-aggregated.xml shows a clear pass/fail ratio that exceeds the flake - threshold. -- **What this means:** A **real regression** — the test consistently fails, so a recent code - change broke the tested functionality. -- **Action:** Investigate the failing test with normal test analysis: - 1. Pick one or more **failed** child runs from the `failures:` list in `` - 2. Navigate to the child run's Prow URL (`humanurl`) - 3. Analyze it as a normal (non-aggregated) job — build-log.txt, JUnit XML, stack traces, - must-gather, interval files - 4. Confirm the root cause via consistent error patterns across multiple failed child runs -- **Root cause:** A product bug or an intentional API/behavior change - -### Mode 2: Insufficient Completed Runs - -Not enough child runs completed successfully to perform the statistical test. For example, -only 5 of 10 jobs produced results because the other 5 failed to install a cluster. - -- **Signal:** Mass test failures across many **unrelated** tests. The failure messages may - reference insufficient data points. You'll see dozens or hundreds of tests failing, all - with messages like: - ```text - Passed 3 times, failed 0 times, skipped 0 times: we require at least 6 attempts to have - a chance at success - ``` - The key indicator: `passed + failed + skipped < total_runs` — results are missing, not - failing. -- **What this means:** The aggregator lacked data points because child runs crashed, timed - out, or failed to install. The mass "failures" are **artifacts of missing data**, not real - regressions. -- **Action:** Investigate why child runs didn't complete — an infrastructure or install - issue, NOT a test issue: - 1. From `junit-aggregated.xml`, identify child runs missing from the results (their - `jobrunid` won't appear in passes, failures, or skips) - 2. Navigate to those child runs via Prow - 3. Check if they failed at installation, hit infrastructure errors, or timed out - 4. The root cause is whatever prevented runs from completing -- **Root cause:** Infrastructure issues, install failures, cloud provider problems, or a - product bug that crashes the cluster before tests can run -- **Common trap:** Don't investigate the individual "failing" tests — they aren't failing. - The aggregator is reporting it lacked data to determine pass/fail. - -### Mode 3: Non-Deterministic Test Presence - -A test only ran in a small subset of completed jobs, even though those jobs completed -successfully. Every test must produce results in every job run; a test that appears in -only some runs is a bug. - -- **Signal:** Failure message says something like: - ```text - Passed 1 times, failed 0 times, skipped 0 times: we require at least 6 attempts to have - a chance at success - ``` - The critical distinction from Mode 2: all 10 child runs completed successfully, but the - test appeared in only 1. Verify all child runs have passing entries for other tests — if - they do, the issue is test-specific, not run-level. -- **What this means:** A test was introduced that doesn't produce results deterministically — - non-deterministic skip logic, or it only runs under conditions that aren't consistently met. -- **Action:** Investigate why the test only runs in some jobs: - 1. Compare the test's presence across child runs - 2. Check the test's skip conditions in the source code - 3. Look for environment-dependent skip logic (platform-specific, feature-gate, or - node-count conditions) -- **Root cause:** Non-deterministic test skip logic, conditional test registration, or a - test that depends on cluster state that varies between runs - -### How to Distinguish Mode 2 from Mode 3 - -Both show tests with fewer results than expected: - -| Characteristic | Mode 2 (Insufficient Runs) | Mode 3 (Non-Deterministic Test) | -|----------------|---------------------------|--------------------------------| -| Affected tests | Many/all tests missing data | One or a few specific tests | -| Child runs completed | Some child runs didn't complete at all | All child runs completed | -| Other tests in same runs | Also missing from incomplete runs | Present and passing | -| Root cause location | Infrastructure/install | Test skip logic | - -**Quick check:** Count how many unique `jobrunid` values appear across ALL testcases. If -some runs are completely absent from all tests, it's Mode 2. If all runs appear in most -tests but are missing from specific tests, it's Mode 3. - ---- - -## Multi-Step CI Jobs vs True Aggregated Jobs - -Don't confuse the two: - -- **Multi-step CI job** — a single job run whose steps (install → test → gather → deprovision) - execute in sequence via the ci-operator step registry, sharing **one cluster** and **one - build-id**. All artifacts live under `{build-id}/artifacts/{target}/`. This is the normal CI - execution model. -- **True aggregated job** — a parent that launches N **independent** child runs, each a - complete multi-step job with its own cluster, build-id, and artifact tree. The parent only - aggregates JUnit results and has no cluster of its own. - -**Why it matters**: for a multi-step failure, everything is under one build-id. For an -aggregated failure, the parent holds only statistical summaries — navigate to child runs for -actual test logs, stack traces, and diagnostics (see -[The Critical Mistake](#the-critical-mistake-analyzing-the-parents-build-logtxt)). - -The child job runs a standard step-registry workflow (pre/test/post); the parent's ci-operator -config is minimal and just configures the aggregator, so investigate the child job's workflow, -not the parent's. For step registry refs, chains, and workflows, see -[ci-infrastructure-changes.md](ci-infrastructure-changes.md#step-registry-deep-dive). - ---- - -## Tracing Failures Across Aggregated Job Boundaries - -### Step-by-Step Investigation Workflow - -1. **Confirm the job is aggregated** - - Check for `aggregated-` prefix in the job name - - Or check for `release-analysis-aggregator` in artifact paths - -2. **Download junit-aggregated.xml** - ```bash - gcloud storage ls \ - "gs://test-platform-results/{bucket-path}/artifacts/release-analysis-aggregator/**/junit-aggregated.xml" - gcloud storage cp .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - ``` - -3. **Count completed runs and classify the failure mode** - - Parse all `` elements - - For each, count unique `jobrunid` values across passes, failures, and skips - - Determine the total number of unique child runs that produced any results - - Compare to expected run count (usually 10) - -4. **Route based on failure mode:** - - **Mode 1 (real regression):** - - Pick 2-3 failed child runs from the `failures:` list - - Analyze each as a normal job using the `ci:prow-job-analysis` skill - - Compare error patterns across runs to confirm consistency - - **Mode 2 (insufficient runs):** - - Identify child runs that are missing from results entirely - - Navigate to those child runs - - Check if they failed at install, timed out, or hit infrastructure issues - - The "failing tests" in the aggregation are red herrings - - **Mode 3 (non-deterministic test):** - - Identify which tests appear in fewer runs than expected - - Check the test source code for skip conditions - - Determine why the test doesn't run deterministically - -5. **Extract the underlying job name** for use in `/payload-aggregate` or `/payload-job` - commands if re-triggering is needed (see "Extracting the Underlying Job Name" above) - -### What to Include in the Analysis Report - -```text -## Aggregated Job Analysis - -- **Parent Job**: {aggregated-job-name} -- **Build ID**: {parent-build-id} -- **Underlying Job**: {underlying-job-name} -- **Payload Tag**: {payload-tag} -- **Failure Mode**: Mode {1|2|3} — {description} - -### Run Summary -- Total child runs: {N} -- Completed: {M} -- Failed to complete: {N-M} - -### Failing Tests (Mode 1) / Missing Runs (Mode 2) / Non-Deterministic Tests (Mode 3) -{details based on failure mode} - -### Child Run Analysis -| Run | Build ID | Status | Key Finding | -|-----|----------|--------|-------------| -| 1 | {id} | Pass | — | -| 2 | {id} | Fail | {error} | -| ... | ... | ... | ... | - -### Root Cause -{synthesized root cause from child run analysis} -``` - ---- - -## Common Aggregated Job Patterns in OpenShift CI - -### Upgrade Aggregated Jobs - -These test OCP upgrades across multiple runs: -```text -aggregated-aws-sdn-upgrade-4.19 -aggregated-azure-ovn-upgrade-4.19 -aggregated-gcp-ovn-upgrade-4.19 -aggregated-metal-ovn-upgrade-4.19 -aggregated-aws-ovn-single-node-upgrade-4.19 -``` - -Each child installs a cluster at one version, upgrades to the target payload, and runs -post-upgrade conformance tests. Failures may be in the install, upgrade, or post-upgrade -test phase. Check JUnit for `install should succeed` and upgrade-specific tests separately; -see [upgrade.md](upgrade.md). - -### HyperShift Aggregated Jobs - -These test HyperShift (hosted control planes): -```text -aggregated-hypershift-ovn-conformance-4.22 -``` - -Child runs create hosted clusters and run conformance tests against them, with more complex -artifact structures (management cluster + hosted cluster) — see [hypershift.md](hypershift.md). - -### Conformance and E2E Aggregated Jobs - -Standard platform conformance testing: -```text -aggregated-aws-ovn-4.19 -aggregated-gcp-ovn-4.19 -``` - -Each child installs and runs the standard E2E suite — the simplest to analyze, following the -standard artifact structure. - ---- - -## Statistical Analysis Considerations - -### When N Child Runs Are Needed - -The aggregator's statistical thresholds: - -- **Minimum attempts:** Typically 6 of 10 runs must complete to have enough data -- **Historical pass rate:** The baseline pass rate for each test from historical data -- **Flake rate threshold:** Usually ~10% — failures below this rate are flakes, not - regressions -- **Minimum pass rate:** The minimum required to not flag a regression (typically 90%) - -If fewer than 6 runs complete, the aggregator **cannot determine** whether a test regressed — -it reports the test as failed due to insufficient data (Mode 2). - -### Interpreting Pass/Fail Ratios - -| Scenario | Interpretation | Action | -|----------|---------------|--------| -| 0 pass / 10 fail | Strong regression | Investigate any single failed child run | -| 3 pass / 7 fail | Likely regression | Investigate failed runs, check if passing runs differ | -| 8 pass / 2 fail | Likely flake | Check if the 2 failures have the same error pattern | -| 10 pass / 0 fail | Not a regression | Test passes consistently | -| 5 pass / 0 fail / 5 missing | Insufficient data (Mode 2) | Investigate why 5 runs didn't complete | - -### Comparing Across Child Runs - -When a test fails in some runs but passes in others: -- **Same error pattern:** Intermittent regression (may depend on timing, load, or race - conditions) -- **Different error patterns:** Multiple unrelated issues, or infrastructure instability -- **Passing runs have no errors:** Confirms the failure is real, not a logging artifact - ---- - -## Special Scenarios - -### "All Children Passed but Parent Failed" - -1. **Parent job timed out** — Child runs were still running when the parent hit its timeout; - they may have eventually passed. - - Check: `prowjob.json` parent `completionTime` vs `startTime`. Duration matching the - timeout confirms a timeout. - - Action: Check child statuses. If they passed, the failure is infrastructure (parent - timeout too short), not a regression. - -2. **Aggregator step itself failed** — The release-analysis-aggregator container crashed - before completing analysis. - - Check: `junit_operator.xml` in the parent's artifacts for step failures. - - Action: CI infrastructure issue, not a test regression. - -3. **Aggregator flagged a test that flaked within passing runs** — A child job can pass - overall even when a test failed and was retried within that run. The aggregator counts each - test's pass/fail across all runs, so a test that flaked in several children can be flagged - even though every child *job* passed. - - Check: The `` message in junit-aggregated.xml for the per-test pass/fail counts. - - Action: A real regression — the test is becoming flaky. - -### "Some Children Failed Differently" - -When child runs fail with different error patterns, identify: - -1. **The real regression:** Which error pattern is new? Check historical data for what's - novel vs. pre-existing flakiness. -2. **Infrastructure vs product failures:** Separate cloud API errors/timeouts from test - assertion errors: - - Infrastructure failures → discard these runs from the analysis - - Test failures → investigate these for regression -3. **Cascading failures:** One root cause can manifest differently across runs (e.g., a - memory leak causing an OOM kill in one run, a timeout in another). Look for a common cause. - -### Investigation Strategy for Mixed Results - -1. **Group by failure pattern** — Categorize each child run by its failure mode -2. **Check the majority pattern** — The most common mode is usually the real signal -3. **Investigate outliers** — Unique failure patterns may reveal secondary issues or - infrastructure problems -4. **Cross-reference timing** — Runs that failed first may show the original error before - cascading effects obscure it - ---- - -## Aggregated Job Retries - -Retries behave differently than for normal jobs (where each retry is an independent -execution): - -- **Retries re-run the aggregation analysis** — they do NOT re-run the underlying child test - jobs -- Child runs from the original attempt are reused -- **Only examine the most recent retry** — previous attempts contain the same underlying - results - -When analyzing an aggregated job that has been retried: -- Set `retries_consistent: only_final_examined` -- Set `retry_summary: "Aggregated job — only final attempt examined (retries re-run aggregation only)"` -- Focus entirely on the latest attempt's `junit-aggregated.xml` - ---- - -## Triggering Aggregated Jobs - -When re-triggering (e.g., on a revert PR): - -- **`/payload-aggregate {underlying-job-name} {count}`** — New aggregated run with N - iterations of the underlying job -- **`/payload-job {underlying-job-name}`** — Single run of the underlying job (faster, no - statistical analysis) - -**Important:** -- Do NOT use `/payload-job` with the aggregated job name — aggregated jobs are not directly - triggerable -- If the aggregated job failed 10/10 runs, a single `/payload-job` run of the underlying job - suffices to validate a fix; otherwise use `/payload-aggregate` for statistical validation -- The `underlying-job-name` **must** be extracted from the artifacts — it cannot be reliably - derived from the aggregated job name - ---- - -## See Also - -- [Artifacts Reference](artifacts.md) — Directory structure for normal (non-aggregated) jobs -- [Test Extension Binaries](test-extension-binaries.md) — component `*-tests-ext` (OTE) binary - failures in a child run -- [HyperShift Reference](hypershift.md) — Additional complexity for HyperShift aggregated jobs -- [Upgrade Reference](upgrade.md) — Upgrade-specific analysis for upgrade aggregated jobs -- [Flaky Test Identification](flaky-test-identification.md) — Distinguishing flakes from - real regressions (relevant for Mode 1 borderline cases) -- [CI Infrastructure](ci-infrastructure-changes.md) — Infrastructure issues that can cause - Mode 2 failures diff --git a/plugins/ci/skills/prow-job-analysis/references/artifacts.md b/plugins/ci/skills/prow-job-analysis/references/artifacts.md deleted file mode 100644 index f51616c38..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/artifacts.md +++ /dev/null @@ -1,1096 +0,0 @@ -# Artifacts Reference - -Directory structure, file contents, and access patterns for Prow CI job artifacts in the -`test-platform-results` GCS bucket. Use it to locate and interpret files when investigating -job failures. - ---- - -## GCS Bucket and Access - -- **Bucket**: `test-platform-results` (publicly accessible, no authentication required) -- **Base URI**: `gs://test-platform-results/{bucket-path}/` -- **Prow UI URL**: `https://prow.ci.openshift.org/view/gs/test-platform-results/{bucket-path}` -- **gcsweb URL**: `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/{bucket-path}` - -**Important**: Prow URLs may show `origin-ci-test` in the path (e.g., -`/view/gs/origin-ci-test/logs/...`), but the actual GCS bucket is always -`test-platform-results`. Always use `gs://test-platform-results/...` for `gcloud storage` -commands. - -### URL Formats - -Both formats are interchangeable: - -```text -# Prow UI -https://prow.ci.openshift.org/view/gs/test-platform-results/{type}/{job-name}/{build-id} - -# gcsweb (direct GCS browser) -https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/{type}/{job-name}/{build-id} -``` - -### Bucket Path by Job Type - -| Job Type | Bucket Path Pattern | Example | -|----------|-------------------|---------| -| **Periodic** | `logs/{job-name}/{build-id}` | `logs/periodic-ci-openshift-release-master-ci-4.21-e2e-aws-ovn/1983307151598161920` | -| **Presubmit** | `pr-logs/pull/{org}_{repo}/{pr-number}/{job-name}/{build-id}` | `pr-logs/pull/openshift_installer/8123/pull-ci-openshift-installer-main-e2e-aws/1962527613477982208` | -| **Postsubmit** | `logs/{job-name}/{build-id}` | `logs/branch-ci-openshift-installer-main-images/1983307151598161920` | - -Note: Periodic and postsubmit jobs share the same `logs/` prefix. Presubmit jobs use -`pr-logs/pull/` and include the org, repo, and PR number in the path. - ---- - -## Top-Level Structure - -Every Prow CI job produces these files at the root of its artifact directory: - -```text -{build-id}/ -├── build-log.txt # ci-operator orchestration log -├── prowjob.json # Job metadata, timing, refs, state -├── clone-log.txt # Git clone output (stdout/stderr) -├── clone-records.json # Structured clone timing data -├── finished.json # Job completion metadata (result, timestamp) -├── started.json # Job start metadata (timestamp, pull refs) -├── podinfo.json # Pod lifecycle, container statuses, events -└── artifacts/ - ├── {target}/ # Main test step artifacts (see below) - ├── ci-operator-step-graph.json # Step dependency graph with timing - ├── ci-operator.log # ci-operator internal log - └── job_labels/ # Machine-detected symptom labels (JSON) -``` - ---- - -## Root-Level Files - -### `build-log.txt` — ci-operator Orchestration Log - -The **primary entry point** for any job investigation: stdout/stderr of the ci-operator -process that orchestrates the multi-stage test. - -**What it contains**: -- ci-operator startup and configuration -- Image resolution and building steps -- Lease acquisition attempts -- Multi-stage test step execution (pre/test/post phases) -- Step pass/fail status with timing -- Final job result - -**Common patterns to search for**: -```text -# Failures -level=error # ci-operator errors -"failed to" # General failure messages -"context deadline exceeded" # Timeout failures -"failed to acquire lease" # Cloud quota lease timeout -"failed to resolve release" # Image resolution failure - -# Step execution -"Running multi-stage test" # Multi-stage test begins -"Step .* succeeded" # Individual step completed -"Step .* failed" # Individual step failed -"Run multi-stage test pre phase" # Phase execution -``` - -**Size**: Typically 10KB–500KB — near the artifact-search script's 512KB default, so pass -`--max-bytes` if the log comes back truncated. - -### `prowjob.json` — Job Metadata - -The serialized ProwJob Kubernetes resource, with full metadata about the job run. - -**Key fields**: - -| JSON Path | Description | -|-----------|-------------| -| `.spec.job` | Full job name (e.g., `periodic-ci-openshift-release-master-ci-4.21-e2e-aws-ovn`) | -| `.spec.type` | Job type: `periodic`, `presubmit`, or `postsubmit` | -| `.spec.refs` | Git refs — org, repo, base_ref, pulls (for presubmit) | -| `.spec.extra_refs` | Additional repo checkouts | -| `.spec.pod_spec.containers[0].args` | ci-operator arguments including `--target=` | -| `.status.state` | Final state: `success`, `failure`, `aborted`, `pending`, `error` | -| `.status.build_id` | Unique build identifier | -| `.status.startTime` | ISO 8601 job start timestamp | -| `.status.pendingTime` | When the job entered pending state | -| `.status.completionTime` | ISO 8601 job completion timestamp | -| `.status.description` | Human-readable status message | -| `.metadata.labels` | Labels including `prow.k8s.io/type`, `prow.k8s.io/job` | -| `.metadata.annotations` | Annotations including release tag information | - -**Extracting key values**: -```bash -# Job name (used for artifact paths) -jq -r '.spec.job' prowjob.json - -# Target name (the ci-operator --target value) -jq -r '.spec.pod_spec.containers[0].args[]' prowjob.json | grep -oP '(?<=--target=)\S+' - -# Job timing -jq '{start: .status.startTime, completion: .status.completionTime, state: .status.state}' prowjob.json - -# PR info (presubmit only) -jq '.spec.refs.pulls[0] | {number, author, sha}' prowjob.json - -# Release payload tag -jq -r '.metadata.annotations["release.openshift.io/tag"] // empty' prowjob.json - -# Upgrade source (upgrade jobs) -jq -r '.metadata.annotations["release.openshift.io/from-tag"] // empty' prowjob.json -``` - -**Important for PR jobs**: The `{target}` (from `--target=`) and the `{JOB_NAME}` (from -`.spec.job`) often differ for presubmit jobs. Use `{JOB_NAME}` **only** for the top-level Prow -bucket path (e.g., `logs/{JOB_NAME}/{build-id}`); keep `artifacts/{target}/...` for step-level -artifacts. Do not substitute one for the other. - -### `podinfo.json` — Pod Lifecycle Details - -Full Kubernetes Pod spec/status and events for the job pod — infrastructure-level detail on -the execution environment. - -**What it contains**: -- Which build cluster node the pod ran on -- Init and main container exit codes and durations -- Image pull times and container startup sequence -- Scheduling events and warnings (OOM kills, evictions, preemptions) -- Container resource limits and requests - -**When to check**: Infrastructure-level issues — why a pod was slow to start, whether -containers crashed or were OOM-killed, or which node hosted the job. - -### `finished.json` — Job Completion Metadata - -Written by Prow when the job completes. Contains: - -| Field | Description | -|-------|-------------| -| `timestamp` | Unix timestamp of job completion | -| `passed` | Boolean — whether the job passed | -| `result` | Result string: `SUCCESS`, `FAILURE`, `ABORTED` | -| `revision` | Git SHA tested | - -### `started.json` — Job Start Metadata - -Written by Prow when the job starts. Contains: - -| Field | Description | -|-------|-------------| -| `timestamp` | Unix timestamp of job start | -| `pull` | PR refs (for presubmit jobs) | -| `repos` | Map of repo → SHA being tested | - -### `clone-log.txt` — Git Clone Output - -Raw stdout/stderr from git clone. Use to debug clone failures or confirm which commits were -checked out. - -### `clone-records.json` — Clone Timing Data - -Structured per-step clone timing. Use to diagnose slow or failed clones. - ---- - -## Artifacts Directory - -### `ci-operator-step-graph.json` — Step Dependency Graph - -Shows all multi-stage test steps, their dependencies, execution order, and timing. Each -step entry includes: -- Step name and phase (pre/test/post) -- Start and completion timestamps -- Dependencies on other steps -- Pass/fail status - -**When to use**: When the phase-level testcases in JUnit XML are absent, use this artifact -to determine which phase a failed step belongs to and the overall execution timeline. - -### `ci-operator.log` — ci-operator Internal Log - -More detailed than `build-log.txt`: image resolution details, step scheduling decisions, and -error details. - -### `job_labels/` — Symptom Labels - -Machine-detected symptom labels attached by the CI system. Each JSON file describes a -detected environmental pattern (e.g., "test failures during high CPU events"). - -These are environmental observations, NOT root causes. Use them as investigative context — -they may explain failures when correlated with other evidence, but root cause analysis is -still required. - -```bash -# List symptom labels -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/job_labels/" 2>/dev/null - -# Download all symptom JSON files (exclude HTML summary) -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/job_labels/*.json" \ - local/job_labels/ --no-user-output-enabled 2>/dev/null || true -``` - ---- - -## Test Step Artifacts (`artifacts/{target}/`) - -The `{target}` directory is named after the ci-operator `--target=` value (e.g., -`e2e-aws-ovn`, `e2e-gcp-ovn-rt-rhcos10-techpreview`). It contains subdirectories for each -multi-stage test step that ran. - -### Step Directory Naming Conventions - -Step directories follow the step-registry naming in `openshift/release`. Common patterns: - -| Step Directory | Purpose | -|---------------|---------| -| `ipi-install-install` | IPI installation step | -| `ipi-install-heterogeneous` | Heterogeneous cluster install | -| `ipi-deprovision-deprovision` | Cloud resource teardown | -| `openshift-e2e-test` | E2E test execution | -| `gather-extra` | Extra diagnostics collection | -| `gather-must-gather` | Must-gather archive collection | -| `gather-audit-logs` | API audit log collection | -| `baremetalds-devscripts-setup` | Metal dev-scripts setup | -| `baremetalds-devscripts-gather` | Metal artifact gathering | -| `ofcir-acquire` | Metal host acquisition (OFCIR) | -| `dump-management-cluster` | HyperShift management dump | - -Each step directory typically contains: -```text -{step-name}/ -├── build-log.txt # Step console output (stdout/stderr) -└── artifacts/ # Step-specific artifacts (varies by step) -``` - ---- - -## E2E Test Artifacts - -### `openshift-e2e-test/` — E2E Test Output - -The primary test step for most OpenShift CI jobs. - -```text -artifacts/{target}/openshift-e2e-test/ -├── build-log.txt # E2E test console output (test runner log) -└── artifacts/ - ├── junit/ - │ ├── junit_e2e_*.xml # E2E test results (JUnit XML) - │ └── e2e-timelines_spyglass_*.json # Disruption interval/timeline data - └── e2e-*.json # Additional test metadata -``` - -### `build-log.txt` (Step-Level) - -Each step has its own `build-log.txt` (stdout/stderr of that step). For `openshift-e2e-test` -it holds the full E2E test runner output: -- Test names as they execute -- Pass/fail results per test -- Stack traces for failed tests -- Timing information - -**Common patterns**: -```text -FAIL # Test failure marker -panic: # Go panic in test code -"Error:" # Assertion errors -"Timed out" # Timeout failures -"STEP:" # Ginkgo test step markers -``` - ---- - -## JUnit XML Files - -Structured test results — the primary source for which tests failed and why. - -### Locations and Naming Patterns - -```bash -# Find all JUnit XML files -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/junit*.xml" -``` - -Common JUnit file patterns: - -| File | Location | Contents | -|------|----------|----------| -| `junit_e2e_*.xml` | `{target}/openshift-e2e-test/artifacts/junit/` | E2E test results | -| `junit_operator.xml` | `artifacts/{target}/` or `artifacts/` | ci-operator step results (phase-level) | -| `junit_install.xml` | `{target}/{install-step}/artifacts/` | Installation test results | -| `junit_metal_setup.xml` | `{target}/ofcir-acquire/artifacts/` | Metal host acquisition results | -| `junit-aggregated.xml` | (see aggregated jobs below) | Aggregated job statistical results | - -### JUnit XML Structure - -```xml - - - - - - - Full error output and stack trace - - - - - Error details - - - - - - -``` - -### ci-operator Phase-Level JUnit (`junit_operator.xml`) - -The ci-operator JUnit includes phase-level testcases that classify each step: - -- `"Run multi-stage test pre phase"` — setup/installation steps -- `"Run multi-stage test test phase"` — functional test steps -- `"Run multi-stage test post phase"` — gather/cleanup steps - -Use these to determine which phase a failed step belongs to. If these entries are absent -(some jobs omit them), fall back to `ci-operator-step-graph.json`. - -### Installation JUnit (`junit_install.xml`) - -Testcases for the installation process. Failed test names follow the pattern -`install should succeed: `. Installation failure stages: - -| Stage | Meaning | Primary Artifacts | -|-------|---------|-------------------| -| `configuration` | Install-config validation failed | Installer log only | -| `infrastructure` | Cloud resource creation failed | Installer log, cloud API errors | -| `cluster bootstrap` | Bootstrap node failed | Log bundle (bootkube, etcd, kube-apiserver) | -| `cluster creation` | Operators failed to deploy | gather-must-gather, operator logs | -| `cluster operator stability` | Operators didn't stabilize | gather-must-gather, operator status | -| `other` | Unknown failure mode | All available logs | - -### Aggregated Job JUnit (`junit-aggregated.xml`) - -Located at: -```text -artifacts/release-analysis-aggregator/openshift-release-analysis-aggregator/artifacts/ - release-analysis-aggregator/{underlying-job-name}/{payload-tag}/junit-aggregated.xml -``` - -Each `` contains `` with YAML-formatted data including `passes:`, -`failures:`, and `skips:` lists. Each entry includes: -- `jobrunid` — build ID of the underlying job run -- `humanurl` — Prow URL for the job run -- `gcsartifacturl` — direct GCS artifact link - ---- - -## Interval / Timeline Files (`e2e-timelines_spyglass_*.json`) - -Structured event data from the E2E test framework — the primary source for disruption -analysis and cluster activity correlation. - -### Location - -```bash -# Find all interval/timeline files -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/e2e-timelines_spyglass_*.json" -``` - -Typical location: -```text -artifacts/{target}/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_{timestamp}.json -``` - -**Upgrade jobs** typically have two timeline files — one per phase (upgrade and conformance). -The first file (sorted by filename) is the upgrade phase; the second is the conformance/e2e -test phase. - -### Format, Event Sources, and Correlation - -The JSON structure, per-source field reference, the full set of event sources (`Disruption`, -`E2ETest`, `OVSVswitchdLog`, `CPUMonitor`, `CloudMetrics`, `EtcdLog`, `EtcdDiskCommitDuration`, -`EtcdDiskWalFsyncDuration`, `AuditLog`, `Alert`, `NodeMonitor`, `MachineMonitor`, -`ClusterVersion`, `ClusterOperator`, `OperatorState`, `KubeletLog`), and the -failure-correlation methodology are documented in -[disruption.md](disruption.md#interval-file-format-and-structure), which owns the interval -format. - ---- - -## Gather-Extra Artifacts - -The `gather-extra` step collects extensive cluster state after test execution — one of the -richest diagnostic sources available. - -```text -artifacts/{target}/gather-extra/ -└── artifacts/ - ├── oc_cmds/ # Cluster state snapshots (oc get output) - ├── pods/ # Pod logs organized by namespace - ├── audit_logs/ # API server audit logs - ├── nodes//journal # Per-node systemd journal (gzip, no .gz extension) - ├── journal_logs/ # Node journal logs (older/alternate layout) - └── must-gather/ # Inline must-gather data (sometimes) -``` - -### `oc_cmds/` — Cluster State Snapshots - -Captured `oc get` output for key cluster resources — one file per command (YAML or text). - -| File | Command Equivalent | Use For | -|------|--------------------|---------| -| `nodes` | `oc get nodes -o yaml` | Node status, conditions, resource capacity | -| `pods` | `oc get pods --all-namespaces` | Pod status across the cluster | -| `events` | `oc get events --all-namespaces` | Kubernetes events (warnings, errors) | -| `clusterversion` | `oc get clusterversion` | OCP version and upgrade status | -| `co` | `oc get clusteroperators` | Cluster operator status (available, degraded) | -| `machines` | `oc get machines -A` | Machine objects and status | -| `machinesets` | `oc get machinesets -A` | MachineSet scaling status | -| `pv` | `oc get pv` | Persistent volume status | -| `csr` | `oc get csr` | Certificate signing requests | - -### `pods/` — Pod Logs by Namespace - -Pod logs organized by namespace. Each namespace directory holds logs for pods running there -at collection time. - -```text -pods/ -├── openshift-etcd/ -│ ├── etcd-master-0/ -│ │ ├── etcd/ -│ │ │ ├── current.log # Current container log -│ │ │ └── previous.log # Previous container log (if restarted) -│ │ └── etcd-readiness/ -│ │ └── current.log -│ └── ... -├── openshift-kube-apiserver/ -│ └── kube-apiserver-master-0/ -│ └── ... -├── openshift-ingress/ -├── openshift-ovn-kubernetes/ -├── openshift-monitoring/ -├── openshift-machine-api/ -└── ... -``` - -**Key namespaces for troubleshooting**: -- `openshift-etcd` — etcd health, leader changes, disk performance -- `openshift-kube-apiserver` — API server errors, request handling -- `openshift-ingress` — Ingress/router issues -- `openshift-ovn-kubernetes` — OVN-Kubernetes networking -- `openshift-monitoring` — Prometheus, alerting issues -- `openshift-machine-api` — Machine lifecycle, scaling -- `openshift-cluster-version` — CVO upgrade progress - -### `audit_logs/` — API Server Audit Logs - -API server audit logs, one JSON audit event per line: -- Request verb, resource, namespace, name -- User/service account making the request -- Response code and status -- Timestamps - -**Use for**: Tracing API request patterns during disruption, identifying request gaps, -correlating API failures with test failures. - -```bash -# Download audit logs -gcloud storage cp -r "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/audit_logs/" \ - local/audit_logs/ --no-user-output-enabled 2>/dev/null || true -``` - -### Node journals — `nodes//journal` (or `journal_logs/`) - -Systemd journal logs from cluster nodes: kernel messages, service logs, and system-level -events. Most jobs place them at `gather-extra/artifacts/nodes//journal`; some use a -flat `journal_logs/` directory. **The per-node `journal` files are gzip-compressed without -a `.gz` extension** — plain `grep` matches nothing; use `zcat`/`zgrep`: - -```bash -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/nodes/" -zgrep -E "Out of memory|Kernel panic" local/nodes/*/journal -``` - -**Key entries to look for**: -- OVS vswitchd stalls: "Unreasonably long poll interval" -- Kernel OOM kills: "Out of memory: Kill process" -- Per-boot kernel and container runtime versions — each boot records the versions in - effect; end-of-run node snapshots (`oc_cmds/nodes`, `nodes.json`) capture only the - final state -- systemd unit lifecycle events (services failing to start, stop, or restart) -- Disk I/O errors -- Network interface events -- kubelet log entries - ---- - -## Must-Gather Archives - -Comprehensive cluster state captured by `oc adm must-gather` — the richest source of cluster -diagnostics. - -### Location Patterns - -Location depends on job type: - -| Pattern | Location | Job Type | -|---------|----------|----------| -| Standard | `{target}/gather-must-gather/artifacts/must-gather.tar` | Most jobs | -| HyperShift Unified | `{target}/dump-management-cluster/artifacts/artifacts.tar` or `.tar.gz` | HyperShift (unified) | -| HyperShift Dump | `{target}/**/artifacts/hypershift-dump.tar` | HyperShift (dual) | -| HyperShift Hosted | `{target}/**/artifacts/**/hostedcluster.tar` | HyperShift (dual) | - -```bash -# Find must-gather archives -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/must-gather*" - -# Find HyperShift dumps -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/hypershift-dump.tar" 2>/dev/null -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/hostedcluster.tar" 2>/dev/null -``` - -### Must-Gather Download & Extraction - -Must-gather is **not** downloaded or extracted automatically — fetch and unpack it manually when a -failure needs cluster-state diagnostics. - -```bash -# 1. Download the archive (path from the Common Artifact Paths / routing table) -mkdir -p .work/prow-job-analysis/{build_id}/must-gather -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-must-gather/artifacts/must-gather.tar" \ - .work/prow-job-analysis/{build_id}/must-gather/ --no-user-output-enabled - -# 2. Extract the outer archive (use `tar -xzf` if it is gzipped / named *.tar.gz) -tar -xf .work/prow-job-analysis/{build_id}/must-gather/must-gather.tar \ - -C .work/prow-job-analysis/{build_id}/must-gather/ - -# 3. Decompress any nested archives (collectors sometimes gzip logs or bundle sub-dumps) -find .work/prow-job-analysis/{build_id}/must-gather/ -name '*.gz' -exec gunzip -f {} + -find .work/prow-job-analysis/{build_id}/must-gather/ -name '*.tar' \ - -exec tar -xf {} -C .work/prow-job-analysis/{build_id}/must-gather/ \; -``` - -The extracted content lands under a long registry-hash directory (the name is preserved, **not** -renamed to `content/`). Point the must-gather-analyzer scripts at that directory — see the -[must-gather plugin](../../../../must-gather/skills/must-gather-analyzer/SKILL.md). - -### Must-Gather Contents (After Extraction) - -After extracting the tar archive, the must-gather directory typically contains: - -```text -must-gather/ -├── {hash-directory}/ # Long registry hash name (preserved as-is after extraction) -│ ├── cluster-scoped-resources/ -│ │ ├── config.openshift.io/ -│ │ ├── machine.openshift.io/ -│ │ ├── operator.openshift.io/ -│ │ └── ... -│ ├── namespaces/ -│ │ ├── openshift-etcd/ -│ │ │ ├── pods/ -│ │ │ ├── core/ -│ │ │ └── ... -│ │ ├── openshift-kube-apiserver/ -│ │ └── ... -│ ├── host_service_logs/ -│ │ └── masters/ -│ │ ├── crio_service.log -│ │ ├── kubelet_service.log -│ │ └── ... -│ └── event-filter.html # HTML event viewer -├── timestamp -└── version -``` - -**Key data in must-gather**: -- **Cluster operators**: Status, conditions, and operator logs -- **Pod YAMLs**: Full pod specs including `containerStatuses` with `exitCode`, - `lastState.terminated.reason`, `restartCount` -- **Container logs**: Current and previous container logs for all pods -- **Events**: Kubernetes events per namespace -- **Node info**: Node conditions, resource usage, taints -- **Operator conditions**: Detailed operator status and degraded reasons - -### HyperShift Must-Gather Patterns - -HyperShift jobs use one of three must-gather patterns: - -1. **Unified Archive** (`dump-management-cluster`) — Single archive with both management - and hosted cluster data. Management at root `output/`, hosted at - `output/hostedcluster-{name}/`. - -2. **Dual Archives** (`gather-must-gather` + hypershift-dump) — Standard must-gather for - management cluster, separate dump for hosted cluster data. - -3. **Standard Only** (`gather-must-gather`) — Standard must-gather only, no - HyperShift-specific dump. - -### Must-Gather Availability - -- Exists only if the cluster was stable enough to run `oc adm must-gather`; no `.tar` means - the cluster was too unstable to collect diagnostics -- Early installation failures (bootstrap, infrastructure) have no must-gather -- Typically collected in the `post` phase (gather steps) - ---- - -## Installer Artifacts - -Produced by the OpenShift installer during cluster creation. - -### Installer Logs - -```bash -# Find installer logs and state file (exclude deprovision — those are from teardown) -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "\.openshift_install(_state\.json|.*\.log)$" | grep -v "deprovision" -``` - -**Location**: Varies by job configuration. Commonly found at: -```text -artifacts/{target}/{install-step}/artifacts/.openshift_install.log -artifacts/{target}/{install-step}/artifacts/.openshift_install_state.json -``` - -**Log format**: Structured text with timestamp, level, and message: -```text -time="2026-01-15T10:23:45Z" level=info msg="Consuming Install Config from target directory" -time="2026-01-15T10:45:12Z" level=error msg="bootstrap failed to complete" -time="2026-01-15T10:45:12Z" level=fatal msg="failed waiting for bootstrapping to complete" -``` - -**Key patterns** (always work backwards from the end): -- `level=error` or `level=fatal` — Error messages (focus on **last** ones, not first) -- `"Still waiting for"` — Components not yet ready at timeout -- `"Cluster operators X, Y, Z are not available"` — Final operator status -- `"context deadline exceeded"` — Installation timeout -- `"terraform"` — Terraform errors (older versions) -- `"clusterapi"` or `"machine-api"` — Cluster API errors (newer versions) - -**Critical analysis principle**: OpenShift installations exhibit **eventual consistency**. -Components report errors while waiting for dependencies — early errors are expected and -usually resolve. Always analyze backwards from the final timeout, not forwards from the start. - -### `install-status.txt` - -Alongside the installer log at `artifacts/{target}/{install-step}/artifacts/install-status.txt` — -the installer's exit code (a single number). `junit_install.xml` translates this into a -human-readable failure mode — prefer it. - -### Installer Log Bundle (`log-bundle-*.tar.gz`) - -```bash -# Find log bundles (metal bundles are gzipped; some cloud bundles are uncompressed .tar) -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "log-bundle.*\.tar(\.gz)?$" -``` - -A tar archive (NOT `.tar.gz`) of detailed node-level diagnostics. Prefer non-deprovision -bundles. - -**Log bundle structure**: -```text -log-bundle-{timestamp}/ -├── bootstrap/ -│ ├── journals/ -│ │ ├── bootkube.log # Bootstrap control plane init -│ │ ├── kubelet.log # Bootstrap kubelet -│ │ ├── crio.log # Container runtime logs -│ │ ├── ironic.log # Ironic logs (metal jobs only) -│ │ ├── metal3-baremetal-operator.log # Metal3 BMO (metal jobs only) -│ │ └── journal.log.gz # Complete system journal -│ └── network/ -│ ├── ip-addr.txt # IP addresses -│ ├── ip-route.txt # Routing table -│ └── hostname.txt # Hostname -├── control-plane/ -│ └── {node-ip}/ -│ └── containers/ -│ ├── metal3-ironic-*.log # Worker provisioning Ironic logs (metal) -│ └── metal3-baremetal-operator-*.log -├── serial/ # Serial console logs -│ ├── {cluster}-bootstrap-serial.log -│ └── {cluster}-master-N-serial.log -├── clusterapi/ -│ ├── *.yaml # Cluster API resources -│ ├── etcd.log # etcd logs -│ └── kube-apiserver.log # API server logs -├── failed-units.txt # Failed systemd units -└── gather.log # Log bundle collection log -``` - -**Analysis by failure mode**: -- **Bootstrap failures**: Check `bootstrap/journals/bootkube.log`, `clusterapi/etcd.log`, - `clusterapi/kube-apiserver.log`, `serial/*-bootstrap-serial.log` -- **Infrastructure failures**: Focus on installer log — cloud API errors, quota, rate limiting -- **Cluster creation**: Check must-gather operator logs -- **Operator stability**: Check must-gather operator conditions and logs - -### `metadata.json` — Cluster Metadata - -Cluster name, ID, infrastructure platform, and region. In the install step artifacts -directory. - ---- - -## Serial Console Logs - -Raw console output of VMs or bare metal nodes, as if watching a physical console. - -### Location - -In log bundles: -```text -log-bundle-{timestamp}/serial/{cluster}-bootstrap-serial.log -log-bundle-{timestamp}/serial/{cluster}-master-N-serial.log -``` - -For metal jobs (libvirt console logs): -```text -artifacts/{target}/baremetalds-devscripts-gather/artifacts/libvirt-logs.tar -``` -Extract to find `{cluster}-bootstrap_console.log`, `{cluster}-master-{N}_console.log`. - -### What They Reveal - -- **Kernel panics**: `panic`, `kernel`, `oops` -- **Ignition failures**: `ignition`, `config fetch failed`, `Ignition failed` -- **Hardware/disk issues**: `mount`, `disk`, `filesystem`, `I/O error` -- **Network configuration**: `dhcp`, `network unreachable`, `DNS`, `timeout` -- **Boot sequence**: Kernel messages, initramfs, CoreOS startup -- **Service failures**: systemd errors, unit failures - ---- - -## Metal (Bare Metal) Job Artifacts - -Metal IPI jobs use dev-scripts with Metal3 and Ironic, producing artifacts not found in cloud -jobs. - -### OFCIR Acquisition - -```text -artifacts/{target}/ofcir-acquire/ -├── build-log.txt # Host acquisition log (JSON with pool, provider, host) -└── artifacts/ - └── junit_metal_setup.xml # JUnit: "[sig-metal] should get working host from infra provider" -``` - -If OFCIR acquisition fails, the installation never starts. Check the JUnit for the test -`[sig-metal] should get working host from infra provider`. - -### Dev-Scripts Logs - -```text -artifacts/{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/ -├── 01_*.log # Requirements and host config -├── 02_*.log # Network setup -├── 03_*.log # Ironic/Metal3 setup -├── 04_*.log # Installer build -├── 05_*.log # Pre-install configuration -├── 06_create_cluster.log # Cluster installation (invokes installer) -├── .openshift_install*.log # Installer logs (dev-scripts invokes the installer) -└── ... -``` - -Dev-scripts invokes the installer, so `.openshift_install*.log` files appear in the devscripts -directories. - -- Failures in steps 01–05 are **dev-scripts setup failures** (host config, Ironic/Metal3 setup) -- Failures in step 06+ are **cluster installation failures** (also analyzed by the standard - install failure analysis) - -### Libvirt Console Logs - -```text -artifacts/{target}/baremetalds-devscripts-gather/artifacts/libvirt-logs.tar -``` - -Extract to get VM/node console logs showing the complete boot sequence. - -### Ironic Logs (in Log Bundle) - -The log bundle contains two sets of Ironic logs. Check by what failed: masters → bootstrap -Ironic; workers → control-plane Ironic. - -- **Bootstrap Ironic** (`bootstrap/journals/ironic.log`) — Master node provisioning -- **Control-plane Ironic** (`control-plane/{node-ip}/containers/metal3-ironic-*.log`) — Worker - node provisioning - -### sosreport - -```text -artifacts/{target}/baremetalds-devscripts-gather/artifacts/sosreport-*.tar.xz -``` - -Hypervisor system diagnostics: system logs, libvirt configuration, and diagnostic command -output. Useful for hypervisor-level issues. - -### Squid Proxy Logs - -```text -artifacts/{target}/baremetalds-devscripts-gather/artifacts/squid-logs-*.tar -``` - -The squid proxy runs on the hypervisor. Logs show **inbound** CI access to the cluster -(CI → cluster), NOT outbound access (cluster → registry). Important for debugging CI -connectivity to the cluster in IPv6/disconnected environments. - ---- - -## Cluster Event Artifacts - -### Kubernetes Events (`oc_cmds/events`) - -Cluster-wide Kubernetes events captured at gather time — warnings, errors, and informational -events across all namespaces. - -### etcd Events and Logs - -Available in multiple locations: -- **Timeline files**: `EtcdLog`, `EtcdDiskCommitDuration`, `EtcdDiskWalFsyncDuration` sources -- **Pod logs**: `gather-extra/artifacts/pods/openshift-etcd/` -- **Log bundle**: `clusterapi/etcd.log` - -Key etcd indicators: -- `"apply request took too long"` — write pressure -- `"slow fdatasync"` — disk I/O bottleneck -- `"waiting for ReadIndex response took too long"` — read latency -- Leader election events — cluster instability -- Commit duration above 25ms or WAL fsync above 10ms thresholds - ---- - -## Monitoring and Metrics Artifacts - -### CloudMetrics (in Timeline Files) - -Azure disk metrics under `source: "CloudMetrics"`: -- Disk IOPS (read/write) -- Queue depth -- Bandwidth -- Latency - -### CPUMonitor (in Timeline Files) - -Node CPU utilization above 95%, under `source: "CPUMonitor"`. - -### Prometheus Alerts (in Timeline Files) - -Firing alerts under `source: "Alert"`. Common critical alerts: -- `ExtremelyHighIndividualControlPlaneCPU` -- `etcdHighCommitDurations` -- `etcdHighNumberOfFailedGRPCRequests` - -### Monitoring Stack Logs - -Monitoring pod logs (if available): -```text -gather-extra/artifacts/pods/openshift-monitoring/ -``` - ---- - -## Cluster State Artifacts - -### Operator Status - -Cluster operator status is available in several places: -- `gather-extra/artifacts/oc_cmds/co` — `oc get clusteroperators` output -- Must-gather `cluster-scoped-resources/config.openshift.io/clusteroperators/` — Full YAML -- Timeline files `source: "ClusterOperator"` — Status transitions over time - -### Node Status - -- `gather-extra/artifacts/oc_cmds/nodes` — `oc get nodes -o yaml` -- Timeline files `source: "NodeMonitor"` — Node condition changes -- Must-gather `cluster-scoped-resources/core/nodes/` — Full node YAML - -### Machine Info - -- `gather-extra/artifacts/oc_cmds/machines` — Machine objects -- `gather-extra/artifacts/oc_cmds/machinesets` — MachineSet status -- Timeline files `source: "MachineMonitor"` — Machine phase changes - ---- - -## Multi-Step Job Navigation - -### Understanding Step Directories - -Multi-stage tests run steps in three phases — **pre** (setup/install), **test** (functional -tests), **post** (gather/cleanup) — each in its own directory under `artifacts/{target}/`. - -To determine which phase a step belongs to: -1. Check `junit_operator.xml` for phase-level testcases -2. Fall back to `ci-operator-step-graph.json` for step dependencies and timing -3. Use naming conventions (e.g., `ipi-install-*` → pre, `gather-*` → post) - -### Navigating Multiple Steps - -```bash -# List all step directories -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/" - -# List artifacts for a specific step -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/{step-name}/" -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/{step-name}/artifacts/" -``` - -### Step-Level Build Logs - -For a failed step, download its `build-log.txt`: - -```bash -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/{step-name}/build-log.txt" \ - local/{step-name}-build-log.txt --no-user-output-enabled -``` - ---- - -## Upgrade Job Artifacts - -Upgrade jobs produce artifacts under **multiple workflow step directories**, one per phase. -Key differences from non-upgrade jobs: - -### Multiple Timeline Files - -Upgrade jobs typically produce two timeline files (one per phase): -```bash -gcloud storage ls "gs://test-platform-results/logs/{job_name}/{build_id}/artifacts/**/e2e-timelines_spyglass_*.json" -``` - -The first file (sorted by filename) is the **upgrade phase**; the second is the -**conformance/e2e test phase**. - -### Upgrade-Specific Cluster State - -```bash -# Cluster version (shows upgrade progress) -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/clusterversion" \ - local/clusterversion --no-user-output-enabled - -# Cluster operators (shows operator status post-upgrade) -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/co" \ - local/co --no-user-output-enabled -``` - -### Upgrade Source Information - -Extract upgrade source from `prowjob.json`: -```bash -# Upgrade source tag -jq -r '.metadata.annotations["release.openshift.io/from-tag"] // empty' prowjob.json - -# Release images -jq -r '.spec.pod_spec.containers[0].env[] | select(.name == "RELEASE_IMAGE_INITIAL") | .value' prowjob.json -jq -r '.spec.pod_spec.containers[0].env[] | select(.name == "RELEASE_IMAGE_LATEST") | .value' prowjob.json -``` - ---- - -## Artifact Discovery and Search - -### Using `gcloud storage ls` - -The most reliable way to find artifacts: - -```bash -# List top-level contents -gcloud storage ls "gs://test-platform-results/{bucket-path}/" - -# List step directories -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/" - -# Recursive search with glob -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/junit*.xml" -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/e2e-timelines_spyglass_*.json" -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/must-gather*" -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/*.tar" - -# Recursive listing (pipe to grep for filtering) -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "\.openshift_install.*\.log$" | grep -v "deprovision" -``` - -### Using the Artifact Search Script - -This skill bundles a Python script (`prow_job_artifact_search.py`) for structured artifact access -with JSON output. Default fetch limit is 512KB; use `--max-bytes` for larger files. - -```bash -# List directory contents -python3 plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py list [subpath] - -# Search for files by glob pattern -python3 plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py search "" [subpath] - -# Fetch a specific file (default 512KB limit) -python3 plugins/ci/skills/prow-job-analysis/prow_job_artifact_search.py fetch [--max-bytes N] -``` - -### Common Search Patterns - -```bash -# Find all interval/disruption data -python3 .../prow_job_artifact_search.py search "**/e2e-timelines_spyglass_*.json" -python3 .../prow_job_artifact_search.py search "**/*intervals*.json" - -# Find JUnit results -python3 .../prow_job_artifact_search.py search "**/junit*.xml" - -# Find must-gather archives -python3 .../prow_job_artifact_search.py search "**/must-gather*" - -# Find node journal logs for a specific node -python3 .../prow_job_artifact_search.py search "**/*worker-c-7t6ng*" artifacts - -# Find specific oc_cmds output -python3 .../prow_job_artifact_search.py search "**/nodes" artifacts -``` - ---- - -## Quick Reference: Artifact Location Summary - -### By Investigation Type - -| What You Need | Where to Look | -|---------------|--------------| -| Job pass/fail and timing | `prowjob.json` | -| ci-operator log | `build-log.txt` (top-level) | -| Test results (which tests failed) | `artifacts/**/junit*.xml` | -| Test console output | `artifacts/{target}/openshift-e2e-test/build-log.txt` | -| Disruption data | `artifacts/**/e2e-timelines_spyglass_*.json` | -| Cluster operator status | `gather-extra/artifacts/oc_cmds/co` | -| Pod status and logs | `gather-extra/artifacts/pods/{namespace}/` | -| API audit logs | `gather-extra/artifacts/audit_logs/` | -| Node journal logs | `gather-extra/artifacts/nodes//journal` (gzip, no extension — use zcat) or `journal_logs/` | -| Cluster events | `gather-extra/artifacts/oc_cmds/events` | -| Must-gather (full cluster state) | `gather-must-gather/artifacts/must-gather.tar` | -| Installer log | `{install-step}/artifacts/.openshift_install.log` | -| Installer log bundle | `{install-step}/artifacts/log-bundle-*.tar` | -| Serial console (VM boot) | `log-bundle-*/serial/` or `libvirt-logs.tar` | -| Step dependency graph | `artifacts/ci-operator-step-graph.json` | -| Symptom labels | `artifacts/job_labels/*.json` | -| Pod lifecycle (job pod) | `podinfo.json` | - -### By Failure Type - -| Failure Type | Primary Artifacts | Secondary Artifacts | -|-------------|-------------------|---------------------| -| **Installation** | Installer log, `junit_install.xml`, log bundle | Must-gather, serial console | -| **E2E test** | JUnit XML, E2E build-log, timeline files | Gather-extra, must-gather | -| **Disruption** | Timeline files, audit logs, etcd logs | Pod logs, journal logs | -| **Upgrade** | ClusterVersion, timeline files, operator status | Gather-extra, must-gather | -| **Infrastructure** | Installer log, cloud API errors | `prowjob.json` timing | -| **CI infrastructure** | `build-log.txt`, `podinfo.json` | `ci-operator-step-graph.json` | -| **Metal install** | Dev-scripts logs, console logs, Ironic logs | sosreport, squid logs | -| **Resource exhaustion** | Timeline files (CPU, cloud metrics), node status | Pod logs, events | -| **Networking** | OVN pod logs, journal logs, timeline files | Audit logs, events | diff --git a/plugins/ci/skills/prow-job-analysis/references/ci-infrastructure-changes.md b/plugins/ci/skills/prow-job-analysis/references/ci-infrastructure-changes.md deleted file mode 100644 index a833eede4..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/ci-infrastructure-changes.md +++ /dev/null @@ -1,1139 +0,0 @@ -# CI Infrastructure Changes Reference - -How CI infrastructure changes cause job failures, and how to distinguish "the product broke" -from "someone changed the CI configuration." Covers the layers between a developer's code -change and the test result. - ---- - -## Table of Contents - -1. [When to Use This Reference](#when-to-use-this-reference) -2. [The `openshift/release` Repo](#the-openshiftrelease-repo) -3. [ci-operator — The Test Orchestrator](#ci-operator--the-test-orchestrator) -4. [Step Registry Deep Dive](#step-registry-deep-dive) -5. [Distinguishing CI Infrastructure vs Product Failures](#distinguishing-ci-infrastructure-vs-product-failures) -6. [How to Check for Recent CI Infrastructure Changes](#how-to-check-for-recent-ci-infrastructure-changes) -7. [Image Promotion and Registry Interactions](#image-promotion-and-registry-interactions) -8. [CI Job Lifecycle Phases](#ci-job-lifecycle-phases) -9. [Detecting CI Infrastructure Failures — Practical Steps](#detecting-ci-infrastructure-failures--practical-steps) -10. [Environment Variables and Overrides](#environment-variables-and-overrides) -11. [Artifact Directory Structure Reference](#artifact-directory-structure-reference) -12. [See Also](#see-also) - ---- - -## When to Use This Reference - -- Job fails before any product code runs (lease acquisition, image build, etc.) -- ci-operator itself crashes or reports internal errors -- Cluster lease or cloud quota issues prevent the job from starting -- Test infrastructure components (Prow, ci-operator, step registry) malfunction -- A new failure correlates with a recent `openshift/release` PR rather than a product PR -- The same failure appears across multiple unrelated repos or jobs simultaneously -- A job that was passing starts failing with no product code changes - ---- - -## The `openshift/release` Repo - -[`openshift/release`](https://github.com/openshift/release) is the single source of truth for -OpenShift CI job configuration. Every CI job — presubmit, postsubmit, periodic — is defined or -generated from config files here. Changes can cause widespread job failures with no product -code changes. - -### Repo Structure - -```text -openshift/release/ -├── ci-operator/ -│ ├── config/ # Per-repo CI configuration -│ │ └── openshift/ # Organization level -│ │ └── installer/ # Repository level -│ │ ├── openshift-installer-master.yaml -│ │ ├── openshift-installer-release-4.18.yaml -│ │ └── ... -│ ├── jobs/ # Generated Prow job definitions (do NOT edit directly) -│ │ ├── openshift/ -│ │ │ └── installer/ -│ │ │ ├── openshift-installer-master-presubmits.yaml -│ │ │ ├── openshift-installer-master-postsubmits.yaml -│ │ │ └── ... -│ │ └── infra-periodics.yaml # Infrastructure periodic jobs -│ └── step-registry/ # Shared test step definitions -│ ├── ipi/ -│ │ └── install/ -│ │ ├── ipi-install-workflow.yaml -│ │ ├── install/ -│ │ │ ├── ipi-install-install-ref.yaml -│ │ │ └── ipi-install-install-commands.sh -│ │ └── rbac/ -│ │ ├── ipi-install-rbac-ref.yaml -│ │ └── ipi-install-rbac-commands.sh -│ ├── gather/ -│ │ ├── must-gather/ -│ │ ├── extra/ -│ │ └── audit-logs/ -│ ├── hypershift/ -│ │ └── ... -│ └── ... -├── cluster/ # Build cluster configuration -│ └── test-deploy/ # CI infrastructure deployments -├── core-services/ # Core CI service configs -│ ├── prow/ # Prow controller configs -│ └── release-controller/ # Release controller configs -└── hack/ # Helper scripts (generators, validators) - └── generators/ # Job config generators -``` - -### ci-operator Config Files - -**Location**: `ci-operator/config///--.yaml` - -These YAML files define how ci-operator builds, tests, and promotes images for a specific repo -branch. Each file contains: - -```yaml -# Example: ci-operator/config/openshift/installer/openshift-installer-master.yaml -base_images: - base: - name: "4.18" - namespace: ocp - tag: base -build_root: - image_stream_tag: - name: release - namespace: openshift - tag: golang-1.22 -images: - - from: base - to: installer - dockerfile_path: Dockerfile.ci -releases: - initial: - integration: - name: "4.18" - namespace: ocp - latest: - integration: - name: "4.18" - namespace: ocp -tests: - - as: e2e-aws-ovn # Test name (becomes the --target) - steps: - cluster_profile: aws # Cloud credentials and config - workflow: openshift-e2e-aws - - as: e2e-gcp - steps: - cluster_profile: gcp - workflow: openshift-e2e-gcp - env: - COMPUTE_NODE_TYPE: n2-standard-4 -promotion: - to: - - name: "4.18" - namespace: ocp -``` - -**Key fields and their impact on job behavior**: - -| Field | Purpose | Failure Impact | -|-------|---------|----------------| -| `base_images` | Images pulled as build inputs | Image pull failures if tag is removed or registry is down | -| `build_root` | Build environment image | Build failures if root image is unavailable | -| `images` | Images built from source | Build errors, Dockerfile changes | -| `releases` | Release payloads to use for testing | `creating_release_images` failures if payload tag doesn't exist | -| `tests[].steps.workflow` | Which step registry workflow to run | Workflow changes affect test execution | -| `tests[].steps.cluster_profile` | Cloud credentials and cluster config | Cloud authentication and quota issues | -| `tests[].steps.env` | Environment variable overrides | Behavior changes from env var modifications | -| `promotion` | Where built images are pushed | Promotion failures affect downstream consumers | - -**Validation rule for ref steps in config files**: In config files (under -`ci-operator/config/`), ref steps cannot have sibling properties like `timeout`, -`best_effort`, `as`, `commands`, `from`, `from_image`, or `resources`. The validator treats -these as literal test step definitions requiring all mandatory fields. This restriction does -NOT apply to workflow files under `ci-operator/step-registry/`, where `timeout` and -`best_effort` on ref steps are valid. For custom timeouts in config files, use the `TIMEOUT` -env var in the job's `env:` section. - -### Job Definitions (Generated) - -**Location**: `ci-operator/jobs///` - -Job definition files are **generated** by `make jobs` from the ci-operator config files; never -edit them directly. The generated files define the Prow job with: - -```yaml -# Example structure (generated, do not edit) -presubmits: - openshift/installer: - - name: pull-ci-openshift-installer-master-e2e-aws-ovn - agent: kubernetes - decorate: true - cluster: build01 - spec: - containers: - - image: ci-operator:latest - command: - - ci-operator - args: - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --target=e2e-aws-ovn -``` - -**Job naming convention**: -- Presubmits: `pull-ci----` -- Postsubmits: `branch-ci----` -- Periodics: `periodic-ci----` - - Release periodics: `periodic-ci-openshift-release-master-ci--` - - Nightly periodics: `periodic-ci-openshift-release-master-nightly--` - -**Job types and their execution model**: - -| Job Type | Trigger | Execution Type ID | Purpose | -|----------|---------|-------------------|---------| -| Presubmit | PR creation/update, `/test` comment | 3 | Validate PR changes | -| Postsubmit | Merge to branch | 2 | Post-merge validation, image promotion | -| Periodic | Cron schedule | 1 | Continuous validation, nightly testing | - -### The Step Registry - -A library of reusable test steps in `ci-operator/step-registry/`, using a hierarchical -composition model: - -```text -Workflow (top-level orchestration) - └── defines pre/test/post phases - ├── Phase: pre (setup/installation) - │ ├── Chain (group of steps) - │ │ ├── Ref (single step) → commands.sh - │ │ └── Ref (single step) → commands.sh - │ └── Ref (single step) → commands.sh - ├── Phase: test (actual testing) - │ └── Ref (single step) → commands.sh - └── Phase: post (teardown/gathering) - ├── Chain: gather - │ ├── Ref: gather-must-gather → commands.sh - │ ├── Ref: gather-extra → commands.sh - │ └── Ref: gather-audit-logs → commands.sh - └── Ref: deprovision → commands.sh -``` - -#### Step Types - -**Workflows** (`*-workflow.yaml`) — Top-level orchestration defining the three phases: - -```yaml -# ci-operator/step-registry/openshift/e2e/aws/openshift-e2e-aws-workflow.yaml -workflow: - as: openshift-e2e-aws - steps: - pre: - - chain: ipi-aws-pre - test: - - ref: openshift-e2e-test - post: - - chain: ipi-aws-post - documentation: |- - Run the OpenShift end-to-end test suite on AWS. -``` - -**Chains** (`*-chain.yaml`) — Ordered groups of refs or other chains: - -```yaml -# ci-operator/step-registry/ipi/aws/pre/ipi-aws-pre-chain.yaml -chain: - as: ipi-aws-pre - steps: - - ref: ipi-install-rbac - - ref: ipi-conf - - ref: ipi-conf-aws - - chain: ipi-install -``` - -**Refs** (`*-ref.yaml`) — Individual executable steps with an associated shell script: - -```yaml -# ci-operator/step-registry/ipi/install/install/ipi-install-install-ref.yaml -ref: - as: ipi-install-install - from: installer - commands: ipi-install-install-commands.sh - resources: - requests: - cpu: 1000m - memory: 2Gi - timeout: 30m - grace_period: 30m - env: - - name: OPENSHIFT_INSTALL_INVOKER - default: openshift-internal-ci/ci-operator - documentation: |- - Runs the openshift-install create cluster command. -``` - -**Commands** (`*-commands.sh`) — The shell script executed by a ref step. Runs inside a -container built from the image specified by the ref's `from` field. - -#### Step Resolution Order - -When ci-operator resolves a workflow: - -1. Look up the workflow definition from the step registry -2. For each phase (pre, test, post), resolve the listed chains and refs -3. Chains are recursively expanded into their constituent refs -4. Each ref maps to a commands.sh script and a container image -5. The final execution plan is a flat list of steps per phase - -#### Key Step Registry Paths - -| Path | Purpose | -|------|---------| -| `ci-operator/step-registry/ipi/install/` | IPI cluster installation steps | -| `ci-operator/step-registry/ipi/conf/` | IPI configuration steps (install-config) | -| `ci-operator/step-registry/ipi/deprovision/` | IPI cluster teardown | -| `ci-operator/step-registry/openshift/e2e/` | E2E test execution | -| `ci-operator/step-registry/gather/` | Artifact gathering (must-gather, extra, audit) | -| `ci-operator/step-registry/hypershift/` | HyperShift-specific steps | -| `ci-operator/step-registry/cucushift/` | Extended platform test steps | -| `ci-operator/step-registry/upi/` | UPI installation steps | - -### Cluster Profiles - -**Location**: `cluster/test-deploy//` or defined in Vault - -Cluster profiles provide cloud credentials, SSH keys, pull secrets, and platform-specific -config for test jobs. Specified in the ci-operator config: - -```yaml -tests: - - as: e2e-aws - steps: - cluster_profile: aws # References the "aws" cluster profile - workflow: openshift-e2e-aws -``` - -**Common cluster profiles and what they provide**: - -| Profile | Platform | Contents | -|---------|----------|----------| -| `aws` | AWS | AWS credentials, region config, instance types | -| `aws-2` | AWS | Alternate AWS account (different quotas) | -| `gcp` | GCP | GCP service account, project config | -| `azure4` | Azure | Azure credentials, subscription config | -| `metal` | Bare Metal | IPMI credentials, network config | -| `packet` | Equinix Metal | API token, project config | -| `vsphere` | vSphere | vCenter credentials, datacenter config | -| `ibmcloud` | IBM Cloud | API key, resource group config | - -**How cluster profiles affect job behavior**: -- Different profiles may have different cloud quotas available -- Region or zone selection within a profile affects resource availability -- Profile changes (e.g., rotating credentials) can cause authentication failures -- Adding/removing profiles requires coordinated changes across config and Vault - -### Templates (Legacy vs Step Registry) - -The step registry is the modern approach. Legacy templates predate it and defined entire test -workflows in a single OpenShift template. Some jobs still use templates, identifiable by a -`--template` argument in the ci-operator invocation instead of `--target` with multi-stage -steps. Template-based jobs are being migrated to the step registry. - -### Leases and Resource Management - -Two systems lease scarce infrastructure to CI jobs; both emit `failed to acquire lease...` -in the top-level `build-log.txt`, and both are CI infrastructure problems — never product bugs: - -- **Boskos** brokers cloud-account / quota "slices" for cloud jobs (aws/gcp/azure). -- **OFCIR** (OpenShift Fleeting CI Resources) leases pre-provisioned hosts for bare-metal and - Equinix (`metal`/`packet`) jobs. - -```text -failed to acquire lease: context deadline exceeded -``` - -```text -failed to acquire lease for cloud quota slice "aws-quota-slice": all resources are in use -``` - -**Common lease failure patterns**: -- `context deadline exceeded` — no lease became available within the timeout -- `all resources are in use` — every account/slice for that platform is occupied -- `failed to release lease` — cleanup failure (does not affect test results) - -For the per-system CRDs and checks (Boskos slices, OFCIR `cip`/`cir`, Hive `ClusterClaim`), see -[Cloud Provider Errors](cloud-provider-errors.md#1-ci-resource-acquisition-job-never-provisions). - ---- - -## ci-operator — The Test Orchestrator - -`ci-operator` orchestrates CI test execution. It runs as a pod in the build cluster and -manages the entire test lifecycle. - -### What ci-operator Does - -1. **Source cloning** — Clones the repo under test (and any additional repos) -2. **Image building** — Builds container images from Dockerfiles in the repo -3. **Release image creation** — Assembles or imports release payload images -4. **Multi-stage test execution** — Runs the workflow's pre/test/post steps -5. **Artifact collection** — Gathers logs, JUnit results, and other artifacts to GCS -6. **Image promotion** — Pushes built images to the CI registry (postsubmit only) - -### ci-operator Arguments - -The most important ci-operator arguments (visible in `prowjob.json`): - -```bash -ci-operator \ - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson \ - --lease-server-credentials-file=/etc/boskos/credentials \ - --target=e2e-aws-ovn \ # Which test to run (maps to tests[].as in config) - --variant= # Config variant (if any) - --resolver-address=... # Step registry resolver endpoint - --gcs-upload-secret=... # GCS upload credentials -``` - -The `--target` argument identifies the test definition in the ci-operator config file. Use the -**job name** from `.spec.job` in `prowjob.json` only for the top-level Prow bucket path; the -step artifacts live under `artifacts/{target}/`, keyed by the `--target` value. The job name -and target can differ for PR jobs, so do not substitute one for the other. - -### Image Building and Promotion - -ci-operator builds images in this order: - -1. **Pull base images** — from `base_images` in config (e.g., `ocp/4.18:base`) -2. **Build root image** — the build environment (Go compiler, tools) -3. **Build source image** — clone the repo into the build root -4. **Build pipeline images** — intermediate build stages -5. **Build output images** — final images defined in `images` config section -6. **Assemble release payload** — create or import the release image (for `releases` config) - -**Image promotion** (postsubmit only): -- After tests pass, built images are pushed to the CI image registry -- The `promotion` config section defines the target image stream -- Promoted images become available for other jobs and release payloads -- Promotion failures do not affect test results but block image availability - -### Release Image Creation and Injection - -The `releases` section defines which release payloads to use: - -```yaml -releases: - initial: # The "from" version (for upgrade tests) - integration: - name: "4.17" - namespace: ocp - latest: # The "to" version (what we're testing) - integration: - name: "4.18" - namespace: ocp -``` - -ci-operator creates release images by: -1. Importing the base release payload from the image stream -2. Replacing images that were built from source in this job -3. Creating a custom release payload with the test changes included - -**Common `creating_release_images` failure**: -```text -error creating release images: failed to import release ... -``` -This occurs when: -- The referenced image stream tag doesn't exist -- The CI registry is experiencing issues -- The release payload import times out -- Network issues between build cluster and registry - -### Multi-Stage Test Execution Model - -ci-operator executes multi-stage tests in three phases: - -```text -PRE phase (setup) → Runs sequentially; failure aborts remaining pre steps AND skips test phase - Post phase still runs for cleanup -TEST phase (testing) → Runs sequentially; failure records test failure - Post phase still runs for cleanup -POST phase (cleanup) → Runs sequentially; failures are recorded but don't change overall result - Always runs, even if pre or test phases failed -``` - -**Step failure propagation rules**: -1. If a **pre** step fails → remaining pre steps are skipped, test phase is skipped, - post phase runs for cleanup -2. If a **test** step fails → remaining test steps may be skipped (depends on config), - post phase runs for cleanup -3. If a **post** step fails → subsequent post steps continue running, the step failure - is noted but does not change the overall job result -4. Steps marked `best_effort: true` do not cause phase failure regardless of outcome -5. Steps with `optional: true` can be skipped without causing failure - -### ci-operator vs Test Errors - -Whether an error comes from ci-operator itself or from test code is critical for correct -root-cause analysis. - -**ci-operator infrastructure errors** — indicate CI infrastructure problems: - -| Error Message | Category | Meaning | -|---------------|----------|---------| -| `failed to resolve release images` | Release resolution | Image stream or tag not found | -| `failed to build [image]` | Image build | Dockerfile or dependency issue | -| `error creating release images` | Release creation | Payload assembly failed | -| `error: the interrupt handler was triggered` | Timeout/interrupt | Job was killed (timeout or preemption) | -| `failed to acquire lease` | Resource management | No cloud quota available | -| `could not resolve ...` | Step resolution | Step registry reference broken | -| `unresolvable tag` | Image resolution | Referenced image tag missing | -| `error: Process interrupted` | Pod kill | Pod killed (node issue, eviction, or preemption) | -| `pod pending timeout` | Scheduling | Pod couldn't be scheduled on build cluster | - -**Test errors** — indicate product or test code problems: - -| Error Pattern | Category | Meaning | -|---------------|----------|---------| -| `FAIL [test name]` | Test failure | A Go test assertion failed | -| `panic:` in test output | Test crash | Test binary crashed | -| `timed out waiting for` | Test timeout | Product not ready in time | -| `install should succeed` | Install failure | OpenShift installation failed | -| Step build-log shows application errors | Application error | Product component failure | - -**Key distinction**: ci-operator errors appear in the **top-level `build-log.txt`** before -test execution begins. Test errors appear in **step-level build logs** under -`artifacts/{target}/{step-name}/build-log.txt`. - -### ci-operator Step Graph - -The `ci-operator-step-graph.json` artifact shows all steps, their dependencies, and timing: - -```json -[ - { - "step": "ipi-install-install", - "duration": "18m32s", - "state": "succeeded", - "started_at": "2024-01-15T10:23:45Z", - "finished_at": "2024-01-15T10:42:17Z" - }, - { - "step": "openshift-e2e-test", - "duration": "45m12s", - "state": "failed", - "started_at": "2024-01-15T10:42:30Z", - "finished_at": "2024-01-15T11:27:42Z" - } -] -``` - -If steps never started or all failed simultaneously at a very early timestamp, suspect an -infrastructure issue over a product issue — confirm in the top-level `build-log.txt`. - -### Common ci-operator Failure Modes - -#### `creating_release_images` - -The job fails during release image assembly. Usual causes: -- The release image stream tag being deleted or moved -- CI registry (registry.ci.openshift.org) outage or slowness -- Network issues between the build cluster and the registry -- A recent change to the `releases` section in ci-operator config - -**Diagnose**: Check the top-level `build-log.txt` for lines starting with `Creating release -images`. The error message typically includes which specific image import failed. - -#### `pod_pending` - -The ci-operator pod or a test step pod could not be scheduled: - -```text -error: pod ci-op-xxxxx/test did not start running within 30m: - containers with unready status: [test] - pod is in Pending phase -``` - -Causes: -- Build cluster at capacity (node pressure) -- Resource requests exceeding available node capacity -- Node selector or affinity constraints not satisfiable -- Image pull backoff (registry issues) - -#### `image_build_failure` - -Image building from source failed: - -```text -failed to build "installer": error building image: ... -``` - -This can be CI infrastructure (missing base image, registry issues) or product code -(Dockerfile errors, compilation failures). Check whether the `base_images` are available and -whether the build error is a compilation error or an image pull error. - -#### `step_resolution_failure` - -A step registry reference could not be resolved: - -```text -could not resolve 'ipi-install-install': no such ref -``` - -The step registry was changed and a reference broke. Check recent `openshift/release` PRs -modifying the step registry. - ---- - -## Step Registry Deep Dive - -### The ipi-install Workflow - -The IPI (Installer-Provisioned Infrastructure) workflow is the most common; its structure -covers the majority of CI job failures. - -**Typical ipi-install workflow execution order**: - -```text -PRE PHASE: - 1. ipi-install-rbac → Set up RBAC for CI service accounts - 2. ipi-conf → Generate base install-config.yaml - 3. ipi-conf-{platform} → Platform-specific install-config tweaks - 4. ipi-install-install → Run openshift-install create cluster - 5. [optional additional setup] - -TEST PHASE: - 6. openshift-e2e-test → Run the e2e test suite - (or custom test steps) - -POST PHASE: - 7. gather-must-gather → Collect must-gather archive - 8. gather-extra → Collect oc command outputs, pod logs - 9. gather-audit-logs → Collect API server audit logs - 10. ipi-deprovision-deprovision → Destroy the cluster and cloud resources -``` - -### Common Gather Steps - -Gather steps run in the POST phase and collect diagnostic artifacts. Critical for post-mortem -analysis, but their failures do not change the overall job result. - -**`gather-must-gather`** — Runs `oc adm must-gather` to collect: -- ClusterOperator status and logs -- Pod status across all namespaces -- Node status and conditions -- Events -- Operator-specific diagnostics - -**`gather-extra`** — Runs a series of `oc` commands to capture: -- `oc get nodes -o yaml` → `artifacts/oc_cmds/nodes` -- `oc get pods --all-namespaces` → `artifacts/oc_cmds/pods` -- `oc get events --all-namespaces` → `artifacts/oc_cmds/events` -- Pod logs by namespace → `artifacts/pods/` -- API server audit logs → `artifacts/audit_logs/` -- Node journal logs → `artifacts/journal_logs/` - -**`gather-audit-logs`** — Specifically collects API server audit logs for detailed -request-level analysis. - -### Pre/Test/Post Execution Model - -The phase model is essential for classifying failures: - -```text -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ PRE Phase │────►│ TEST Phase │────►│ POST Phase │ -│ (setup) │ │ (testing) │ │ (cleanup) │ -└──────────────┘ └──────────────┘ └──────────────┘ - │ fail │ fail │ fail - ▼ ▼ ▼ - Skip TEST Record failure Log warning - Run POST Run POST Continue POST -``` - -**JUnit artifacts encode phase information**: - -The ci-operator JUnit XML (`junit_operator.xml`) includes phase-level testcases: -- `"Run multi-stage test pre phase"` — indicates pre phase status -- `"Run multi-stage test test phase"` — indicates test phase status -- `"Run multi-stage test post phase"` — indicates post phase status - -Individual step results also appear as testcases in the JUnit XML, allowing you to -identify exactly which step in which phase failed. - -### How Step Failures Propagate - -When a step fails, the behavior depends on its configuration: - -1. **Normal step failure**: The step exits with a non-zero exit code. ci-operator records - the failure and applies phase-level propagation rules (see above). - -2. **Step timeout**: If a step exceeds its `timeout` value, ci-operator kills the pod - and records a timeout failure. The `grace_period` allows the step to clean up before - being forcefully terminated. - -3. **`best_effort` step**: Even if it fails, the phase continues as if it succeeded. - Common for gather steps where partial collection is acceptable. - -4. **`optional` step**: Can be skipped entirely without causing failure. Used for - steps that may not be needed in all configurations. - -5. **Resource quota exceeded**: If the step pod's resource requests exceed what's - available on the build cluster, the pod stays Pending until timeout. - ---- - -## Distinguishing CI Infrastructure vs Product Failures - -The most important diagnostic skill. Getting it wrong wastes time investigating the wrong repo. - -### Decision Framework - -```text -Is the failure in the top-level build-log.txt (ci-operator output)? - ├── YES → Likely CI infrastructure - │ Check: lease errors, image pull errors, pod scheduling, step resolution - └── NO → Is the failure in a step build-log under artifacts/? - ├── Is the failure in a PRE phase step? - │ ├── ipi-install-install → Could be product (installer) or infra (cloud) - │ ├── ipi-conf-* → Usually CI config issue - │ └── Other setup → Depends on step content - ├── Is the failure in a TEST phase step? - │ └── Usually product or test code issue - │ But check for: test infra issues, flaky tests, env setup failures - └── Is the failure in a POST phase step? - └── Usually does not affect test result - Gather failures mean fewer diagnostics available -``` - -### Strong Indicators of CI Infrastructure Issues - -1. **Same failure across multiple unrelated jobs** - - If jobs for `openshift/installer`, `openshift/origin`, and `openshift/cluster-version-operator` - all fail with the same error at the same time → CI infrastructure issue - - Check [Sippy](https://sippy.dptools.openshift.org/) for widespread failure patterns - - Check [search.ci.openshift.org](https://search.ci.openshift.org/) for the error pattern - -2. **Failure before test code runs** - - Errors in the top-level `build-log.txt` during lease acquisition or step resolution - are CI infrastructure; image-build errors can be product (compile error) or infra - - No artifacts under `artifacts/{target}/` means ci-operator failed before test execution - -3. **ci-operator error reasons** - - `creating_release_images` → CI registry or image stream issue - - `pod_pending` → Build cluster capacity issue - - `acquiring_lease` → Cloud quota exhaustion - - `building_image` → May be product (compile error) or infra (base image missing) - - `resolving_step` → Step registry broken reference - -4. **Build farm / build cluster issues** - - Node pressure causing pod evictions: check if `prowjob.json` shows very short - duration with no test output - - Registry problems: `ImagePullBackOff` errors in build-log.txt - - DNS resolution failures: `lookup registry.ci.openshift.org: no such host` - - GCS upload failures: artifacts missing but tests may have passed - -5. **Correlation with `openshift/release` PRs** - - Check if a recent PR to `openshift/release` modified the step, workflow, chain, - or config used by the failing job - - Use the timing: if failures started at a specific time, check what merged around then - -6. **Cloud provider outages** - - Multiple jobs on the same cloud failing with API errors - - AWS: `RequestLimitExceeded`, `InsufficientInstanceCapacity` - - GCP: `QUOTA_EXCEEDED`, `ZONE_RESOURCE_POOL_EXHAUSTED` - - Azure: `OperationNotAllowed`, `QuotaExceeded` - -### Strong Indicators of Product Code Issues - -1. **Failure only in specific job variant** - - If only the `e2e-aws-ovn` job for `openshift/installer` fails but other installer - jobs pass → likely product issue specific to that configuration - -2. **Failure correlates with product PR** - - The failure started in a specific PR's presubmit → very likely product issue - - The failure appeared in periodics after a specific merge → product regression - -3. **Test assertion failures** - - `FAIL: TestFoo` in a test step build-log → test or product issue - - Stack traces pointing to product code → product issue - - Stack traces pointing to test framework code → test infrastructure issue - -4. **Step script errors** - - If the error is a scripting issue in a CI step (unbound variable, syntax error, - missing command, bad exit code from a shell script), read the step build-log to - identify the shell error, then check recent commits to that step's `*-commands.sh` - in `openshift/release`. This is a CI infrastructure issue caused by a step registry - change, not a product bug — the fix is a PR to `openshift/release`. Include the - responsible PR in your evidence. - -### Ambiguous Cases - -Some failures can be either CI or product: - -| Symptom | CI Infrastructure | Product Code | -|---------|-------------------|--------------| -| `install should succeed` | Cloud quota, lease timeout, region outage | Installer bug, config generation error | -| Image pull failure | Registry outage, tag moved | Wrong image reference in product code | -| Test timeout | Build cluster slow, resource contention | Product performance regression | -| Network connectivity error | Build cluster networking, DNS | Product networking bug, CNI issue | -| `OOMKilled` | Build cluster node pressure, wrong resource requests | Memory leak in product code | - -For ambiguous cases: -1. Check if the failure reproduces consistently (product) or is intermittent (likely infra) -2. Check if other jobs on the same cluster/profile show similar issues -3. Check the timing correlation with both product PRs and `openshift/release` PRs - ---- - -## How to Check for Recent CI Infrastructure Changes - -### Finding Relevant `openshift/release` PRs - -#### By Time Correlation - -If failures started at a specific time, check what merged to `openshift/release` around then: - -```text -https://github.com/openshift/release/pulls?q=is%3Apr+is%3Amerged+sort%3Aupdated-desc -``` - -Filter by the path that affects the failing job: -- `ci-operator/config///` — job configuration changes -- `ci-operator/step-registry//` — step definition changes -- `cluster/test-deploy/` — cluster profile changes -- `core-services/` — core CI service changes - -#### By Step Registry Path - -If a specific step is failing, find PRs that modified it: - -```text -https://github.com/openshift/release/pulls?q=is%3Apr+is%3Amerged+path%3Aci-operator/step-registry/ipi/install/ -``` - -#### By Job Config Path - -If a specific job's configuration may have changed: - -```text -https://github.com/openshift/release/pulls?q=is%3Apr+is%3Amerged+path%3Aci-operator/config/openshift/installer/ -``` - -### Checking Step Registry Changes - -To investigate a step change: - -1. **Identify the failing step** from the JUnit XML or build-log.txt -2. **Map step name to file path**: Step names use dashes (`ipi-install-install`) and map to - directory paths (`ci-operator/step-registry/ipi/install/install/`) -3. **Check recent changes** to the step's ref, chain, or commands: - ```bash - git log --oneline -20 -- ci-operator/step-registry/ipi/install/install/ - ``` -4. **Check if the step was modified** in a recent workflow change that includes it - -Use `/ci:list-step` to see the full hierarchy of a workflow or chain, including all refs and -their command scripts with repo paths. - -### Identifying Cluster Profile Changes - -Cluster profile changes can affect all jobs using that profile: - -1. Check if credentials were rotated (usually announced in advance) -2. Check for changes to profile defaults (regions, instance types) -3. Check Vault for secret updates (requires privileged access) - -### Lease and Quota Changes - -Lease configuration changes affect resource availability: - -1. Check for changes to Boskos/OFCIR configuration in `openshift/release` -2. Check if cloud account quotas were modified -3. Check if new jobs were added that compete for the same quota pool - -### pj-rehearse — Testing Config Changes - -The `pj-rehearse` system in `openshift/release` runs rehearsal jobs for config changes before -they merge. When investigating a suspected config change issue: - -1. Find the `openshift/release` PR that modified the config -2. Check if `pj-rehearse` ran and passed for that PR -3. If rehearse passed but production fails, the issue may be environmental (different cluster, - timing, or resource availability) - ---- - -## Image Promotion and Registry Interactions - -### How Images Flow Through CI - -```text -Developer PR - │ - ▼ -Presubmit Job (ci-operator) - │ Build images from source - │ Create test release payload - │ Run tests - │ ✗ Images are NOT promoted - │ - ▼ -PR Merges - │ - ▼ -Postsubmit Job (ci-operator) - │ Build images from source - │ Run tests (if configured) - │ ✓ Promote images to CI registry - │ registry.ci.openshift.org/ocp/4.18:component-name - │ - ▼ -Release Controller - │ Assembles new release payload from promoted images - │ Creates nightly/CI payload tag - │ Triggers acceptance jobs - │ - ▼ -Periodic Jobs - │ Test against assembled release payload - │ Use RELEASE_IMAGE_LATEST or similar env vars - │ - ▼ -Release (if acceptance passes) -``` - -### CI Image Registry - -The CI image registry at `registry.ci.openshift.org` stores: - -- **Built images**: Components built from source by postsubmit jobs -- **Release payloads**: Assembled by the release controller -- **Base images**: Shared base images used by builds -- **Test images**: Images specifically for test infrastructure - -Image references use OpenShift image stream notation: -```text -registry.ci.openshift.org/ocp/4.18:installer -registry.ci.openshift.org/ocp/release:4.18.0-0.nightly-2024-01-15-010101 -``` - -### Registry Pull Failures - -**Registry outage or slowness**: -```text -error: unable to pull image registry.ci.openshift.org/ocp/4.18:base: ... -``` -- Affects many jobs simultaneously -- Check registry status and recent incidents -- Retrying the job usually helps once the registry recovers - -**Rate limiting**: -```text -toomanyrequests: You have reached your pull rate limit -``` -- External registries (quay.io, docker.io) may rate-limit CI -- Internal registry (registry.ci.openshift.org) has higher limits but can still be affected - -**Image not found / `manifest unknown`**: -```text -manifest unknown: manifest unknown -``` -This can indicate: -- The image tag was deleted or moved -- The promotion job hasn't run yet (timing issue between post-submit and consumer) -- The image stream tag reference is incorrect -- A recent change to the `releases` or `base_images` section in ci-operator config - pointed to a non-existent tag - -### `manifest unknown` Timing Issues - -A common scenario after a product merge: - -1. PR merges to `openshift/installer` -2. Postsubmit job starts to build and promote images -3. Meanwhile, a periodic job starts and tries to use the new payload -4. The periodic job fails with `manifest unknown` because the postsubmit hasn't finished - promoting yet - -This is a transient CI infrastructure issue, NOT a product bug. The periodic job passes on the -next run once the postsubmit completes promotion. - -### Promotion Failures - -When a postsubmit promotion fails: -- The component's image in the CI registry becomes stale -- Subsequent release payloads will use the older image version -- This can cause payload rejection, test failures in periodic jobs that depend on the updated - image, and version skew between components - -**Detect**: Check the postsubmit job for the repo. If its most recent run failed or hasn't run, -the image promotion may be stale. - ---- - -## CI Job Lifecycle Phases - -A CI job runs through eight phases: (1) Prow scheduling, (2) ci-operator startup, (3) source -cloning, (4) image building, (5) lease acquisition, (6) multi-stage test execution -(pre/test/post), (7) artifact gathering, (8) teardown. Per-phase failure indicators and causes -are covered in the failure-modes and practical-steps sections above and below. - -**Key rule**: Failures at phases 1–5 occur before test execution and point to CI -infrastructure (image building can also fail on a product compile error). -Failures at phase 6 require careful analysis to distinguish CI from product issues. -Failures at phases 7–8 are informational and don't change the test result. - ---- - -## Detecting CI Infrastructure Failures — Practical Steps - -### Step 1: Check `build-log.txt` First - -The top-level `build-log.txt` shows ci-operator output. Read the first ~50 lines and the last -~100 lines: - -- **First lines**: ci-operator version, configuration loading, early setup -- **Last lines**: the final error, step summary, exit reason - -If the build log never reaches `"Running multi-stage test"` or `"Executing step"`, the failure -is in CI infrastructure before test execution. - -### Step 2: Check `prowjob.json` Timing - -```bash -jq '{ - start: .status.startTime, - pending: .status.pendingTime, - completion: .status.completionTime, - state: .status.state, - duration_minutes: ((.status.completionTime | fromdateiso8601) - - (.status.startTime | fromdateiso8601)) / 60 -}' prowjob.json -``` - -**Interpretation**: -- Long gap between `pendingTime` and `startTime` → scheduling issues (CI infrastructure) -- Very short total duration with no test artifacts → early CI infrastructure failure -- Normal duration with test artifacts → test-level failure (product or CI config) -- Duration near the job's timeout → timeout issue (could be either) - -### Step 3: Check `ci-operator-step-graph.json` - -This artifact shows all steps, their execution status, and timing. Look for: - -- Steps that never started → infrastructure failure prevented execution -- All steps failed simultaneously → cluster-level or infrastructure issue -- Only one step failed → focus investigation on that specific step -- Timing gaps between steps → resource contention or scheduling delays - -### Step 4: Check JUnit XML - -Download and parse `junit_operator.xml` from the artifacts: - -```bash -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/junit*.xml" -``` - -The JUnit XML encodes: -- Phase-level results (`"Run multi-stage test pre phase"`) -- Individual step results (each step as a testcase) -- Failure messages and error details - -### Step 5: Check for Symptom Labels - -The CI system attaches machine-detected symptom labels to job runs: - -```bash -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/job_labels/" -``` - -Each JSON file under `job_labels/` describes a detected symptom with a summary and explanation. -These are environmental observations, NOT definitive root causes. They provide context (e.g., -"test failures occurred during high CPU events") but require correlation with other evidence. - -### Step 6: Cross-Reference with Other Jobs - -Check whether the failure is isolated or widespread: - -1. **Same job, recent runs**: Does it fail consistently or intermittently? -2. **Same workflow, different repos**: Do other repos using the same workflow fail? -3. **Same cloud profile, different jobs**: Do other jobs on the same cloud fail? -4. **Same build cluster, different jobs**: Is the build cluster having issues? - -Use [Sippy](https://sippy.dptools.openshift.org/) to check pass rates for the job -over time and identify when the failure pattern started. - -Use [search.ci.openshift.org](https://search.ci.openshift.org/) to search for the -specific error message across all recent CI jobs. - ---- - -## Environment Variables and Overrides - -CI jobs are parameterized through environment variables; these help diagnose -configuration-driven failures. - -### Common Environment Variables - -| Variable | Purpose | Source | -|----------|---------|--------| -| `RELEASE_IMAGE_LATEST` | Release payload image to test | ci-operator config / override | -| `RELEASE_IMAGE_INITIAL` | Initial version for upgrade tests | ci-operator config / override | -| `OPENSHIFT_INSTALL_INVOKER` | Identifies the installer caller | Step registry ref | -| `CLUSTER_TYPE` | Cloud platform type | Cluster profile | -| `TEST_SUITE` | Which e2e test suite to run | ci-operator config env | -| `COMPUTE_NODE_TYPE` | Cloud instance type for workers | ci-operator config env | - -### Multistage Parameter Overrides - -Step registry steps can define parameters that are overridable: - -```yaml -# In the ref definition -env: - - name: TIMEOUT - default: "3600" - documentation: "Test timeout in seconds" -``` - -These can be overridden in the ci-operator config: - -```yaml -tests: - - as: e2e-aws - steps: - env: - TIMEOUT: "5400" -``` - -Or via the gangway API using the `MULTISTAGE_PARAM_OVERRIDE_` prefix: - -```json -{ - "pod_spec_options": { - "envs": { - "MULTISTAGE_PARAM_OVERRIDE_TIMEOUT": "5400" - } - } -} -``` - ---- - -## Artifact Directory Structure Reference - -See [Artifacts Reference](artifacts.md) for the complete GCS artifact tree and paths. - -**Pitfall**: The `{target}` directory under `artifacts/` is named after the `--target` value, -**not** the job name. Only the top-level Prow bucket path uses the job name from `.spec.job` in -`prowjob.json`. For PR jobs the two can differ, so use `--target` for step-artifact paths and -`.spec.job` only for the bucket path. - ---- - -## See Also - -- [Artifacts Reference](artifacts.md) — Complete artifact directory structure and paths -- [Cloud Provider Errors](cloud-provider-errors.md) — AWS/GCP/Azure-specific failure patterns -- [Flaky Test Identification](flaky-test-identification.md) — Distinguishing flaky tests from real failures -- [Install Failure Reference](install/general.md) — OpenShift installation failure analysis -- [Test Extension Binaries](test-extension-binaries.md) — component `*-tests-ext` (OTE) binary failures diff --git a/plugins/ci/skills/prow-job-analysis/references/cloud-provider-errors.md b/plugins/ci/skills/prow-job-analysis/references/cloud-provider-errors.md deleted file mode 100644 index 69aa4a0a6..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/cloud-provider-errors.md +++ /dev/null @@ -1,303 +0,0 @@ -# Cloud Provider Errors Reference - -Cloud infrastructure failures in CI: resource-lease exhaustion, cloud API quota -and rate limits, provisioning failures, leaked resources, credential/auth errors, -and region/zone availability. These fail a job **before or during** cluster -creation — typically environment issues rather than product bugs; -[§6](#6-outage-vs-config-vs-quota-vs-leak) helps classify. - -## When to Use - -- Job never starts: `build-log.txt` shows a lease/host/claim acquisition failure -- `junit_install.xml` failure mode is `infrastructure` (see - [Install — General](install/general.md#infrastructure-failures)) -- Installer log shows cloud API errors (quota, throttling, auth, capacity) -- Many jobs on **one** cloud fail simultaneously while other clouds pass - -**Use a different reference for:** - -- Lease mechanics at the ci-operator layer, `openshift/release` PR correlation → - [CI Infrastructure Changes](ci-infrastructure-changes.md#leases-and-resource-management) -- Node CPU/memory/disk-IOPS pressure on a *running* cluster (EBS burst credits, - Azure managed-disk IOPS caps) → [Resource Exhaustion](resource-exhaustion.md) -- Azure `CloudMetrics` disk saturation driving disruption → - [Disruption](disruption.md) -- Reading the installer log itself (work-backwards, eventual consistency) → - [Install — General](install/general.md) - -## Where Cloud Errors Surface - -| Location | Failure class | Meaning | -|----------|---------------|---------| -| `build-log.txt` (early, before test phase) | Lease / host / claim acquisition | No CI capacity was free — job never provisioned | -| `.openshift_install*.log` (non-deprovision) | Cloud API quota / throttle / capacity / auth | Installer could not create resources | -| `hive-install-*` provision pod logs | Hive-managed pool install | ClusterDeployment provision failed | -| deprovision `.openshift_install*.log`, `ipi-deprovision-deprovision` | Teardown / leaks | Failed cleanup → leaked resources | - -Find installer logs (exclude teardown to isolate provisioning): - -```bash -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "\.openshift_install.*\.log$" | grep -v deprovision -``` - ---- - -## 1. CI Resource Acquisition (job never provisions) - -The cluster does not exist yet. The error is in the top-level `build-log.txt`, -emitted before any product code runs. These are **always** CI infrastructure -issues. Retry once; if fleet-wide, escalate to Test Platform. - -### Boskos — cloud quota leases - -Boskos hands out cloud-account/quota "slices". Exhaustion: - -```text -Failed to acquire resource, current capacity: 0 free -failed to acquire lease for cloud quota slice "aws-quota-slice": all resources are in use -failed to acquire lease: context deadline exceeded -``` - -- `current capacity: 0 free` / `all resources are in use` — every account/slice - for that platform is occupied. Transient under high load; sustained means a - leak is holding leases (see [§5](#5-cascading-resource-leaks)). -- `context deadline exceeded` — no lease freed within the wait window. - -The ci-operator failure reason is `acquiring_lease`. See -[CI Infrastructure Changes](ci-infrastructure-changes.md#leases-and-resource-management) -for the lease lifecycle and how to correlate with `openshift/release` changes. - -### OFCIR — bare-metal / Equinix host pools - -OFCIR (OpenShift Fleeting CI Resources) provisions physical hosts for `metal`/`packet` -(Equinix) jobs. Two CRDs, inspected on the OFCIR management cluster: - -| CRD (short) | Key fields | Check | -|-------------|-----------|-------| -| `CIPool` (`cip`) | `.spec.provider`, `.spec.type`, `.spec.size`, `.status.state`, `.status.size` | Is the pool healthy and at requested size? | -| `CIResource` (`cir`) | `.spec.poolRef.name`, `.spec.state` (desired), `.status.state` (current), `.status.address` | Did a host reach `available` and get assigned? | - -```bash -oc get cip # PROVIDER PRIORITY STATE SIZE REQ-SIZE TYPE -oc get cir # ADDRESS STATE REQ-STATE POOL RES-ID -``` - -Acquisition fails when no `cir` reaches the `available` state (stuck -provisioning, or all `inuse`), or the `cip` `.status.size` is below `.spec.size` -(provider can't deliver hosts). Common CIR states: `available`, `inuse`, -`maintenance`, `error`. A pool full of `error`/`maintenance` CIRs = provider-side -hardware/capacity problem, not a product bug. `.status.size < .spec.size` points -at the underlying `provider` (e.g. Equinix) failing to allocate. - -`oc get cip/cir` needs access to the OFCIR management cluster. From job artifacts alone, read -`artifacts/{target}/ofcir-acquire/build-log.txt` and `junit_metal_setup.xml` — see -[install/metal.md](install/metal.md#ofcir-openshift-fleeting-ci-resources). - -### Hive — ClusterPool / ClusterClaim - -Jobs using pre-baked pools (`cluster_claim` / `hive-openshift-...` profiles) -claim a hibernating cluster instead of installing one. The ci-operator step -waits for the `ClusterClaim` `ClusterRunning` condition to be `True`; on timeout: - -```text -failed to wait for the created cluster claim to become ready: ... -timed out waiting for cluster claim to become ready -no clusters in pool are ready to be claimed -``` - -ci-operator reasons: `acquiring_cluster_claim` / `utilizing_cluster_claim`. -Root cause lives on the Hive management cluster (`hosted-mgmt`); pool definitions -are in `openshift/release` under `clusters/hosted-mgmt/hive/pools//`. - -| Object | Field / condition | Signal | -|--------|-------------------|--------| -| `ClusterPool` | `.status.Ready` / `.status.Standby` / `.status.Size` | All zero → no capacity to claim | -| `ClusterPool` | `CapacityAvailable: False`, `MissingDependencies: True`, `AllClustersCurrentImages: False` | Pool degraded / stuck | -| `ClusterDeployment` | `ProvisionFailed`, `ProvisionStopped`, `.status.installRestarts` | Underlying install keeps failing | -| `ClusterProvision` | `.spec.stage` = `Failed` (vs `Provisioning`/`Complete`) | Provision job failed | - -Frequent pool-install root causes: bad `install-config` in the pool's -`InstallConfigSecretTemplateRef`, invalid platform credentials/certs, or **cloud -quota / API rate limits** (§2) — a claim timeout is often a cloud-quota problem -one layer down. Installer logs are in the `hiveutil` container of the -`hive-install-*` pod (`-l hive.openshift.io/job-type=provision`). - ---- - -## 2. Cloud Quota & API Errors (during infrastructure provisioning) - -Failure mode `infrastructure`; errors are in the non-deprovision installer log. -For how to read that log see -[Install — General](install/general.md#infrastructure-failures). Grep quickly: - -```bash -grep -iE "quota|limit exceeded|throttl|rate exceed|insufficient|not available|denied" \ - .openshift_install*.log -``` - -### AWS - -| Category | Error strings | Notes | -|----------|--------------|-------| -| Throttling | `RequestLimitExceeded`, `Throttling: Rate exceeded`, `Client.RequestLimitExceeded` | Transient; retry. Fleet-wide = account API pressure | -| vCPU quota | `VcpuLimitExceeded: You have requested more vCPU capacity than your current vCPU limit` | Per-family, per-region (e.g. Standard On-Demand) | -| Instances/volumes | `InstanceLimitExceeded`, `VolumeLimitExceeded`, `MaxSpotInstanceCountExceeded` | Often leaks from failed teardown | -| Elastic IP | `AddressLimitExceeded: The maximum number of addresses has been reached` | Default 5 EIP/region (soft) | -| Capacity | `InsufficientInstanceCapacity`, `Insufficient capacity` | AWS-side shortage in that AZ — see [§7](#7-region--zone-availability) | -| IAM (hard) | `LimitExceeded: Cannot exceed quota for UsersPerAccount: 5000` | Non-adjustable — indicates leaked IAM users | - -### GCP - -| Category | Error strings | Notes | -|----------|--------------|-------| -| CPU quota | `Quota 'CPUS' exceeded. Limit: N in region R`, `QUOTA_EXCEEDED` | Per-region CPU pool | -| Addresses/disk | `Quota 'IN_USE_ADDRESSES' exceeded`, `Quota 'DISKS_TOTAL_GB' exceeded`, `Quota 'SSD_TOTAL_GB' exceeded` | Common with concurrent jobs / leaks | -| Rate limit | `rateLimitExceeded`, `Rate Limit Exceeded`, `userRateLimitExceeded` | Transient; project-wide API limit | -| Zone capacity | `ZONE_RESOURCE_POOL_EXHAUSTED`, `does not have enough resources available` | GCP-side — try another zone ([§7](#7-region--zone-availability)) | - -### Azure - -| Category | Error strings | Notes | -|----------|--------------|-------| -| Core quota | `OperationNotAllowed: Operation results in exceeding approved Cores quota`, `QuotaExceeded` | Per-family (e.g. `standardDSv3Family`), per-region | -| Public IP | `PublicIPCountLimitReached` | Regional public-IP cap | -| Resource groups | `ResourceGroupQuotaExceeded` / subscription RG limit (~980) | Leaked RGs from failed teardown | -| DNS zones | subscription DNS-zone limit reached; record-set-per-zone limit | Leaked private/public zones | -| SKU / zone | `SkuNotAvailable`, `ZonalAllocationFailed` | SKU/zone restriction — see [§7](#7-region--zone-availability) | - -### Hard vs soft quotas - -- **Soft** (adjustable via support/console; CI manages via account rotation): - vCPU/core counts, in-use addresses, disk GB, most per-region limits. Recover - by retrying on another account or waiting for teardown to release resources. -- **Hard** (fixed ceiling — no increase possible): e.g. AWS `UsersPerAccount: - 5000`, IAM roles/policies per account. Hitting a hard limit means **leaked - resources are not being cleaned up**; the only remedy is manual deletion of - orphaned resources in the account (§5). A hard-limit error is a signal to hunt - for a deprovision leak, not to request a quota bump. - ---- - -## 3. Infrastructure Provisioning: CAPI vs Terraform/IPI - -The provisioning engine varies by version; error signatures differ. Identify -which is in play before searching: - -| Engine | Versions | Log signatures | Where to look | -|--------|----------|----------------|---------------| -| **Cluster API (CAPI)** | Newer (default) | `Machine`/`MachineSet`/`InfraMachine` errors, `failed to reconcile`, CAPI controller logs in the installer's local bootstrap | `.clusterapi_output/`, machine-controller logs in the log bundle; installer log `msg="Creating infrastructure resources"` then CAPI errors | -| **Terraform** | Older | `terraform apply`, `Error: ...`, `error(s) applying`, provider plugin errors | `terraform` lines in `.openshift_install*.log`; `.tfstate` in the log bundle | - -Both surface the *same* underlying cloud API errors from §2 — CAPI wraps them in -Machine reconcile failures, Terraform in `apply` errors. Trace to the cloud -error string, then classify with §6. CAPI capacity/quota failures often appear as -Machines stuck `Provisioning` with the cloud error in the machine's status/events. - ---- - -## 4. Credential / Auth Failures - -Distinct from quota: the request is *rejected*, not throttled. Usually a rotated, -expired, or misconfigured CI credential (infra issue), occasionally a missing -IAM permission for a newly-used API (can be a product/config change). - -| Cloud | Error strings | Typical cause | -|-------|--------------|---------------| -| AWS | `AuthFailure`, `UnauthorizedOperation`, `AccessDenied`, `InvalidClientTokenId`, `SignatureDoesNotMatch` | Bad/rotated key, missing IAM permission | -| AWS STS | `ExpiredToken`, `RequestExpired`, `The security token included in the request is expired`, `AssumeRole` failures | Short-lived STS token expired mid-install (long runs) | -| GCP | `PERMISSION_DENIED`, `Request had invalid authentication credentials`, `oauth2: cannot fetch token`, `invalid_grant` | Service-account key invalid; workload-identity federation misconfigured | -| Azure | `AuthorizationFailed`, `invalid_client`, `AADSTS700016`, `AADSTS7000215`, `ClientSecretCredential authentication failed`, `ManagedIdentityCredential ... failed` | Expired client secret; SP lacks role; managed-identity not attached | - -Signals it is **credential** not permission: fails immediately, affects every API -call, and correlates with a credential rotation. A single `AccessDenied` on one -new API call amid otherwise-successful calls suggests a missing permission for a -newly-introduced resource (check recent installer/product changes). - ---- - -## 5. Cascading Resource Leaks - -A classic cause of *sustained* quota/lease exhaustion: - -```text -failed deprovision → orphaned cloud resources → quota fills → new installs fail - → Boskos slices never free → leases exhausted -``` - -A deprovision that errors leaves VPCs, EIPs, IAM users, disks, RGs, or DNS zones -behind. These accumulate across many jobs until a quota (often a **hard** one, -§2) is hit. Symptoms: quota/lease errors that **do not clear on retry** and -worsen over time; hard-limit errors like `UsersPerAccount: 5000`. - -Inspect teardown (do NOT confuse with provisioning logs): - -```bash -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "deprovision|ipi-deprovision-deprovision" -``` - -Look in the deprovision `.openshift_install*.log` for `failed to delete`, `still -exist`, timeouts, and auth errors during cleanup. Leaks are identified/cleaned by -cluster tags (e.g. `kubernetes.io/cluster/`); recovery requires manual -deletion of orphaned resources by Test Platform. Note it in findings — a single -job's leak degrades the whole shared account. - ---- - -## 6. Outage vs Config vs Quota vs Leak - -| Evidence | Classification | Action | -|----------|---------------|--------| -| One job fails, retry passes | Transient (throttle/capacity/lease contention) | Retry; no action | -| Many jobs, **one** cloud, others fine | Quota exhaustion or provider outage | Check quota + provider status page | -| All clouds / all jobs fail at same layer | CI platform issue (registry, lease server) | Not cloud-specific — see [CI Infrastructure Changes](ci-infrastructure-changes.md) | -| Error worsens over time, hard-limit hit | Resource leak | Hunt deprovision failures (§5) | -| `AccessDenied`/`AuthFailure` on every call | Credential rotation/expiry | Escalate credential (§4) | -| `AccessDenied` on one new API only | Missing permission (possible config/product change) | Check recent install-config / role changes | -| Validation error before any API call | Config error | Check `install-config.yaml` / cluster profile | - -**Outage vs config heuristics:** outages hit multiple *unrelated* jobs on the -same cloud/region at once and correlate with the provider's status page; config -errors reproduce deterministically on every run of *that* job and often correlate -with a recent `openshift/release` or install-config change. Quota/capacity is in -between — fleet-wide but self-resolving as load or leaks clear. - ---- - -## 7. Region / Zone Availability - -Capacity is per availability-zone; a shortage is provider-side, not a bug. - -| Cloud | Strings | Meaning | -|-------|---------|---------| -| AWS | `InsufficientInstanceCapacity`, `There is no Spot capacity available`, `Unsupported: ... not available in ... zone` | AZ lacks the instance type | -| GCP | `ZONE_RESOURCE_POOL_EXHAUSTED`, `does not have enough resources available to fulfill the request` | Zone exhausted | -| Azure | `SkuNotAvailable`, `ZonalAllocationFailed`, `AllocationFailed` | SKU/zone can't allocate | - -Correlate with the instance/SKU family and region in the job's cluster profile. -Common with large/newer instance types in busy regions. Usually transient -(retry) or fixed by the job targeting a different zone/type. Distinguish from -account **quota** (§2): capacity errors mention *availability*; quota errors -mention *limit/quota exceeded*. - ---- - -## Quick Triage Checklist - -1. **Where did it fail?** `build-log.txt` before test phase (§1 acquisition) vs - installer log (§2 provisioning) vs deprovision log (§5 leak). -2. **Grep the log** for `quota|limit|throttl|insufficient|denied|expired|capacity`. -3. **Classify** the string: acquisition, quota, capacity, auth, or leak. -4. **Retry test:** transient (passes on retry) vs persistent (real limit/leak/config). -5. **Scope:** one job (transient/config) vs one cloud (quota/outage) vs all clouds - (CI platform → [CI Infrastructure Changes](ci-infrastructure-changes.md)). -6. **Hard-limit or worsening?** Suspect a leak (§5); flag for manual cleanup. - -## See Also - -- [CI Infrastructure Changes](ci-infrastructure-changes.md) — lease lifecycle, cloud-profile/quota changes, `openshift/release` correlation -- [Install — General](install/general.md) — `infrastructure` failure mode, installer-log reading, CAPI/Terraform -- [Resource Exhaustion](resource-exhaustion.md) — node CPU/memory/disk-IOPS pressure on running clusters -- [Disruption](disruption.md) — Azure `CloudMetrics` disk saturation and disruption -- [Artifacts](artifacts.md) — artifact directory structure and paths diff --git a/plugins/ci/skills/prow-job-analysis/references/disruption.md b/plugins/ci/skills/prow-job-analysis/references/disruption.md deleted file mode 100644 index ddcf37ea8..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/disruption.md +++ /dev/null @@ -1,1273 +0,0 @@ -# Disruption Analysis Reference - -Interpreting disruption and interval data from OpenShift CI jobs: what the data means, how -to read it, and how to determine whether disruption is a cause, a symptom, or incidental -background noise. - ---- - -## Table of Contents - -1. [What Disruption Data Is](#what-disruption-data-is) -2. [How Disruption Monitoring Works](#how-disruption-monitoring-works) -3. [Interval File Format and Structure](#interval-file-format-and-structure) -4. [Key Signal Sources](#key-signal-sources) -5. [Backend Classification](#backend-classification) -6. [API Backend Disruption Monitoring](#api-backend-disruption-monitoring) -7. [Source-Node Analysis](#source-node-analysis) -8. [Disruption During Upgrade vs Conformance](#disruption-during-upgrade-vs-conformance) -9. [Symptom Labels](#symptom-labels) -10. [Correlating Disruption with Cluster Events](#correlating-disruption-with-cluster-events) -11. [Interpreting Disruption for Root Cause Analysis](#interpreting-disruption-for-root-cause-analysis) -12. [Common Root Cause Patterns](#common-root-cause-patterns) -13. [Cross-Run Comparison](#cross-run-comparison) -14. [Disruption Allowance Thresholds](#disruption-allowance-thresholds) -15. [Tooling: The Disruption Parser](#tooling-the-disruption-parser) -16. [Sippy Intervals Viewer](#sippy-intervals-viewer) - ---- - -## What Disruption Data Is - -### The OpenShift Disruption Monitoring Framework - -A continuous disruption monitoring framework runs alongside E2E tests. It probes key API -backends and records any period when those backends stop responding. Probes run for the -entire test job — from before the first test through the last test completing — giving a -complete picture of API availability over time. - -Disruption data is NOT a test result — it is an observational record. Separate disruption -*tests* evaluate whether observed disruption exceeded acceptable thresholds. Disruption -events appear in the interval data whether or not a disruption test ultimately fails. - -### What Constitutes "Disruption" - -A disruption event is recorded when a probe request to an API backend fails: - -- **Connection refused** — the backend's TCP port is not accepting connections -- **Connection timeout** — the connection attempt timed out without a response -- **EOF** — the connection was reset or closed unexpectedly -- **HTTP error** — the backend returned an error status code (5xx) -- **Elevated latency** — the response took longer than the configured threshold -- **DNS failure** — the backend hostname could not be resolved -- **Stopped responding** — the backend accepted the connection but did not respond - -Each failure is recorded as an interval event with `from` and `to` timestamps marking the -window when the backend was unresponsive. - -### What Disruption Is NOT - -- **Disruption is not a root cause.** It is always a symptom of something else — a node - draining, etcd leader election, OVS stall, or CPU starvation. Always look for the - underlying event. -- **Disruption is not the same as a test failure.** A job can have disruption events and still - pass (if within allowance thresholds), or it can fail with zero disruption (if a functional - test fails for unrelated reasons). -- **Brief transient disruption is not always a bug.** During upgrades, brief disruption is - expected as control plane components restart. The CI framework has allowances for this. - ---- - -## How Disruption Monitoring Works - -### Probe Architecture - -Lightweight HTTP pollers continuously issue GET requests to cluster API endpoints. Each poller: - -1. Establishes a connection (new or reused) to the target endpoint -2. Sends a GET request (typically to a health or readiness endpoint) -3. Records the result (success or failure with error type) -4. Waits a short interval, then repeats - -Probes run on **worker nodes** by default, simulating a workload trying to reach the control -plane. This matters: disruption observed from a specific worker node may be caused by issues -local to that node (OVS stall, CPU starvation) rather than a cluster-wide problem. - -### Connection Types: New vs Reused - -Every API backend is monitored with two connection strategies: - -- **New connections** (`connection: "new"`): Each probe creates a fresh TCP connection, - exercising DNS resolution, TCP handshake, TLS negotiation, and load balancer routing. - Disruption on new but not reused connections typically indicates a connection-establishment - problem (DNS, load balancer, TCP accept queue). - -- **Reused connections** (`connection: "reused"`): Probes reuse persistent HTTP connections, - exercising the request-processing path without connection overhead. Disruption on reused - but not new connections typically indicates a request-processing stall (slow backend, - resource contention) rather than a connectivity problem. - -**Why this matters for diagnosis:** - -| New Conn | Reused Conn | Likely Issue | -|----------|-------------|-------------| -| ✅ Disrupted | ✅ Disrupted | Backend or node is genuinely down — nothing gets through | -| ✅ Disrupted | ❌ Clean | Connection establishment problem — DNS, load balancer, TLS | -| ❌ Clean | ✅ Disrupted | Request processing problem — slow handler, connection draining | -| ❌ Clean | ❌ Clean | No disruption on this backend | - -Disruption affecting both connection types simultaneously points to a backend-level or -infrastructure-level issue rather than a connection-layer problem. - -### Host-to-Host Monitoring - -The framework also monitors direct host-to-host connectivity between nodes. These probes -send requests directly from a worker node to specific control plane endpoints using IP -addresses (bypassing DNS and load balancers). - -Host-to-host data localizes the problem to specific network paths. The disruption locator -encodes the source node, destination node, and endpoint IP: - -```text -host-to-host-from-node-{src-node}-to-node-{dst-node}-endpoint-{ip} -``` - -This lets the parser detect **single-source fan-out** patterns — all disruption originating -from one node — which immediately localizes the problem to that node. - ---- - -## Interval File Format and Structure - -### File Naming and Location - -Disruption data is stored in E2E timeline files (also called interval files): - -```text -artifacts/{target}/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_{timestamp}.json -``` - -`{timestamp}` is a `YYYYMMDD-HHMMSS` string of when the file was generated. - -**Upgrade jobs** typically produce **two** timeline files — one per phase: -- First file (sorted by filename timestamp) → **upgrade phase** -- Second file → **conformance/E2E test phase** - -**Non-upgrade jobs** typically produce **one** timeline file covering the entire test phase. - -```bash -# Find all timeline files for a job run -gcloud storage ls "gs://test-platform-results/logs/{job_name}/{build_id}/artifacts/**/e2e-timelines_spyglass_*.json" -``` - -### JSON Structure - -The timeline file is a JSON array (or object with an `items` array) of event records. Each -record represents an observed event over a time interval: - -```json -{ - "level": "Error", - "source": "Disruption", - "locator": { - "type": "Disruption", - "keys": { - "backend-disruption-name": "kube-api-new-connections", - "connection": "new" - } - }, - "message": { - "reason": "DisruptionBegan", - "humanMessage": "kube-api-new-connections stopped responding to GET requests over new connections", - "annotations": { - "reason": "DisruptionBegan" - } - }, - "from": "2026-03-21T21:50:24Z", - "to": "2026-03-21T21:50:26Z" -} -``` - -Host-to-host backends additionally carry a `disruption` locator key encoding the source node, -target node, and endpoint (see the Field Reference below). Plain API backends like the example -above do not — don't mix the two when writing jq filters. - -### Field Reference - -| Field | Type | Description | -|-------|------|-------------| -| `level` | string | Severity: `"Error"`, `"Warning"`, or `"Info"`. Disruption events are Error or Warning. | -| `source` | string | Event category. `"Disruption"` for disruption events. See [Key Signal Sources](#key-signal-sources) for all sources. | -| `locator.type` | string | Event type classification (e.g., `"Disruption"`, `"Node"`, `"Pod"`) | -| `locator.keys` | object | Key-value pairs identifying the specific resource or backend | -| `locator.keys.backend-disruption-name` | string | The backend being monitored (e.g., `"kube-api-new-connections"`) | -| `locator.keys.connection` | string | `"new"` or `"reused"` — the connection strategy used | -| `locator.keys.disruption` | string | For host-to-host backends: encodes source node, target node, and endpoint IP | -| `message.reason` | string | Event reason: `"DisruptionBegan"`, `"DisruptionEnded"`, etc. | -| `message.humanMessage` | string | Human-readable description including error details | -| `message.annotations` | object | Structured metadata (varies by source) | -| `from` | string | ISO 8601 timestamp — when the event started | -| `to` | string | ISO 8601 timestamp — when the event ended | - -### Duration Calculation and Significance - -Duration is `to - from`, in seconds: - -| Duration | Interpretation | -|----------|---------------| -| 1–2 seconds | Brief blip — may be a transient network hiccup or load balancer failover | -| 3–10 seconds | Moderate disruption — likely a real event (node drain, pod restart, OVS stall) | -| 10–30 seconds | Significant disruption — sustained backend unavailability, likely impactful | -| 30+ seconds | Severe disruption — prolonged outage, almost certainly causes test failures | -| 60+ seconds | Critical disruption — something is fundamentally broken on the cluster | - -**Clustering**: Disruption events on the same backend close together in time (within seconds) -often represent the same incident observed by multiple probes. The parser uses a configurable -`--window` parameter (default 60 seconds) to group events and find concurrent cluster activity. - ---- - -## Key Signal Sources - -The timeline JSON contains events from many sources beyond disruption. These concurrent -events are essential for understanding **why** disruption occurred. - -### `Disruption` — API Backend Disruption Events - -The core disruption events. Level is `Error` or `Warning`. Always the starting point for -analysis. - -### `CPUMonitor` — CPU Starvation Indicators - -Reports when a node's CPU utilization exceeds 95%. CPU starvation cascades to everything on -the node: - -```json -{ - "source": "CPUMonitor", - "level": "Warning", - "message": { - "humanMessage": "99% of all CPU cores on node ip-10-0-23-45.ec2.internal are in use" - }, - "from": "2026-03-21T21:49:00Z", - "to": "2026-03-21T21:52:00Z" -} -``` - -**Interpretation**: When CPU >95% appears on the same node that is the source of disruption -events, CPU starvation is almost certainly the proximate cause. OVS vswitchd (the network -packet processor) is especially sensitive — when it cannot get CPU cycles, all network -traffic on that node freezes. - -### `EtcdLog` — etcd Health and Performance - -Reports etcd operational events: - -| Message Pattern | Meaning | Severity | -|----------------|---------|----------| -| `"apply request took too long"` | etcd is under write pressure — raft proposals are slow | High | -| `"slow fdatasync"` | Disk I/O bottleneck preventing WAL writes | Critical | -| `"waiting for ReadIndex response took too long"` | etcd read latency elevated | Medium | -| `"leader changed"` or `"elected leader"` | Leader election occurred — brief unavailability | Medium | -| `"failed to send out heartbeat"` | Network or load issue between etcd members | High | -| `"database space exceeded"` | etcd database size limit hit | Critical | -| `"took too long to execute"` | General slow operation warning | Medium | - -```json -{ - "source": "EtcdLog", - "level": "Warning", - "locator": { "keys": { "node": "ci-op-xxx-master-0" } }, - "message": { - "humanMessage": "apply request took too long (1.234s)" - }, - "from": "2026-03-21T21:50:10Z", - "to": "2026-03-21T21:50:10Z" -} -``` - -**Interpretation**: etcd issues cascade to the API server (which depends on etcd for all -reads and writes), causing API disruption across all backends. etcd warnings concurrent with -disruption across multiple API backends make etcd pressure almost certainly the root cause. - -### `EtcdDiskCommitDuration` — etcd Disk Commit Latency - -Reports when etcd's disk commit duration exceeds the 25ms threshold — a direct measure of -disk write performance for etcd transactions. - -### `EtcdDiskWalFsyncDuration` — etcd WAL Fsync Latency - -Reports when etcd's Write-Ahead Log fsync duration exceeds the 10ms threshold. WAL fsync -is the most latency-sensitive disk operation in etcd — slow fsync directly delays raft -proposal commits. - -**Interpretation for both**: Elevated disk latency → etcd cannot commit transactions -quickly → API server requests backed up → API disruption. This is a classic cascade -path on cloud platforms, especially Azure where managed disk IOPS are explicitly capped. - -### `OVSVswitchdLog` — OVN/OVS Packet Processing Stalls - -Reports when Open vSwitch (OVS) vswitchd experiences unreasonably long poll intervals — it -was unable to process network packets: - -```json -{ - "source": "OVSVswitchdLog", - "level": "Warning", - "locator": { "keys": { "node": "ci-op-xxx-worker-westus-db64f" } }, - "message": { - "humanMessage": "Unreasonably long 9235ms poll interval (9000ms user, 0ms system)" - }, - "from": "2026-03-21T21:50:20Z", - "to": "2026-03-21T21:50:20Z" -} -``` - -**Poll interval severity scale:** - -| Interval | Impact | -|----------|--------| -| 500–1000ms | Degraded networking — packets delayed but eventually forwarded | -| 1000–5000ms | Networking effectively frozen for 1–5 seconds | -| 5000ms+ | Severe — node was completely unable to process network traffic | -| 10000ms+ | Extreme — OVS was starved for 10+ seconds, expect all connections to fail | - -**Interpretation**: OVS stalls are the most direct cause of host-to-host disruption. OVS -stalls on the same node that is the source of disruption events give a clear causal chain: -CPU starvation → OVS cannot run → packets not forwarded → disruption recorded. Check -CPUMonitor events on the same node to confirm the root cause. - -### `CloudMetrics` — Cloud Provider Metrics - -Reports cloud provider infrastructure metrics that exceed thresholds. Currently focused on -Azure disk I/O: - -```json -{ - "source": "CloudMetrics", - "level": "Warning", - "message": { - "humanMessage": "Average value of 100.00 for metric OS Disk IOPS Consumed Percentage is over the threshold of 50.00" - }, - "from": "2026-03-21T21:49:00Z", - "to": "2026-03-21T21:52:00Z" -} -``` - -**Azure disk metrics and their thresholds:** - -| Metric | Threshold | What it Means | -|--------|-----------|---------------| -| OS Disk IOPS Consumed Percentage | 50% warning, 90%+ critical | How much of the disk's IOPS budget is being used | -| OS Disk Queue Depth | 3.0 (warning), 10+ (critical) | How many I/O operations are queued waiting for the disk | -| OS Disk Bandwidth Consumed Percentage | 50% | How much of the disk's throughput is being used | -| Data Disk IOPS Consumed Percentage | 50% | Same as above for data disks | - -**Interpretation**: Azure managed disks have hard IOPS caps based on disk tier. At 100% IOPS, -all additional I/O queues up, directly impacting etcd (which needs low-latency disk writes) -and cascading to API disruption. Disk I/O saturation is a recurring root cause of -disruption on Azure. - -### `AuditLog` — API Server Audit Events - -Reports API request failure rates during disruption windows: - -```json -{ - "source": "AuditLog", - "level": "Info", - "message": { - "humanMessage": "1 requests made during this time failed out of 611 total" - } -} -``` - -**Interpretation for disruption diagnosis** — this distinction separates "the API server was -overloaded" from "the API server was unreachable": -- **Audit entries show failures during disruption** → API server received requests but - couldn't process them (internal issue — etcd slow, resource pressure) -- **No audit entries during disruption** → requests never reached the API server - (connectivity issue — network failure, load balancer problem, node isolation) - -### `Alert` — Prometheus Alerts - -Reports Prometheus alerts that were firing during the test: - -```json -{ - "source": "Alert", - "level": "Warning", - "locator": { "keys": { "alert": "ExtremelyHighIndividualControlPlaneCPU" } }, - "message": { - "humanMessage": "alert ExtremelyHighIndividualControlPlaneCPU is firing" - } -} -``` - -**Key alerts related to disruption:** - -| Alert | What it Means | -|-------|---------------| -| `ExtremelyHighIndividualControlPlaneCPU` | A control plane node has extremely high CPU — confirms CPU starvation | -| `etcdHighCommitDurations` | etcd commits are taking too long — confirms disk pressure | -| `etcdHighNumberOfFailedGRPCRequests` | etcd gRPC requests are failing — etcd is degraded | -| `KubeAPILatencyHigh` | API server latency is elevated — confirms API slowness | -| `MCDDrainError` | Machine Config Daemon cannot drain a node — upgrade may be stalled | - -### `NodeMonitor` / `MachineMonitor` — Node and Machine State - -Reports node condition changes (NotReady, pressure conditions) and machine phase transitions: - -```json -{ - "source": "NodeMonitor", - "level": "Warning", - "locator": { "keys": { "node": "ci-op-xxx-master-0" } }, - "message": { - "humanMessage": "node condition Ready changed to Unknown" - } -} -``` - -**Interpretation**: Node state changes during disruption windows distinguish planned -operations (node drain during upgrade) from unexpected events (node going NotReady due to -resource pressure). - -### `ClusterVersion` / `ClusterOperator` — Operator Lifecycle - -Reports CVO upgrade progress and operator status transitions: - -```json -{ - "source": "ClusterOperator", - "locator": { "keys": { "name": "kube-apiserver" } }, - "message": { - "humanMessage": "condition/Progressing changed: True → operator is updating" - } -} -``` - -**Interpretation**: These events establish the upgrade timeline — when each operator started -updating, finished, and whether it became degraded. Correlating disruption with operator -transitions shows whether it is expected (operator restarting during upgrade) or unexpected. - -### `PodLog` — Pod Lifecycle Events - -Reports specific pod lifecycle events observed during the test. - -### `KubeletLog` — Kubelet Events - -Reports kubelet log events from cluster nodes, including pod start/stop, volume mount, -and node health activities. - -### `E2ETest` — Test Execution Events - -Reports which E2E tests were running at any given time: - -```json -{ - "source": "E2ETest", - "level": "Info", - "locator": { "keys": { "e2e-test": "[sig-api-machinery] API server should handle large requests" } }, - "message": { - "annotations": { "status": "Passed" } - }, - "from": "2026-03-21T21:45:00Z", - "to": "2026-03-21T21:50:00Z" -} -``` - -**Interpretation for disruption analysis**: Tests running during a disruption window matter -for two reasons: - -1. **Tests that fail during disruption** are usually *victims* — the disruption caused them - to fail, not the reverse. -2. **Tests that pass but consistently appear during disruption across multiple runs** are - potentially *causes* — they may be creating the resource pressure that triggers disruption. - -In multi-run analysis, cross-referencing which tests were active during disruption across all -runs is one of the most powerful techniques for identifying workload-induced disruption. - ---- - -## Backend Classification - -### What "Backends" Are - -A "backend" is a specific API endpoint or service that is continuously polled. Each backend -has a unique name encoding what is monitored and how. - -### Backend Types - -The parser classifies backends into four categories: - -| Type | Name Pattern | Example | Root Cause Indicator | -|------|-------------|---------|---------------------| -| **Non-cache** | Standard API names | `kube-api-new-connections` | Component or cluster networking problem | -| **Cache** | Contains `cache` prefix | `cache-kube-api-new-connections` | Likely etcd or global networking problem | -| **Canary** | `ci-cluster-network-liveness` | `ci-cluster-network-liveness-new-connections` | Test infrastructure network issues | -| **Cloud** | Cloud network-liveness | `cloud-network-liveness-azure-new-connections` | Cloud provider issues | - -### Cache vs Non-Cache Backends - -The distinction is architecturally significant: - -- **Non-cache backends** hit the API server's normal request path, including reading from - etcd for any non-cached resource. -- **Cache backends** are served from the API server's watch cache (an in-memory cache of - etcd state). They do NOT hit etcd directly for reads. - -**Why this matters:** - -- **Only non-cache backends** disrupted, cache clean → likely etcd read performance (cache - hits avoid the problem). -- **Both cache and non-cache** fail simultaneously → problem is at the API server or network - level (below the cache layer). -- **Only cache backends** fail (rare) → watch-cache corruption or API server memory pressure. - -### The "All Four Variants" Diagnostic Pattern - -**Critical diagnostic signal**: When all four variants of a backend fail simultaneously: - -```text -openshift-api-new-connections (non-cache, new) -openshift-api-reused-connections (non-cache, reused) -cache-openshift-api-new-connections (cache, new) -cache-openshift-api-reused-connections (cache, reused) -``` - -This pattern points at **control plane node resource exhaustion**: - -```text -Disk I/O saturation → etcd stalls → API server cannot process any requests → all variants fail -``` - -If so, this is a resource issue, NOT a networking issue. Confirming evidence: -- `EtcdLog` events showing `slow fdatasync` or `apply took too long` -- `CPUMonitor` events on control plane nodes -- `Alert` events for `ExtremelyHighIndividualControlPlaneCPU` -- `CloudMetrics` showing disk IOPS at 100% - -### Network Liveness Backends - -Two special backend types monitor infrastructure health: - -- **`ci-cluster-network-liveness`** (canary): Polls an external endpoint from within the CI - cluster. Disruption here means the CI infrastructure's own network is unreliable. - **When this is disrupted, the run's disruption data for other backends is unreliable** — - disruption events on normal backends may be test-infrastructure artifacts, not real cluster - issues. - -- **Cloud network-liveness** (e.g., `cloud-network-liveness-azure`): Polls within the cloud - provider's network. Disruption here points to cloud provider networking, not OpenShift. - -### Common Backend Names - -| Backend Name | What It Monitors | -|-------------|-----------------| -| `kube-api-new-connections` | Kubernetes API server (new TCP connections) | -| `kube-api-reused-connections` | Kubernetes API server (persistent connections) | -| `openshift-api-new-connections` | OpenShift API server (new connections) | -| `openshift-api-reused-connections` | OpenShift API server (persistent connections) | -| `oauth-api-new-connections` | OAuth API server (new connections) | -| `oauth-api-reused-connections` | OAuth API server (persistent connections) | -| `image-registry-new-connections` | Internal image registry | -| `service-load-balancer-with-pdb-reused` | Service load balancer with pod disruption budget | -| `host-to-host-new-connections` | Direct node-to-node connectivity (new) | -| `host-to-host-reused-connections` | Direct node-to-node connectivity (reused) | -| `cache-kube-api-new-connections` | Kubernetes API via watch cache (new) | -| `cache-openshift-api-reused-connections` | OpenShift API via watch cache (reused) | - ---- - -## API Backend Disruption Monitoring - -### How the Framework Monitors API Endpoints - -Each API backend has a dedicated polling goroutine that: - -1. Creates a connection (new or reuses existing) -2. Sends a GET request to the API server's health or resource endpoint -3. Records success (response within timeout) or failure (error or timeout) -4. Generates interval events for disruption begin/end transitions - -Polling runs continuously from test start to finish, typically every 1–2 seconds, giving -second-level resolution of when disruption starts and stops. - -### Grace Periods During Upgrades - -Some upgrade disruption is expected and the framework accounts for it: - -- **API server rolling restarts**: kube-apiserver or openshift-apiserver pods restarting may - briefly stop serving requests -- **Load balancer drain**: cloud load balancers may take time to remove a draining backend - and route to a healthy one -- **etcd member restart**: etcd pod restart causes a brief quorum disruption - -Disruption tests have **allowance thresholds** permitting some upgrade disruption before -counting it as a failure. See [Disruption Allowance Thresholds](#disruption-allowance-thresholds). - -### Sustained vs Transient Disruption - -| Pattern | Duration | Typical Cause | -|---------|----------|---------------| -| **Single blip** | 1–2s | Load balancer failover, brief network hiccup | -| **Brief burst** | 3–10s | API server pod restart, etcd leader election | -| **Sustained** | 10–60s | Node drain, severe resource pressure, component crash | -| **Prolonged** | 60s+ | Fundamental infrastructure issue, hung process, failed drain | - -**Rule of thumb**: Transient blips (1–2s) during upgrade are almost always expected. -Sustained disruption (10s+) during conformance testing is almost always a real problem. - ---- - -## Source-Node Analysis - -### Why Source-Node Analysis Matters - -For host-to-host backends, the disruption locator encodes which nodes are involved, -determining whether disruption is: - -- **Localized to one node** → problem on that specific node -- **Spread across many nodes** → cluster-wide or destination-side problem - -This is one of the most powerful diagnostic signals in disruption data. - -### Pattern: Single-Source Fan-Out - -**All disruptions originate from one source node, hitting many target nodes.** - -```text -Worker-A → Master-0 ✗ disrupted -Worker-A → Master-1 ✗ disrupted -Worker-A → Master-2 ✗ disrupted -Worker-B → Master-0 ✓ clean -Worker-B → Master-1 ✓ clean -Worker-C → Master-0 ✓ clean -``` - -**Interpretation**: The problem is on Worker-A, not the masters or the network. The node -cannot send packets to *any* destination — classic symptoms: -- OVS vswitchd stall on Worker-A (check `OVSVswitchdLog` events for that node) -- CPU starvation on Worker-A (check `CPUMonitor` events) -- Disk I/O pressure causing system-wide slowdown on Worker-A - -**Action**: Focus entirely on that node — what workloads ran on it, whether it was under -resource pressure, and whether OVS stalled. - -### Pattern: Multi-Source - -**Disruptions originate from multiple source nodes.** - -```text -Worker-A → Master-0 ✗ disrupted -Worker-B → Master-0 ✗ disrupted -Worker-C → Master-0 ✗ disrupted -Worker-A → Master-1 ✓ clean -Worker-B → Master-1 ✓ clean -``` - -**Interpretation**: Multiple sources hitting the same destination suggests a destination-side -problem. If Master-0 receives all the disruption, check: -- Master-0's health (resource pressure, etcd status) -- Whether Master-0 is the etcd leader (leader election during restart) -- Network path specifically to Master-0 - -### Pattern: Cluster-Wide - -**Disruptions from many sources to many destinations.** - -```text -Worker-A → Master-0 ✗ disrupted -Worker-B → Master-1 ✗ disrupted -Worker-C → Master-2 ✗ disrupted -``` - -**Interpretation**: A cluster-wide issue affecting the entire network or control plane -simultaneously: -- Cloud provider network issue -- All control plane nodes under resource pressure simultaneously -- etcd quorum loss -- Widespread CPU starvation (typically from heavy E2E test workloads) - -### Pattern: Unknown - -**Backend type does not include node information in the disruption path.** Applies to -ingress-routed backends like `image-registry`, where the connection goes through the router -and the framework cannot determine which nodes are involved. - ---- - -## Disruption During Upgrade vs Conformance - -### Completely Different Significance - -The same disruption event has radically different significance depending on phase: - -| Phase | Context | Expected Disruption | Concern Level | -|-------|---------|-------------------|---------------| -| **Upgrade** | Control plane components are restarting, nodes are draining | Yes — brief disruption during API server and etcd restarts is normal | Low (if brief and within allowance) | -| **Conformance** | Cluster should be stable, running E2E tests | No — the cluster should be fully operational | High (any sustained disruption is unexpected) | - -### Phase Detection in Timeline Files - -The parser assigns a phase per event based on which timeline file it came from (in upgrade -jobs with two files): -- First file (sorted by filename) → `"upgrade"` phase -- Second file → `"conformance"` phase - -In single-file (non-upgrade) jobs, all events are `"conformance"` phase. - -### Interpreting Upgrade Phase Disruption - -During upgrade, these are **expected** and not bugs: - -- Brief (1–5s) API disruption during kube-apiserver rolling restart -- Brief (1–5s) disruption during etcd member restart -- Node NotReady events during node drain/reboot (MCO upgrade) -- Operator condition transitions (Progressing=True, briefly Degraded) - -These become **concerning** when: -- Disruption exceeds the allowance threshold (typically 10–30s depending on backend) -- Disruption is sustained (e.g., 120s) rather than brief — a sign the upgrade itself is broken - (stuck drain, failed etcd restart), not normal restart churn -- Multiple control plane components are disrupted simultaneously for extended periods -- A node fails to come back after drain/reboot -- An operator stays degraded after the upgrade window closes - -### Interpreting Conformance Phase Disruption - -During conformance testing, **any sustained disruption is unexpected**. Common causes: -- Resource pressure from E2E test workloads -- Leftover instability from a problematic upgrade -- Infrastructure issues (cloud provider, CI cluster networking) -- Product bugs triggered by specific test scenarios - ---- - -## Symptom Labels - -### What Symptom Labels Are - -The CI system may attach **symptom labels** to job runs — machine-detected environmental -observations stored as JSON files in `artifacts/job_labels/`. - -**Critical distinction: Symptom labels are NOT root causes.** They describe environmental -conditions detected during the run, which may or may not relate to the failure being -investigated. - -### How Labels Are Generated - -Automated analyzers scan the job's interval data and artifacts after completion, look for -specific patterns (e.g., "disruption events concurrent with high CPU measurements"), and -attach a label describing the observation. - -### Accessing Symptom Labels - -Download the `job_labels/*.json` files with the commands in the -[`job_labels/` section of artifacts.md](artifacts.md#job_labels--symptom-labels). Each JSON -file contains a summary and explanation of the detected symptom. - -### Common Symptom Labels and Their Meaning - -**Node-related symptoms:** -- **Node pressure / NotReady**: A node reported memory, disk, or PID pressure, or went - NotReady. Find what caused the pressure — heavy workloads, resource leaks, pod evictions. -- **Node cordon/uncordon**: A node was cordoned (drained) and uncordoned. Expected during - upgrades (MCO rolling reboot); during conformance suggests node maintenance or recovery. - -**Operator-related symptoms:** -- **Operator degraded**: A cluster operator entered Degraded. Check which one and whether it - recovered. Operators degraded at job completion are especially significant. -- **Operator unavailable**: An operator lost Available=True. More severe than Degraded — - usually means the operator's workloads are not running. -- **Operator progressing**: An operator was still updating. Normal during upgrade; abnormal - during conformance. - -**Network-related symptoms:** -- **Disruption duration exceeded threshold**: Total disruption on one or more backends - exceeded allowances. The most direct disruption-related symptom. -- **Network error rates elevated**: Higher than normal error rates on network probes. - -**Resource-related symptoms:** -- **CPU starvation detected**: One or more nodes showed sustained CPU above 95%. Correlate - with disruption source nodes. -- **Disk I/O pressure**: Cloud metrics showed disk I/O saturation. Correlate with etcd - performance and API disruption. - -### How to Use Symptom Labels - -1. **Inform investigation direction** — "CPU starvation detected" → prioritize CPUMonitor - events and OVS stalls. -2. **Corroborate findings** — a "disk I/O pressure" label strengthens a disk-I/O root-cause - hypothesis. -3. **Never use as conclusions** — a symptom label alone is not sufficient for root cause. - Always perform the full investigation. -4. **Include in reports** — mention relevant labels in the "Known Symptoms Seen" section, with - the caveat that they are environmental observations. - ---- - -## Correlating Disruption with Cluster Events - -### The Time-Alignment Challenge - -All event sources in the timeline file use the same clock (the cluster's system time), so -timestamps are directly comparable within a single file. Watch for: - -- **Different granularity**: Some events are second-precise, others minute-precise -- **Event reporting lag**: Some events (like CloudMetrics) are aggregated over time windows - and may report at the end of the window rather than the start -- **Multi-file analysis**: Two timeline files (upgrade + conformance) share the same clock - and can be compared directly - -### Building a Timeline of What Happened - -The most effective technique is a chronological timeline overlaying disruption events with -cluster activity: - -```text -21:49:00Z CPUMonitor: worker-db64f CPU at 98% -21:49:05Z OVSVswitchdLog: worker-db64f poll interval 9235ms -21:49:10Z Disruption: host-to-host from worker-db64f to master-0 BEGAN -21:49:10Z Disruption: host-to-host from worker-db64f to master-1 BEGAN -21:49:10Z Disruption: host-to-host from worker-db64f to master-2 BEGAN -21:49:15Z EtcdLog: master-0 apply request took too long -21:49:20Z AuditLog: 3 of 200 requests failed -21:49:25Z Disruption: kube-api-new-connections BEGAN -21:49:30Z E2ETest: [sig-api-machinery] test FAILED -21:50:00Z OVSVswitchdLog: worker-db64f poll interval recovered -21:50:05Z Disruption: host-to-host endpoints ENDED -21:50:10Z Disruption: kube-api-new-connections ENDED -``` - -From this timeline, the causal chain is clear: -1. CPU starvation on worker-db64f -2. OVS stalled (cannot process packets) -3. Host-to-host disruption from that node -4. API disruption (requests from that node's probes fail) -5. E2E test fails (running on or probing through that node) - -### Overlaying Disruption with Specific Event Types - -**Node drain/reboot events (MCO upgrades):** -- `NodeMonitor` events showing node condition changes -- `MachineMonitor` events showing machine phase transitions -- `ClusterOperator` events for `machine-config` operator showing Progressing=True -- Expect disruption during the drain window; investigate if it persists after - -**Operator transitions:** -- `ClusterOperator` events showing Available/Degraded/Progressing changes -- Correlate with specific backend disruption (e.g., `kube-apiserver` operator progressing - → kube-api disruption) -- Check if the operator recovered (Progressing=False, Available=True) - -**Pod restarts and evictions:** -- `PodLog` events showing pod lifecycle changes -- `KubeletLog` events showing eviction decisions -- Correlate with the namespace of the disrupted backend's underlying pods - -**Certificate rotations:** -- May cause brief TLS handshake failures -- `new-connections` disruption without `reused-connections` disruption -- Usually very brief (1–2s) and self-resolving - ---- - -## Interpreting Disruption for Root Cause Analysis - -### Disruption as Symptom, Not Cause - -**This is the most important principle**: Disruption events describe what happened (API -became unavailable), not why it happened. The investigation must always trace from the -disruption event backwards to the underlying cause. - -The investigation flow: -```text -Disruption detected → What was the source pattern? → What concurrent events occurred? -→ What does the causal chain look like? → What is the root cause? -``` - -### Did Disruption Cause the Test Failure? - -Not all disruption causes test failures, and not all test failures come from disruption. -Establish the link rather than assuming it: - -1. **Temporal overlap** — did the test fail inside an active disruption window? No → disruption - is not the direct cause. -2. **Test dependency** — tests that make API calls, create resources, or check operator status - are affected by API disruption; tests checking static config or audit policies are not. -3. **Error text** — `"connection refused"`, `"timeout"`, `"EOF"` point to disruption; - `"assertion failed"`, `"expected X got Y"` point to a functional bug, not disruption. -4. **Cross-run consistency** — consistent fail-with-disruption / pass-without is causal; - inconsistent is likely coincidental. - ---- - -## Common Root Cause Patterns - -### Pattern 1: CPU Starvation → OVS Stall → Network Disruption - -**A classic disruption pattern.** A node runs out of CPU, OVS cannot process packets, -and all network traffic from/to that node fails. - -**Identifying signals:** -- Single-source fan-out in source-node analysis -- `CPUMonitor` >95% on the source node -- `OVSVswitchdLog` stalls on the source node (often >5000ms poll intervals) -- `Alert`: `ExtremelyHighIndividualControlPlaneCPU` - -**Cascade:** -```text -Heavy workloads/tests → CPU exhaustion → OVS vswitchd starved → packets frozen -→ host-to-host disruption → API requests timeout → API disruption recorded -→ tests using API fail -``` - -**Root cause**: Whatever workloads consumed the CPU. Check `E2ETest` events for tests running -during the window. - -### Pattern 2: Disk I/O Saturation → etcd Stalls → API Disruption - -**Common on Azure.** Disk IOPS hit the cap, etcd cannot write, API server cannot commit -transactions. - -**Identifying signals:** -- `CloudMetrics` showing disk IOPS at 100% -- `EtcdDiskCommitDuration` above 25ms -- `EtcdDiskWalFsyncDuration` above 10ms -- `EtcdLog` showing `slow fdatasync`, `apply took too long` -- All four variants of API backends fail (cache + non-cache, new + reused) -- `Alert`: `etcdHighCommitDurations` - -**Cascade:** -```text -Disk I/O pressure → etcd WAL fsync slow → etcd cannot commit proposals -→ API server writes stall → API reads also stall (backpressure) -→ all API backends show disruption -``` - -**Root cause**: Whatever caused disk I/O pressure. On Azure, often the base VM disk tier -being too slow for the combined workload. Can also come from heavy logging, test workloads -writing to disk, or etcd compaction running during high load. - -### Pattern 3: etcd Leader Election → Brief API Unavailability - -**Normal during upgrades.** When etcd pods restart, a leader election occurs and the API -server briefly cannot process writes. - -**Identifying signals:** -- `EtcdLog` showing `leader changed` or `elected leader` -- Brief (1–5s) disruption on all API backends -- Correlated with `ClusterOperator` etcd showing Progressing=True -- Usually during upgrade phase, not conformance - -**Expected during upgrade** if brief. Becomes a problem if leader elections happen repeatedly -or outside the upgrade window. - -### Pattern 4: Node Drain → Expected API Disruption - -**Normal during upgrades.** MCO drains and reboots nodes one at a time for OS updates. - -**Identifying signals:** -- `NodeMonitor` showing node condition changes (Ready → NotReady) -- `ClusterOperator` machine-config showing Progressing=True -- API disruption timed to the drain window -- Node comes back Ready after reboot - -**Expected during upgrade.** Becomes a problem if: -- The drain takes too long (stuck draining pods) -- The node doesn't come back after reboot -- Multiple nodes drain simultaneously (breaking quorum) - -### Pattern 5: OVN Restart → Network Disruption Window - -**OVN-Kubernetes pod restart causes a brief networking outage on the affected node.** - -**Identifying signals:** -- `OVSVswitchdLog` stalls correlated with OVN pod restart -- Disruption localized to one node (single-source fan-out) -- Brief duration (typically 5–15s during restart) -- May correlate with OVN-Kubernetes operator update during upgrade - -### Pattern 6: Single-Node Disruption vs Cluster-Wide - -**How to tell the difference:** - -| Signal | Single-Node | Cluster-Wide | -|--------|------------|--------------| -| Source pattern | Single-source fan-out | Multi-source | -| CPU alerts | One node | Multiple nodes | -| OVS stalls | One node | Multiple nodes | -| API disruption | May be limited to probes from one node | All probes fail | -| etcd events | Usually clean (unless on control plane) | Often shows pressure | - -### Pattern 7: CI Infrastructure Network Issue - -**The test infrastructure itself has network problems, contaminating disruption data.** - -**Identifying signals:** -- `ci-cluster-network-liveness` backend shows disruption -- Network liveness status = "degraded" or "unreliable" -- Disruption events may be artifacts of CI network, not cluster issues -- Other backends' disruption may be inflated or spurious - -**Action**: Note the unreliable network liveness status prominently in the report. Do not -draw conclusions solely from disruption counts in runs with unreliable network liveness. -Non-disruption signals (etcd logs, CPU, alerts) remain valid even when network liveness is -unreliable. - ---- - -## Cross-Run Comparison - -### How to Compare Disruption Across Runs - -1. **Identify common backends**: Which backends show disruption across runs? - - | Backend | Run 1 | Run 2 | Run 3 | Pattern | - |---------|-------|-------|-------|---------| - | kube-api-new | 5 events | 3 events | 0 events | Intermittent | - | host-to-host-new | 11 events | 8 events | 12 events | Consistent | - | cache-openshift-api | 0 events | 0 events | 0 events | Clean | - -2. **Compare timing (relative)**: Are disruptions at similar points in the job? - - Same relative time across runs → likely triggered by a specific test or phase - - Random timing across runs → more likely infrastructure-dependent - -3. **Compare source patterns**: Is the source node pattern consistent? - - Same node type (e.g., always a worker) → workload scheduling issue - - Different nodes each run → infrastructure-dependent - -### Cross-Run Comparison Patterns - -| Pattern | What It Means | Investigation Focus | -|---------|--------------|---------------------| -| Same backends at similar times | Product bug or test sequencing issue | Which test is running at that relative time? | -| Same backends at different times | Infrastructure-sensitive but product-related | What makes this backend vulnerable? | -| Different backends across runs | Environment-specific | Cloud provider, node types, resource allocation | -| `ci-cluster-network-liveness` disrupted in some runs | Those runs' disruption data is unreliable | Note caveat; still include for non-disruption signals | -| Cache backends consistently disrupted | Systemic etcd or networking issue | etcd health, disk I/O across all runs | -| Non-cache only | Component-specific problem | Focus on the specific API server/operator | -| Consistent source-node fan-out (same node type) | Workload scheduling → CPU starvation pattern | What's scheduled on those nodes? | - -### Using Historical Data (Sippy) - -For systematic comparison beyond the runs you're analyzing, use Sippy: - -- **Disruption dashboard**: historical disruption rates for specific backends and job types -- **Test pass rates**: whether disruption-related test failures are increasing -- **Sippy intervals viewer**: visual comparison of interval data across runs: - `https://sippy.dptools.openshift.org/sippy-ng/job_runs/{build_id}/{job_name}/intervals` - -### Determining Code Change vs Infrastructure Issue - -When a disruption regression appears: - -1. **Check code-change correlation** — did a PR land that could affect the disrupted - component? Use the `fetch-new-prs-in-payload` skill to identify recent changes. -2. **Check multiple job types** — disruption in only one job type (e.g., only Azure) may be - infrastructure-specific; disruption across AWS, Azure, and GCP is more likely a product - issue. -3. **Check the baseline** — use Sippy historical data to establish whether current disruption - is elevated versus the job's normal rate. -4. **Check infrastructure changes** — CI infrastructure, cloud provider, or VM type changes - that could explain the disruption. - ---- - -## Disruption Allowance Thresholds - -### What Allowances Are - -The CI framework defines allowable disruption thresholds per backend per job type — the -maximum disruption (in seconds) considered acceptable for a scenario. Exceeding the allowance -fails the corresponding disruption test. - -### How Thresholds Work - -- Each disruption test monitors a specific backend -- The test sums total disruption seconds during its phase -- Total exceeds the allowance → test fails -- Allowances differ by phase (upgrade vs non-upgrade) and by backend - -### Why Allowances Exist - -Some disruption is architecturally unavoidable: -- API server rolling restart during upgrade → brief (1–3s) disruption per restart -- etcd member restart → brief quorum disruption -- Load balancer health check propagation delay - -Allowances prevent these expected transients from failing tests while still catching genuine -regressions. - -### Interpreting Failures Against Thresholds - -A failed disruption test means disruption **exceeded** the baseline for that scenario — a -meaningful signal that something made it worse. Investigate the disruption beyond the expected -upgrade-related amount; the root cause is whatever pushed it over the threshold (a product -regression, heavier test workloads, infrastructure degradation). - ---- - -## Tooling: The Disruption Parser - -`parse_disruption.py` is the primary tool for structured disruption analysis — it extracts, -classifies, and correlates disruption data from timeline files. The -[analyze-disruption skill](../../analyze-disruption/SKILL.md) owns this tool and documents its -full usage, flags (`--backends`, `--window`, `--format`, `--job-name`, `--build-id`, -`--target`), JSON output schema, and feature list (backend classification, phase detection, -source-node fan-out, concurrent-event extraction, OVS/CPU/cloud/etcd summarization, network -liveness). Quick invocation: - -```bash -python3 plugins/ci/skills/analyze-disruption/parse_disruption.py \ - [timeline2.json ...] --window 60 --format text -``` - -### Network Liveness Assessment Scale - -| Status | Total Liveness Events | Meaning | -|--------|----------------------|---------| -| `clean` | 0 | No test infra or cloud network issues | -| `minor` | 1–5 | Negligible blips — disruption data is reliable | -| `degraded` | 6–50 | Some disruption may be from infra/cloud — use caution | -| `unreliable` | 50+ | Massive network-liveness disruption — other disruption data is unreliable | - ---- - -## Sippy Intervals Viewer - -### Visual Analysis - -The Sippy intervals viewer shows all interval data for a job run — disruption events, cluster -activity, and test execution: - -```text -https://sippy.dptools.openshift.org/sippy-ng/job_runs/{build_id}/{job_name}/intervals -``` - -### What It Shows - -- Horizontal timeline with all events plotted against time -- Color-coded events by source and severity -- Disruption events prominently highlighted -- Test execution intervals showing when each test ran -- Cluster operator state transitions -- Node state changes - -### When to Use - -- **Quick visual overview** before diving into detailed analysis -- **Pattern recognition** — visual correlation of multiple event types is often faster than - reading raw data -- **Sharing with humans** — the easiest way to show someone what happened during a run -- **Cross-run visual comparison** — open multiple tabs to compare disruption patterns - ---- - -## Key Artifacts for Disruption Analysis - -Quick reference for locating disruption-relevant artifacts: - -| Artifact | Path | Purpose | -|----------|------|---------| -| Timeline data | `artifacts/{target}/openshift-e2e-test/artifacts/junit/e2e-timelines_spyglass_*.json` | Disruption events + concurrent cluster activity | -| Audit logs | `artifacts/{target}/gather-extra/artifacts/audit_logs/` | API request details during disruption | -| etcd pod logs | `artifacts/{target}/gather-extra/artifacts/pods/openshift-etcd/` | etcd health and leader changes | -| Journal logs | `artifacts/{target}/gather-extra/artifacts/journal_logs/` | Node-level OVS and systemd logs | -| Symptom labels | `artifacts/job_labels/*.json` | Machine-detected symptom patterns | -| Cluster operators | `artifacts/{target}/gather-extra/artifacts/oc_cmds/co` | Operator status at gather time | -| Node status | `artifacts/{target}/gather-extra/artifacts/oc_cmds/nodes` | Node conditions at gather time | -| Pod status | `artifacts/{target}/gather-extra/artifacts/pods/` | Pod logs organized by namespace | - ---- - -## Analysis Decision Tree - -When starting a disruption investigation: - -```text -1. Was ci-cluster-network-liveness disrupted? - ├─ Yes → Mark run as potentially unreliable; note caveat in report - │ Continue analysis — non-disruption signals are still valid - └─ No → Disruption data is reliable - -2. What is the source-node pattern? - ├─ Single-source fan-out → Focus on that specific node - │ ├─ Check CPUMonitor on that node - │ ├─ Check OVSVswitchdLog on that node - │ └─ Identify workloads running on that node - ├─ Multi-source → Check if targeting same destination - │ ├─ Same destination → Destination node issue - │ └─ Different destinations → Cluster-wide issue - └─ Unknown → Cannot localize; check all signal sources - -3. Which backend types are disrupted? - ├─ All four variants of a backend → Suspect control plane resource exhaustion - │ └─ Check etcd, disk I/O, CPU on control plane nodes - ├─ Non-cache only → Component-specific or networking issue - ├─ Cache only (rare) → Watch-cache or memory issue - └─ Cloud/canary liveness → Infrastructure issue - -4. What phase was disruption in? - ├─ Upgrade → Expected if brief; concerning if prolonged - │ └─ Check: is disruption within allowance thresholds? - └─ Conformance → Unexpected; investigate root cause - -5. What concurrent events are present? - ├─ OVS stalls → CPU starvation chain (see Pattern 1) - ├─ etcd pressure → Disk I/O chain (see Pattern 2) - ├─ CloudMetrics → Cloud provider resource limits - ├─ Node state changes → Planned operation or node failure - └─ Alerts firing → Confirms severity; check alert details -``` - ---- - -## Example: Complete Disruption Analysis - -### Scenario - -A periodic e2e-azure-ovn-upgrade job shows 11 disruption events on `host-to-host-new-connections` -during the conformance phase. - -### Step-by-Step Analysis - -**1. Run the parser:** -```bash -python3 plugins/ci/skills/analyze-disruption/parse_disruption.py \ - timeline_upgrade.json timeline_conformance.json \ - --window 60 --format text -``` - -**2. Check network liveness:** -```text -Network liveness: clean — No test infra or cloud network issues -``` -→ Disruption data is reliable. - -**3. Check source pattern:** -```text -Source node analysis: All 11 disruptions originate from worker-db64f hitting 3 target nodes. -This indicates a source-side networking issue (OVS/CPU/disk pressure on the source node). -``` -→ Single-source fan-out from worker-db64f. Problem is on that one node. - -**4. Check concurrent events:** -```text -Key signals: - - OVS vswitchd stalls: 4 events, max poll interval 9235ms on 1 node - - CPU >95%: worker-db64f - - Azure disk IOPS saturated (100%) -``` -→ CPU starvation on worker-db64f caused OVS to stall, which caused the disruption. - -**5. Check phase:** -```text -Phase breakdown: conformance: 11 -``` -→ All disruption during conformance phase (unexpected, not during upgrade). - -**6. Root cause hypothesis:** - -CPU starvation on worker node `worker-db64f` during conformance testing caused OVS vswitchd -to stall for up to 9.2 seconds, freezing all network traffic from that node. This caused -11 host-to-host disruption events across all three control plane nodes. Concurrent Azure -disk IOPS saturation (100%) suggests the underlying cause may be disk I/O pressure -cascading to CPU contention. - -**Investigation next steps:** -- What E2E tests were running during the disruption window? -- What workloads were scheduled on worker-db64f? -- Is this pattern consistent across multiple runs of this job? diff --git a/plugins/ci/skills/prow-job-analysis/references/flaky-test-identification.md b/plugins/ci/skills/prow-job-analysis/references/flaky-test-identification.md deleted file mode 100644 index e5abcc8a1..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/flaky-test-identification.md +++ /dev/null @@ -1,319 +0,0 @@ -# Flaky Test Identification Reference - -The triage layer for a failing test. Before debugging a failure as a product bug, decide -which of three things broke: the **infrastructure**, the **product** (a real regression), -or the **test's own determinism** (a flake). Misclassifying a flake or infra failure as a -regression wastes work on the wrong repo; misclassifying a regression as a flake ships a bug. -This reference is the decision methodology — the deep-dive references own each root-cause class. - -## When to Use - -- A failure does not reproduce on retry, or a `/retest` passes -- The failure looks unrelated to the PR's diff -- Deciding whether to investigate, retry, or quarantine a test -- Confirming a suspected regression is real before filing a bug or reverting -- The same test fails in one job but passes elsewhere - -**Use a different reference for:** a failure you have already confirmed is a real product -regression in a plain e2e test → [test-failure.md](test-failure.md); an extension-binary -(`*-tests-ext`) failure → [test-extension-binaries.md](test-extension-binaries.md); the -mechanics of statistical verdicts across parallel runs → [aggregated.md](aggregated.md). - ---- - -## The Three-Way Classification - -Every failing `` is exactly one of these. Identify the class first; it decides -which repo to open. - -| Class | What broke | Fix lives in | Primary reference | -|-------|-----------|--------------|-------------------| -| **Infrastructure** | CI / cloud / shared env, before or around the test | `openshift/release`, cloud account, Test Platform | [ci-infrastructure-changes.md](ci-infrastructure-changes.md), [cloud-provider-errors.md](cloud-provider-errors.md) | -| **Product regression** | The code under test | Product repo / the PR | [test-failure.md](test-failure.md) (plain e2e), [test-extension-binaries.md](test-extension-binaries.md), [install/general.md](install/general.md), [upgrade.md](upgrade.md) | -| **Test-code flake** | The test's own determinism (races, timing, ordering) | The test source | this reference | - -Infrastructure failures and flakes **both** pass on retry, so a retry-pass alone does not -prove the test is at fault. Separate them by **scope** (how many jobs) and **reason** (the -ci-operator tag), covered below. Product regressions rarely pass on retry. - -### Fast triage signals - -| Signal | Infra | Regression | Flake | -|--------|:-----:|:----------:|:-----:| -| Passes on retry | Yes | Rarely | Yes | -| Same failure in 3+ unrelated jobs at once | Yes | No | No | -| Fails on all platforms **and** correlates with a PR | No | Yes | No | -| Fails on one platform / variant only | Possible | Possible | Possible | -| ci-operator failure reason is set (table below) | Yes | No | No | -| Sippy pass rate sits in an 80–99% band | — | — | Yes | -| Failure predates the PR (Sippy / search.ci) | — | No | Known flake | -| Identical error string on every occurrence | Sometimes | Yes | No (varies) | - ---- - -## Infrastructure: the "reason" and the "3+ jobs" heuristics - -### A ci-operator failure reason means the test never got a fair run - -When ci-operator fails before or around test execution, it tags a machine-readable **reason** -(visible in the build-log summary and the job's Prow/Sippy failure classification). A set -reason routes you **out** of this reference — the environment failed, not the product test. - -| Reason | Phase | Meaning | Deep dive | -|--------|-------|---------|-----------| -| `importing_release` | release setup | Base release payload could not be imported (tag missing, registry slow) | [ci-infrastructure-changes.md](ci-infrastructure-changes.md) | -| `creating_release_images` | release setup | Release payload assembly failed | [ci-infrastructure-changes.md](ci-infrastructure-changes.md) | -| `pod_pending` | scheduling | Step pod never scheduled (build-cluster capacity) | [ci-infrastructure-changes.md](ci-infrastructure-changes.md), [resource-exhaustion.md](resource-exhaustion.md) | -| `acquiring_cluster_claim` / `utilizing_cluster_claim` | cluster claim | Hive `ClusterClaim` never became ready | [cloud-provider-errors.md](cloud-provider-errors.md) | - -Also `acquiring_lease`, `building_image`, `resolving_step` — see -[ci-infrastructure-changes.md](ci-infrastructure-changes.md). Action: re-run the job; if it -recurs fleet-wide, escalate to Test Platform. Do not debug the test. - -### Same failure across 3+ distinct jobs = shared infrastructure - -One test (or error string) failing across **three or more distinct job names** -(different platforms, repos, or configs) at the same time implicates a shared dependency -rather than a per-test bug: CI registry, lease pool, cloud region, a step-registry change, or -the payload. Confirm breadth before blaming the test: - -```text -# search.ci — is this error hitting many jobs right now? -https://search.ci.openshift.org/?search=&maxAge=48h&type=junit -``` - -- Hits across unrelated jobs/repos → infrastructure - ([ci-infrastructure-changes.md](ci-infrastructure-changes.md), - [cloud-provider-errors.md](cloud-provider-errors.md)). -- Hits confined to one job or one PR → stay in this reference. - -Inverse of the product-bug heuristic in -[Test-name pattern analysis](#test-name-pattern-analysis-breadth-across-platforms) below. - ---- - -## Baseline flakiness with Sippy - -Sippy aggregates pass rates across all CI runs — the authority on "is this test already -unreliable?" Look it up before attributing a failure to a change. - -- Test details: `https://sippy.dptools.openshift.org/sippy-ng/tests/{release}/details?test={test-name}` -- Skills: `fetch-test-report` / `ci:fetch-test-report` (pass rate, test ID, Jira component); - `ci:query-test-result`; `fetch-test-runs` (per-run outputs for similarity analysis); - `ci:list-unstable-tests` (tests below a 95% pass rate). - -| Sippy pass rate | Reading | -|-----------------|---------| -| ~100%, stable, then this run failed | Real signal — regression or a fresh flake; investigate | -| 80–99% across many runs | Established flake — this failure is likely noise | -| Below ~80% and steady | Chronically broken / quarantine candidate; failure ≠ your change | -| ~100% then a step drop starting at a date | Regression — correlate the date with merges | - -A step-drop from a stable ~100% baseline is the strongest regression signal; a wide, stable -80–99% band is the strongest flake signal. - -## Known flake (pre-existing)? - -Before attributing a failure to the PR under investigation, prove the test was healthy -**before** it: - -1. **Sippy trend** — did the pass-rate drop begin before the PR's merge/run date? An earlier - drop means the PR did not cause it. -2. **search.ci history** — the same failure in periodics predating the PR is pre-existing: - ```text - https://search.ci.openshift.org/?search=&maxAge=336h&type=junit - ``` -3. **Existing bug** — search Jira (`OCPBUGS`) for the test name; an open flake bug confirms - known-flaky status. -4. **In-run flake marker** — the same test both fails and passes within this one run (see - [JUnit interpretation](#junit-interpretation)) = it flaked here, not a hard failure. - -If any of these holds, the PR did not cause the failure. - ---- - -## Test-name pattern analysis (breadth across platforms) - -The cross-platform breadth of a **single test** separates product bugs from environment noise: - -| Pattern | Most likely | Action | -|---------|-------------|--------| -| Same test fails across many platforms (aws + gcp + azure + metal) | Product bug — platform-independent code path | Treat as a regression; find the common PR / payload | -| Same test fails on **one** platform only | Environment / flake — cloud capacity, timing, that CNI/topology | Check that platform's infra ([cloud-provider-errors.md](cloud-provider-errors.md), [networking.md](networking.md), [resource-exhaustion.md](resource-exhaustion.md)) | -| Fails only in one variant (`proxy`, `fips`, `sno`, `ipv6`) | Variant-specific product path or that variant's known races | Check the matching reference / variant | -| Fails only in one job on one PR | Product change in the PR, or a fresh flake | Diff the PR; check the Sippy baseline | - -Inverse of the infra "3+ jobs" heuristic: there, *many different tests/jobs* share one failure -(shared infra); here, *one test* fails across many platforms (shared product code path). - ---- - -## Known flake categories - -### Timing-sensitive - -Hardcoded sleeps, tight poll intervals, fixed deadlines, or wall-clock assumptions. Grep the -test/step log: - -```text -context deadline exceeded -timed out waiting for( the)? condition -[Tt]imed out after .*s -``` - -These pass or fail inconsistently across fast/slow machines and fail more under build-cluster -load. Confirm with Sippy (banded pass rate) and interval timing ([disruption.md](disruption.md)). - -### Cloud-specific capacity - -A test or step failing on **one** cloud during provisioning or scaling — AZ capacity, quota, -throttling — is environment, not the test. Signatures (`InsufficientInstanceCapacity`, -`ZONE_RESOURCE_POOL_EXHAUSTED`, `SkuNotAvailable`, `RequestLimitExceeded`) and full triage: -[cloud-provider-errors.md](cloud-provider-errors.md). - -### Proxy / mirror races - -Disconnected / proxy / ipv6 jobs: pods pull before ICSP/IDMS or the mirror is ready → -transient `ImagePullBackOff`, `manifest unknown`, or `i/o timeout` that self-heal; a -proxy/SOCKS pod restart blips connectivity. Clears on retry = race, not product. Full pull -chain: [networking.md](networking.md#image-mirroring-race-conditions). - -### State-leaking / order-dependent - -A test that relies on prior-test state, or leaks namespaces/CRDs/cluster-scoped objects, -flakes based on run order or parallelism. Signature: fails in the full suite, passes in -isolation; the failure mentions leftover or missing resources. - ---- - -## Aggregated jobs: the statistical flake-vs-regression verdict - -Aggregated jobs (`aggregated-` prefix) run N copies of a job (usually 10) and apply -statistics — the definitive separator of flake from regression. Full treatment: -[aggregated.md](aggregated.md). The thresholds that drive the verdict: - -- **Minimum attempts** — at least ~6 of 10 runs must complete, or the result is - *insufficient data*, not a real failure. -- **Flake rate ~10%** with **minimum pass rate ~90%** — failures under the flake rate are - tolerated as flakes rather than flagged as regressions. - -| Ratio (of completed runs) | Verdict | -|---------------------------|---------| -| 0 pass / 10 fail | Strong regression | -| 3 pass / 7 fail | Regression | -| 8 pass / 2 fail | Flake — compare the 2 error strings for a common cause | -| 10 pass / 0 fail | Not a regression | -| 5 pass / 0 fail / 5 missing | Insufficient data — investigate why 5 did not complete (infra), not the test | - -**Non-deterministic test presence** — a test present in only some completed runs (message -like `Passed 1 times, failed 0 times ... require at least 6 attempts`) is a test bug -(conditional or non-deterministic registration), not a product regression. Distinguishing it -from insufficient-data: [aggregated.md](aggregated.md). - ---- - -## Symptom labels: correlation, not cause - -`job_labels/*.json` are machine-detected environmental observations (e.g. high CPU or disk -pressure during the run): - -```text -gs://test-platform-results/{bucket-path}/artifacts/job_labels/*.json # skip label-summary.html -``` - -A symptom **explains** how a flake could occur (CPU starvation → OVS stall → test timeout) -but does **not** prove the test is correct, nor attribute cause on its own. Use it to support -a flake hypothesis when it temporally overlaps the failure window; never cite it as the root -cause by itself. (Also covered in [ci-infrastructure-changes.md](ci-infrastructure-changes.md).) - ---- - -## JUnit interpretation - -Test results are the source of truth over an alarming build-log line. openshift-tests writes -`junit_e2e_*.xml`; ci-operator writes `junit_operator.xml`; install writes `junit_install.xml`. - -```bash -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/**/junit*.xml" -``` - -### Fields that matter - -- `` — run totals. openshift-tests does **not** - emit a standard `flakes` attribute; flakes are encoded as duplicate testcases (below). -- `` — one test. The child element decides status: - - `` / `` — did not pass; the `message` and body are the evidence. - - `` — excluded by an environment selector; **not** a failure. - - none — passed. - -### pass / fail / skip / flake - -- **skip ≠ fail** — skips are environment-gated (platform, feature-gate, topology). A test - that used to run and now skips is a coverage change that can hide a regression, not a pass. -- **In-run flake convention** — openshift-tests retries a failed test; if the retry passes, - the suite tolerates it. In the XML the same `name` then appears **twice**: once with a - `` and once passing. Same-name fail+pass in one file = **flaked this run**, not a - hard failure. Sippy derives its flake counts from exactly this pattern. -- A `` that also has a passing twin did not fail the suite — trust the JUnit verdict. - -### "Test failed" vs "test infrastructure failed" - -Not every `` is the product under test. Synthetic / harness testcases represent the -scaffolding, and their failure is an infra signal: - -| Testcase (synthetic) | Means | Class | -|----------------------|-------|-------| -| `Run multi-stage test pre/test/post phase` | ci-operator phase wrapper | Setup/infra, not the product test | -| `[sig-trt] ... extension binary ... should load successfully` | Extension extract/protocol failure | Test infra — [test-extension-binaries.md](test-extension-binaries.md) | -| `[sig-arch]` / monitor / invariant testcases | Post-run cluster invariants (disruption, alerts) | Cluster/infra signal, not the named test | -| `ipi-*`, `gather-*` step testcases | Setup / teardown steps | Infra (pre) or informational (post) | -| Real `[sig-x] should ` with `` | The product behavior under test | Product or test-code | - -An `` (rather than ``), or a failure whose message is a -harness/extraction/timeout-before-start error, is test-infrastructure — the product was never -exercised. A `` inside a genuine product-behavior testcase is the one worth product -investigation — root-cause it with [test-failure.md](test-failure.md). - ---- - -## Triage checklist - -1. **Reason set?** A ci-operator failure reason (table above) → infrastructure; re-run, do not - debug the test. -2. **Breadth?** search.ci — the same error in 3+ unrelated jobs → shared infra; one job/PR → - continue. -3. **JUnit truth?** Confirm a real product `` `` — not synthetic, not a - fail+pass flake twin, not a ``. -4. **Baseline?** Sippy pass rate — stable ~100% (signal) vs an 80–99% band (flake) vs a - step-drop at a date (regression). -5. **Pre-existing?** search.ci / Jira before the PR date → known flake, not this change. -6. **Platform breadth?** Many platforms → product bug; one platform/variant → - environment/flake (route to that reference). -7. **Aggregated?** Apply the ratio + minimum-attempts rules → regression vs flake vs - insufficient-data. -8. **Symptoms?** `job_labels/*.json` support (do not prove) a flake hypothesis in the failure - window. - -Once the checklist confirms a **real product regression** in a plain e2e test, hand off to -[test-failure.md](test-failure.md) for root-cause analysis (test source → cluster state → -originating error). - -## See Also - -- [test-failure.md](test-failure.md) — root-cause analysis for a confirmed product regression - in a plain e2e test: test source, cluster-state correlation, container tracing -- [ci-infrastructure-changes.md](ci-infrastructure-changes.md) — ci-operator reasons, - lease/registry/step-registry, the infra-vs-product framework, symptom labels -- [aggregated.md](aggregated.md) — statistical thresholds, failure Modes 1/2/3, minimum - attempts, non-deterministic presence -- [cloud-provider-errors.md](cloud-provider-errors.md) — cloud capacity/quota/claim - signatures behind one-platform flakes -- [networking.md](networking.md) — image mirror / proxy races, DNS/OVN timing flakes -- [test-extension-binaries.md](test-extension-binaries.md) — extension load/protocol failures - and their synthetic JUnit -- [disruption.md](disruption.md) — interval timing for timing-sensitive flakes -- [resource-exhaustion.md](resource-exhaustion.md) — CPU/memory/disk pressure behind timeouts - and OVS stalls -- [hypershift.md](hypershift.md) — failing tests in HCP/hypershift jobs; correlate management - vs hosted cluster -- [artifacts.md](artifacts.md) — full artifact tree and paths diff --git a/plugins/ci/skills/prow-job-analysis/references/hypershift.md b/plugins/ci/skills/prow-job-analysis/references/hypershift.md deleted file mode 100644 index 69472f6f4..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/hypershift.md +++ /dev/null @@ -1,186 +0,0 @@ -# HyperShift Reference - -HyperShift (Hosted Control Planes / HCP) runs a guest cluster's control plane as **pods in a -management cluster** instead of on dedicated control-plane machines. Every HyperShift job -therefore spans **two clusters**, and root cause almost always requires correlating both. - -## When to Use - -- Job name contains `hypershift`, `hcp`, or an HCP platform variant (`e2e-aws`, `e2e-kubevirt`, - `e2e-agent`, `e2e-azure`, `e2e-powervs` under the `openshift/hypershift` repo) -- Refs point at `openshift/hypershift` (org `openshift_hypershift` in PR-job paths) -- Failure references `HostedCluster`, `NodePool`, `HostedControlPlane`, or a `clusters-*` namespace -- Two must-gathers are present, or a `dump-management-cluster` / `hypershift-dump` artifact exists - -## Architecture: Two Clusters, One Job - -| Thing | Where it runs | What it is | -|-------|---------------|------------| -| **Management cluster** | Real nodes (the CI-installed OCP cluster) | Hosts hosted control planes as pods; runs the HyperShift operator | -| **Hosted (guest) cluster** | Worker nodes only | Tenant cluster — no control-plane machines; its API server is a pod on the mgmt cluster | -| **HostedCluster (HC)** | CR in mgmt ns `clusters` | Desired state + top-level status of one hosted cluster | -| **NodePool (NP)** | CR in mgmt ns `clusters` | Worker-node scaling/config for the hosted cluster (replaces MachineConfigPool) | -| **HostedControlPlane (HCP)** | CR in mgmt ns `clusters-` | Rendered control-plane state; owns the control-plane pods | - -**Naming rule:** a HostedCluster `/` (default ns `clusters`) has its control-plane -pods in namespace `-` — i.e. `clusters-`. This `clusters-` namespace is -the single most important location in a HyperShift job. - -## Identifying HyperShift Jobs - -| Signal | Example | -|--------|---------| -| PR job, hypershift repo | `pull-ci-openshift-hypershift-main-e2e-aws` | -| Periodic conformance | `periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-ovn-conformance` | -| Aggregated | `aggregated-hypershift-ovn-conformance-4.22` → see [aggregated.md](aggregated.md) | -| Step-registry usage | job uses `ci-operator/step-registry/hypershift/*` steps → see [ci-infrastructure-changes.md](ci-infrastructure-changes.md) | - -These are plain Go tests run as CI steps, not OTE extension binaries. When no specific test -name is provided, analyze the failed CI steps from the build log and step graph (see -[artifacts.md](artifacts.md) and [ci-infrastructure-changes.md](ci-infrastructure-changes.md)), -not the JUnit XML. Example Go test names: `TestCreateCluster`, -`TestCreateClusterCustomConfig`, `TestNodePool`, `TestKarpenter/EnsureHostedCluster/...`. - -## Artifacts: Where Each Cluster's Data Lives - -Full path table and directory tree are in [artifacts.md](artifacts.md#hypershift-must-gather-patterns). -The three patterns, and which cluster each half maps to: - -| Pattern | Detect (under `artifacts//`) | Management data | Hosted data | -|---------|----------------------------------------|-----------------|-------------| -| **Unified** | `dump-management-cluster/artifacts/artifacts.tar[.gz]` | extracted `output/` root | `output/hostedcluster-/` | -| **Dual** | `gather-must-gather/…/must-gather.tar` **and** `**/artifacts/hypershift-dump.tar` (or `**/hostedcluster.tar`) | standard must-gather | the `hostedcluster-*` dir inside the dump (may be absent) | -| **Standard only** | `gather-must-gather/…/must-gather.tar` alone | standard must-gather | none — management only | - -### Finding the hosted cluster namespace from artifacts - -The hosted namespace name (`clusters-`) is rarely printed directly — derive it: - -```bash -# 1. Unified/dual dump: the hostedcluster- dir names the cluster -find . -type d -name 'hostedcluster-*' # -> hostedcluster- => ns clusters- - -# 2. Management must-gather: the control-plane namespace IS clusters- -ls /namespaces/ | grep '^clusters-' - -# 3. build-log.txt: the hypershift CLI names it at create time -grep -Eo 'clusters-[a-z0-9-]+' build-log.txt | sort -u -``` - -### Key log/CR locations (paths relative to each cluster's must-gather root) - -| Data | Location | -|------|----------| -| HCP component pods & logs | **mgmt** `namespaces/clusters-/pods/` (unified: `output/namespaces/clusters-/`) | -| HyperShift operator logs | **mgmt** `namespaces/hypershift/pods/operator-*/` | -| HostedCluster / NodePool CRs | **mgmt** dump `hostedcluster.yaml` / `nodepool.yaml`, or `namespaces/clusters/hypershift.openshift.io/` | -| Hosted cluster operators / workloads | **hosted** `cluster-scoped-resources/config.openshift.io/clusteroperators*` and `namespaces//` | - -## Correlating Across Both Clusters - -1. **Download and extract both halves separately.** HyperShift jobs produce a management-side and a - hosted-side archive (e.g. `hypershift-dump.tar` / `hostedcluster.tar`, or a unified - `dump-management-cluster/artifacts/artifacts.tar` — see the [artifacts reference](artifacts.md) - for exact paths and general extraction steps): - - ```bash - mkdir -p must-gather-mgmt && tar -xf hypershift-dump.tar -C must-gather-mgmt/ - mkdir -p must-gather-hosted && tar -xf hostedcluster.tar -C must-gather-hosted/ - # Decompress any nested archives in either root - find must-gather-mgmt must-gather-hosted -name '*.gz' -exec gunzip -f {} + - ``` - - Then run the [must-gather-analyzer](../../../../must-gather/skills/must-gather-analyzer/SKILL.md) - scripts (`analyze_clusteroperators.py`, `analyze_pods.py --problems-only`, - `analyze_nodes.py --problems-only`, `analyze_events.py --type Warning`, `analyze_etcd.py`) - against **each root independently** and label every finding `[Management]` or `[Hosted]`. -2. **Check for mgmt → hosted causation.** The hosted control plane *is* - management-cluster pods, so a hosted-API symptom can have a management-side cause. -3. **Cross-cluster dependency chains** to trace (don't stop at the first symptom): - - ```text - Mgmt node MemoryPressure/NotReady ─▶ HCP pods (clusters-) evicted/Pending - ─▶ hosted kube-apiserver down ─▶ hosted ClusterOperators Degraded ─▶ tests fail - HyperShift operator reconcile error ─▶ HostedControlPlane rollout stalls ─▶ HC !Available - Mgmt registry/etcd/disk-IOPS issue ─▶ HCP ImagePullBackOff / etcd slow ─▶ hosted API 5xx - NodePool not ready ─▶ no hosted workers ─▶ hosted pods Pending ─▶ workload tests fail - ``` - -4. **Correlate on time** — align the test-failure window (interval files, see - [disruption.md](disruption.md)) with warning events in *both* clusters (±5 min). - -## HostedCluster & NodePool Status - -Read `.status.conditions[]` — `reason` and `message` carry the actionable detail. For HC, -healthy = `Available=True`, `Degraded=False`, `Progressing=False` (once settled). - -**HostedCluster** — check in this order: -`Available` → `ValidConfiguration` / `ValidReleaseImage` → `InfrastructureReady` → -`EtcdAvailable` → `KubeAPIServerAvailable` → `ValidHostedControlPlaneConfiguration` → -`ClusterVersionSucceeding` → `Degraded`. Also `.status.version.history[0]` for the target/state. - -**NodePool** — `Ready`, `AllMachinesReady`, `AllNodesHealthy`, `ValidReleaseImage`, -`UpdatingVersion`/`UpdatingConfig` (stuck = rollout wedged), `AutoscalingEnabled`. Compare -`.spec.replicas` vs `.status.replicas`; a mismatch with `AllMachinesReady=False` means workers -never provisioned — check the CAPI provider pods (`capi-provider`, `cluster-api`) in -`clusters-` and cloud capacity/quota ([cloud-provider-errors.md](cloud-provider-errors.md)). - -## Failure Patterns - -### Management-side (in `clusters-` or `hypershift` ns) - -- **HyperShift operator errors** — `namespaces/hypershift/pods/operator-*`: reconcile failures, - admission-webhook rejects, missing RBAC/pull-secret. Blocks HCP creation for *all* hosted clusters. -- **HostedControlPlane rollout failure** — control-plane pods not rolling out: `control-plane-operator` - (orchestrates the HCP), `kube-apiserver`, `etcd`, `kube-controller-manager`, `kube-scheduler`, - `openshift-apiserver`, `oauth-openshift`, `konnectivity-server`, `ignition-server`, - `hosted-cluster-config-operator`, `cluster-version-operator`. Look for `CrashLoopBackOff`, - `ImagePullBackOff`, `Pending` (mgmt scheduling/quota), high `restartCount`. -- **Management node pressure** — a mgmt cluster packing many HCPs hits CPU/memory/disk limits; - cascades to every hosted cluster on that node. See [resource-exhaustion.md](resource-exhaustion.md). - -### Hosted-side (in the hosted must-gather) - -- **Worker nodes / NodePool** — nodes `NotReady` or absent → NodePool not `Ready`; hosted pods `Pending`. -- **Hosted ClusterOperators Degraded** — can be *downstream* of a management-side control-plane - outage; confirm the HCP pods were healthy during the window before blaming the hosted operator. -- **Application workloads** — the test's own pods in the hosted cluster (namespace correlation). - -### Common failure modes - -- **Hosted etcd** — `etcd-0/1/2` pods in `clusters-`; `etcdserver: request timed out`, - `leader changed`, `slow fdatasync`. Backed by mgmt-cluster PVCs, so mgmt disk-IOPS pressure - surfaces here as hosted API latency. Cross-ref [resource-exhaustion.md](resource-exhaustion.md). -- **Ingress / connectivity** — `konnectivity-server` (mgmt) ↔ `konnectivity-agent` (hosted) tunnel - the control plane to guest workers; `router` pods serve `*.apps.`. `failed to dial`, - tunnel drops, or router not ready → hosted routes/webhooks unreachable. (Konnectivity is - HCP-specific — debug it here; router/ingress mechanics: [networking.md](networking.md#load-balancer-and-ingress).) -- **Kubelet connectivity** — hosted kubelets reach the hosted apiserver via its exposed endpoint - (LB/route/NodePort). `context deadline exceeded` / `the server is currently unable to handle the - request` from kubelets → apiserver pod down or endpoint mis-provisioned (check `InfrastructureReady`, - AWS endpoint service). - -## Common Failure Patterns — Quick Reference - -| Symptom | Likely cause | Where to look | -|---------|--------------|---------------| -| `HostedCluster` never `Available` | HyperShift operator or HCP rollout stalled | `hypershift/operator-*` logs; HC `.status.conditions` reason | -| HCP pods `Pending`/`Evicted` | Mgmt node pressure / quota | `[Management]` nodes+events, [resource-exhaustion.md](resource-exhaustion.md) | -| HCP pods `ImagePullBackOff` | Mgmt pull-secret / registry / mirror | `[Management]` pod events, [networking.md](networking.md) | -| Hosted operators Degraded, HCP healthy | Genuine hosted-side issue | `[Hosted]` clusteroperators + pods | -| NodePool `Ready=False`, replicas short | CAPI provisioning / cloud capacity | `capi-provider`/`cluster-api` in `clusters-`, [cloud-provider-errors.md](cloud-provider-errors.md) | -| Hosted API 5xx / timeouts | HCP etcd or kube-apiserver | `etcd-*`/`kube-apiserver-*` in `clusters-` | -| `*.apps.` unreachable | Router / konnectivity tunnel | `router`, `konnectivity-*`, [networking.md](networking.md) | -| Only management must-gather present | Standard-only pattern or hosted collection failed | confirm no `hostedcluster-*` dir; note in report | - -## See Also - -- [Artifacts](artifacts.md) — full must-gather path table, extraction, directory tree -- [Test Extension Binaries](test-extension-binaries.md) — component `*-tests-ext` (OTE) binary failures -- [Upgrade](upgrade.md) — HyperShift management-plane vs hosted-cluster upgrades, dual ClusterVersion -- [Aggregated Jobs](aggregated.md) — HyperShift conformance/aggregated child-run analysis -- [Resource Exhaustion](resource-exhaustion.md) — node CPU/memory/disk pressure and eviction - mechanics (the mgmt→HCP cascade is described above) -- [Networking](networking.md) — OVN/DNS/ingress/registry within either cluster -- [Cloud Provider Errors](cloud-provider-errors.md) — NodePool/infrastructure provisioning failures -- [must-gather-analyzer](../../../../must-gather/skills/must-gather-analyzer/SKILL.md) — per-cluster diagnostic scripts diff --git a/plugins/ci/skills/prow-job-analysis/references/install/general.md b/plugins/ci/skills/prow-job-analysis/references/install/general.md deleted file mode 100644 index 45a2ce9a5..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/install/general.md +++ /dev/null @@ -1,871 +0,0 @@ -# Install Failure Analysis — General - -**Use when** a CI job fails an `install should succeed` test, or when an upgrade job fails during -its initial install phase (before the upgrade begins). Definitive workflow for OpenShift install -failures on **all** platforms: classify the failure, locate artifacts, identify the failed stage, -diagnose root cause. - -For bare metal jobs (name contains "metal" or "baremetal"), you must read [metal.md](metal.md) -(dev-scripts, Ironic, libvirt console logs). - ---- - -## Quick-Start Investigation Workflow - -Follow this sequence for every install failure: - -1. **Parse the job name** → Identify platform, job type, and special modes (§ Job Type Identification) -2. **Download and parse `junit_install.xml`** → Determine the failure stage (§ Failure Stage Classification) -3. **Check symptom labels** → Collect machine-detected environmental context (§ Symptom Labels) -4. **Download installer logs** → `.openshift_install*.log`, excluding deprovision (§ Installer Logs) -5. **Download the log bundle** → find the `log-bundle-*` directory (recent jobs store it exploded) or the `log-bundle-*.tar`, then pull its full contents (§ Log Bundle Analysis) -6. **Analyze based on failure stage** → Route to the correct diagnostic section (§ Stage-Specific Analysis) -7. **Check must-gather availability** → For cluster creation / operator stability failures (§ Must-Gather) -8. **Synthesize root cause** → Combine evidence from all sources (§ Root Cause Determination) - ---- - -## Job Type Identification - -Job names encode the test environment. Parse these patterns to know what to expect. - -### Platform Detection - -| Pattern in Name | Platform | Notes | -|-----------------|----------|-------| -| `aws` | Amazon Web Services | Most common platform | -| `gcp` | Google Cloud Platform | | -| `azure` | Microsoft Azure | | -| `vsphere` | VMware vSphere | | -| `openstack` | Red Hat OpenStack | | -| `metal`, `baremetal` | Bare metal | Route to [metal.md](metal.md) for metal-specific analysis | -| `ovirt` | oVirt/RHV | | -| `nutanix` | Nutanix | | -| `ibmcloud` | IBM Cloud | | -| `powervs` | IBM Power VS | | -| `none` | Platform-agnostic | No platform-specific infrastructure provisioning | - -### Installation Type Detection - -| Pattern in Name | Install Type | Key Characteristics | -|-----------------|-------------|---------------------| -| (default — no special pattern) | IPI (Installer-Provisioned Infrastructure) | Installer manages all cloud resources | -| `upi` | UPI (User-Provisioned Infrastructure) | External infra provisioning, installer configures cluster | -| `assisted` | Assisted Installer | Agent-based installation workflow | -| `single-node`, `sno` | Single-Node OpenShift | All workloads on one node; prone to resource exhaustion | -| `compact` | Compact cluster | 3-node cluster with schedulable control plane | -| `microshift` | MicroShift | Minimal single-node OpenShift variant | - -### Special Mode Detection - -| Pattern in Name | Mode | Implications for Analysis | -|-----------------|------|---------------------------| -| `upgrade` | Upgrade job | Performs a **fresh install first**, then upgrades. Installation failures are still install failures — the upgrade never started | -| `upgrade-from-stable-4.X` | Minor upgrade | Installs previous minor version (e.g., 4.20 for a 4.21 job), then upgrades | -| `upgrade` without `upgrade-from-stable` | Micro upgrade | Installs earlier payload in same stream, then upgrades to newer | -| `fips` | FIPS mode | Watch for crypto library errors, TLS/SSL handshake failures, certificate validation issues, hash algorithm incompatibilities | -| `ipv6` | IPv6 networking | Usually **disconnected** (no internet), uses local mirror registry | -| `dualstack` | Dual-stack IPv4/IPv6 | May be disconnected | -| `techpreview` | TechPreview feature gates | Enables additional feature gates not active in Default clusters. Bootstrap failures may be in TechPreview-gated code paths that won't reproduce in Default clusters | -| `proxy` | HTTP proxy environment | Cluster traffic routed through proxy | -| `rt` | Real-time kernel | Uses real-time RHCOS kernel | -| `ovn` | OVN-Kubernetes networking | Standard CNI plugin | -| `sdn` | OpenShift SDN | Legacy CNI plugin | - -### Upgrade Jobs and Installation Failures - -Jobs with "upgrade" in the name perform a **fresh install first**, then upgrade. If the install -fails, the upgrade never begins — analyze it as a pure **installation failure**. The installed -version is: - -- **Major upgrade** (e.g. 4→5): the newest release of the previous major (a 5.0 job installs the latest 4.x first) -- **Minor upgrade** (`upgrade-from-stable-4.X`): the *previous* minor release (a 4.21 job installs 4.20 first) -- **Micro upgrade** (no `upgrade-from-stable`): an earlier build of the same minor release - ---- - -## Failure Stage Classification - -### Locating junit_install.xml - -`junit_install.xml` is the authoritative source for the failure stage. Location varies by job -config — always search for it: - -```bash -# Search recursively for junit_install.xml -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep "junit_install.xml" -``` - -**`{bucket-path}` is the *specific job run's* path** (e.g. `logs/{job-name}/{build-id}` for -periodics, `pr-logs/pull/{org}_{repo}/{pr}/{job-name}/{build-id}` for presubmits). Always scope -every `gcloud storage` command to this single run — go up a level and you enumerate tens of -thousands of unrelated jobs. - -`install-status.txt` holds only the installer's exit code (a single number); `junit_install.xml` -translates it into a human-readable failure mode. **Always prefer `junit_install.xml`.** - -### Failure Modes - -The JUnit file contains `install should succeed: ` test cases. Failed cases indicate the -failure mode: - -| JUnit Failure Mode | Installation Stage | What Failed | Primary Logs | -|-------------------|-------------------|-------------|--------------| -| `configuration` | Pre-installation | Failed to create install-config.yaml. Extremely rare — validation errors or missing fields | Installer log only (no log bundle exists) | -| `infrastructure` | Infrastructure creation | Failed before creating all cloud resources. Often cloud quota, rate limiting, or API outages | Installer log — cloud API errors (log bundle may be partial or absent) | -| `cluster bootstrap` | Bootstrap phase | Failed to bootstrap the cluster. Bootstrap is typically an ephemeral VM running a temporary kube-apiserver | Log bundle: bootkube.log, etcd.log, kube-apiserver.log | -| `cluster creation` | Operator initialization | One or more operators was unable to stabilize after bootstrap completed | gather-must-gather (if available), installer log | -| `cluster operator stability` | Operator stabilization | Operators never reached stable state (available=True, progressing=False, degraded=False) | Operator-specific logs in must-gather (if available) | -| `other` | Unknown | Unknown install failure — could be any of the above or something novel | Full log analysis of all available artifacts required | - -### Installation Stages in Detail - -The full lifecycle, to target analysis: - -1. **Pre-installation** (Failure mode: `configuration`) - - Validation of `install-config.yaml` - - Credential checks - - Image resolution - - **Common failures**: Invalid install-config, missing required fields, validation errors - -2. **Infrastructure Creation** (Failure mode: `infrastructure`) - - Creating cloud resources (VMs, networks, load balancers, DNS records, storage) - - **Provisioning methods** vary by OpenShift version: - - **Newer versions**: Use **Cluster API (CAPI)** — look for Machine/MachineSet/MachineDeployment errors - - **Older versions**: Use **Terraform** — look for terraform state/apply errors in installer log - - **Common failures**: Quota exceeded, rate limiting, API outages, permission/credential errors - -3. **Bootstrap** (Failure mode: `cluster bootstrap`) - - Bootstrap node boots with temporary control plane - - Bootstrap etcd and kube-apiserver start - - Bootstrap creates master nodes via machine-API or CAPI - - **Common failures**: etcd won't start, API server won't start, bootkube errors, network issues - -4. **Master Node Bootstrap** - - Master nodes boot and join bootstrap etcd cluster - - Masters form permanent control plane - - Bootstrap control plane transfers to masters - - **Common failures**: Masters can't reach bootstrap, network/DNS issues, ignition failures - -5. **Bootstrap Complete** - - Bootstrap node is decommissioned - - Masters run permanent control plane - - Cluster operators begin initialization - - **Common failures**: Control plane not transferring, master nodes not ready - -6. **Cluster Operator Initialization** (Failure mode: `cluster creation`) - - Core cluster operators start deploying - - Operators create their managed workloads - - Initial operator stabilization begins - - **Common failures**: Operators can't deploy, resource conflicts, dependency issues - -7. **Cluster Operator Stabilization** (Failure mode: `cluster operator stability`) - - Operators reach stable state: `available=True`, `progressing=False`, `degraded=False` - - Worker nodes join the cluster - - **Common failures**: Operators stuck progressing, degraded state, availability issues - -8. **Install Complete** - - All cluster operators are available and stable - - Cluster is fully functional - - Installation succeeded - ---- - -## Key Artifact Paths - -### Installer Logs - -`.openshift_install.log` is the single most important artifact — it records every installer step. - -```bash -# Find installer logs (IMPORTANT: exclude deprovision logs) -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep -E "\.openshift_install.*\.log$" | grep -v "deprovision" -``` - -- **Format**: structured log with timestamp, level, message - ```text - time="2025-03-15T10:23:45Z" level=info msg="Creating infrastructure resources..." - time="2025-03-15T10:24:12Z" level=error msg="Failed to create VPC: QuotaExceeded" - ``` -- **Log levels**: `info`, `warning`, `error`, `fatal` -- Deprovision logs (cluster teardown) are NOT relevant to install failures — always exclude them -- Multiple logs may exist (retries, multiple install phases) — download all non-deprovision logs - -### Log Bundle - -The log bundle contains node-level diagnostics collected during or after the install attempt. For -cloud IPI jobs the installer collects it (via the `gather-bootstrap` step); search recursively: - -```bash -# Find log bundles (prefer non-deprovision) — matches the exploded dir and the legacy tarball -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep "log-bundle" -``` - -GCS stores artifacts **decompressed** — CI gunzips everything to sweep for secrets, so bundles are -`log-bundle-*.tar` (never `.tar.gz`). Recent jobs go further and upload the bundle **exploded** as a -`log-bundle-*/` directory instead of a tarball; locate that directory and download its full -contents with `gcloud storage cp -r`. Prefer non-deprovision bundles — they capture the failure -state during installation. - -### Log Bundle Structure - -```text -log-bundle-{timestamp}/ -├── bootstrap/ -│ ├── journals/ -│ │ ├── bootkube.log # Bootstrap control plane initialization (CRITICAL) -│ │ ├── kubelet.log # Bootstrap kubelet -│ │ ├── crio.log # Container runtime -│ │ ├── ironic.log # (Metal only) Ironic provisioning -│ │ ├── metal3-baremetal-operator.log # (Metal only) -│ │ └── journal.log.gz # Complete system journal (gzipped) -│ └── network/ -│ ├── ip-addr.txt # IP addresses -│ ├── ip-route.txt # Routing table -│ └── hostname.txt # Hostname -├── serial/ -│ ├── {cluster}-bootstrap-serial.log # Bootstrap node console -│ └── {cluster}-master-N-serial.log # Master node consoles -├── clusterapi/ -│ ├── etcd.log # etcd cluster logs (CRITICAL) -│ ├── kube-apiserver.log # API server logs (CRITICAL) -│ └── *.yaml # Kubernetes resource definitions -├── control-plane/ -│ └── {node-ip}/ -│ └── containers/ # Control plane container logs -├── failed-units.txt # Failed systemd units (STRONG indicator) -└── gather.log # Log bundle collection process log -``` - -### Other Important Artifacts - -| Artifact | Path Pattern | Purpose | -|----------|-------------|---------| -| `prowjob.json` | `{build-id}/prowjob.json` | Job metadata (target, timing, refs, state) | -| `build-log.txt` | `{build-id}/build-log.txt` | ci-operator orchestration log | -| `metadata.json` | `artifacts/{target}/ipi-install-install/artifacts/metadata.json` | Cluster metadata (cluster ID, infra ID) | -| Ignition configs | `artifacts/{target}/ipi-install-install/artifacts/*.ign` | Machine ignition configurations | -| Symptom labels | `artifacts/job_labels/*.json` | Machine-detected symptom patterns | -| `install-status.txt` | `artifacts/{target}/.../install-status.txt` | Installer exit code (prefer junit_install.xml) | -| Must-gather | `artifacts/{target}/gather-must-gather/artifacts/must-gather.tar` | Cluster state diagnostic archive | - ---- - -## Reading the Installer Log Effectively - -### CRITICAL: Eventual Consistency Behavior - -The single most important concept: installs exhibit **eventual consistency**. Components report -errors while waiting for dependencies to become ready (e.g. ingress waits on networking, which -waits on DNS, which waits on the API server). These intermediate errors are **expected and -normal**, and **early errors often resolve themselves — they are NOT the root cause.** - -### Error Analysis Strategy: Work Backwards - -**Always start at the END of the installer log and work backwards:** - -1. **Read the LAST error/fatal messages** — the most important; they show what was still broken - when the install timed out. -2. **Find the final "Still waiting for..." message** — identifies which component(s) failed to - stabilize (e.g. `"Cluster operators X, Y, Z are not available"`). -3. **Read 10-20 lines of context** around the final errors for the immediate circumstances. -4. **Trace backwards** to the earliest persistent error for the component from step 1. -5. **Ignore early errors that disappeared** — an error at minute 5 not recurring after minute 10 - was transient startup noise, not the root cause. - -**Example**: install fails at 40 min with "kube-apiserver not available"; a "ingress operator -degraded" error at 5 min is likely irrelevant. Focus on what was still broken at the 40-min mark. - -### Key Log Patterns - -#### Final/Fatal Errors -```text -level=fatal msg="Install failed: ..." -level=error msg="Cluster operator authentication Degraded is True..." -level=error msg="context deadline exceeded" -``` - -#### Timeout / Waiting Messages -```text -level=info msg="Still waiting for the cluster to initialize..." -level=info msg="Cluster operators authentication, console, monitoring still initializing" -level=warning msg="Cluster operator X has not yet been available" -``` - -#### Infrastructure Provisioning Errors -```text -level=error msg="Failed to create ...": cloud API errors -level=error msg="Error creating vpc": quota/rate limit -level=error msg="terraform apply" # older versions -level=error msg="clusterapi" # newer versions -``` - -#### Bootstrap Phase Errors -```text -level=info msg="Waiting up to 20m0s for the Kubernetes API..." -level=error msg="Failed waiting for Kubernetes API" -level=info msg="Waiting up to 30m0s for bootstrapping to complete..." -level=error msg="Failed to wait for bootstrapping to complete" -``` - -#### Cluster Operator Status Messages -```text -level=info msg="Cluster operator console Available is False..." -level=error msg="Cluster operators dns, ingress, monitoring are not available" -level=warning msg="Cluster operator X conditions: ..." -``` - -### Timestamp Correlation - -The installer log uses ISO 8601 timestamps (`time="2025-03-15T10:23:45Z"`). Use them to: -- Build an event timeline -- Correlate installer log entries with log bundle entries -- Measure how long components were failing before the timeout -- Determine which error appeared first in a failure chain - ---- - -## Stage-Specific Analysis - -### Configuration Failures - -**Failure mode**: `configuration` in junit_install.xml - -**Extremely rare** — the installer fails to create or validate `install-config.yaml`. - -**What to check** (installer log only — no log bundle exists): -- Validation errors: missing required fields, invalid values -- Credential format errors -- Image resolution failures - -**Common patterns**: -```text -level=fatal msg="failed to fetch Install Config: ..." -level=error msg="invalid install-config.yaml: ..." -``` - -### Infrastructure Failures - -**Failure mode**: `infrastructure` in junit_install.xml - -Occurs before the cluster is created — the installer cannot provision the required cloud resources. - -**What to check**: -- **Primary**: installer log — search for cloud provider API errors -- Log bundle is usually absent or incomplete - -**Common causes by platform**: - -| Platform | Common Errors | Log Pattern | -|----------|--------------|-------------| -| AWS | Quota exceeded, rate limiting, capacity | `RequestLimitExceeded`, `InsufficientInstanceCapacity`, `VcpuLimitExceeded` | -| GCP | Quota exceeded, rate limiting | `QUOTA_EXCEEDED`, `rateLimitExceeded` | -| Azure | Quota exceeded, operation not allowed | `QuotaExceeded`, `OperationNotAllowed` | -| vSphere | Connection timeouts, resource pool | `context deadline exceeded`, connection refused | -| OpenStack | Quota, floating IP exhaustion | `No more IP addresses`, quota errors | - -**Provisioning method** (varies by OpenShift version): -- **Newer**: **Cluster API (CAPI)** — search installer log for `clusterapi`/`machine-api` and - Machine/MachineSet/MachineDeployment errors -- **Older**: **Terraform** — search for `terraform apply` failures and terraform state errors - -**Distinguish from product bugs**: most infra failures are transient (quota, rate limiting, cloud -outages), not product bugs. If the same job succeeds on retry, it was likely transient infra. - -### Bootstrap Failures - -**Failure mode**: `cluster bootstrap` in junit_install.xml - -The most complex failure mode. The bootstrap node is an ephemeral VM running a temporary control -plane (kube-apiserver, etcd) to orchestrate creating the permanent control plane on masters. - -**CRITICAL**: You MUST thoroughly examine the log bundle. Do not guess from a single error — -build a complete timeline. - -**Analysis procedure**: - -1. **Read `bootstrap/journals/bootkube.log` thoroughly** — the most important file. Identify - every process that started, crashed, or errored; note timestamps for a chronological - sequence. For any crashed process (non-zero exit, `ContainerDied`), read its stderr/stdout - in surrounding lines. -2. **For crashed processes**: exit codes tell you *that* it crashed; error output tells you - *why*. Validate the termination reason — check for OOM kills, host restarts, kill-by-signal, - resource limits (consult surrounding logs and infra signals like dmesg/journal entries and - resource utilization). Distinguish software defects from infra/resource-induced terminations. -3. **Pursue errors**: read context, follow references to other components' logs, trace causation - back to the originating failure. The first error is often a symptom, not the cause. -4. **Check supporting logs** (cross-reference timestamps with `bootkube.log`): - - `clusterapi/kube-apiserver.log` — API server startup and errors - - `clusterapi/etcd.log` — etcd cluster formation and member health - - `bootstrap/journals/kubelet.log` — kubelet container management -5. **Check serial console logs**: `serial/{cluster-name}-bootstrap-serial.log` — look for kernel - panics, ignition failures, disk errors, network configuration issues. -6. **Check `failed-units.txt`** — failed systemd units; strong indicator of specific service failures. - -**Common bootstrap failure patterns**: - -| Pattern | Symptom | Root Cause Investigation | -|---------|---------|------------------------| -| etcd won't start | `bootkube.log`: etcd container crashes or restarts | Check `clusterapi/etcd.log`, bootstrap network config, disk performance | -| API server won't start | `bootkube.log`: kube-apiserver crash loops | Check `clusterapi/kube-apiserver.log`, etcd health, certificate issues | -| Masters can't join bootstrap | Masters never appear in etcd member list | Check master serial logs, network connectivity, DNS resolution | -| Bootstrap VM network issues | DHCP failures, unreachable addresses | Check `bootstrap/network/`, serial console log | -| Ignition failure | Bootstrap or master fails to provision | Check serial logs for ignition errors | -| Certificate issues | TLS handshake errors, cert validation fails | Check time sync, cert generation in bootkube.log | -| Disk pressure | Components killed or evicted | Check for OOM, disk full in journal/serial logs | - -### Cluster Creation Failures - -**Failure mode**: `cluster creation` in junit_install.xml - -The cluster bootstrapped, but one or more operators failed to deploy and stabilize. - -**What to check**: - -1. **Installer log** — the final "Cluster operators X, Y, Z are not available" message - identifies which operators failed. -2. **Must-gather availability** — check if `must-gather*.tar` exists: - ```bash - gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-must-gather/artifacts/" 2>&1 \ - | grep "must-gather.*\.tar" - ``` - - **Yes**: extract and examine operator-specific logs. - - **No**: cluster was too unstable to collect diagnostics — rely on installer log and log - bundle only. **Do NOT suggest downloading must-gather if the .tar file doesn't exist.** -3. **Operator logs in must-gather** (if available): degraded operators and their conditions, - operator-specific logs for the failing operators, resource conflicts or dependency issues. -4. **Log bundle** — may add context on operator deployment. - -### Cluster Operator Stability Failures - -**Failure mode**: `cluster operator stability` in junit_install.xml - -Like cluster creation, but operators reached a partially running state and couldn't fully stabilize. - -**What to check**: -- Operators with `available=False`, `progressing=True`, or `degraded=True` -- Operator logs for stuck operations -- Time-series of operator status changes; operators flipping between states -- Must-gather availability before suggesting to review it (same as cluster creation) - -**Common operator stability failure patterns**: - -| Operator | Common Issues | Where to Look | -|----------|--------------|---------------| -| `authentication` | OAuth server issues, identity provider config | OAuth pods, authentication operator logs | -| `console` | Console deployment fails, route issues | Console namespace pods | -| `ingress` | Router pods not ready, DNS issues | Ingress controller logs, DNS operator | -| `monitoring` | Prometheus/alertmanager startup | Monitoring namespace, PVC issues | -| `network` | OVN/SDN initialization | Network operator, ovnkube pods | -| `dns` | CoreDNS pods not ready | DNS operator logs | -| `image-registry` | Storage backend issues | Image registry operator, PVC | -| `kube-apiserver` | API server rollout issues | Kube-apiserver operator logs | -| `etcd` | Member health, quorum | etcd operator, etcd pod logs | -| `machine-config` | `RequiredPoolsFailed`; bootstrap-vs-in-cluster rendered MC mismatch | MCD pod logs (`gather-extra/artifacts/pods/openshift-machine-config-operator_machine-config-daemon-*.log`) | - -**machine-config degraded — rendered-MachineConfig mismatch**: when the installer says -`Cluster operator machine-config Degraded is True with RequiredPoolsFailed` and node -annotations show `bootstrap generated MC rendered-- and in-cluster generated -MC rendered-- for this node do not match`, the bootstrap Machine Config -Server rendered a different config than the in-cluster controller, so the pool can never -reconcile. Open the machine-config-daemon log and find the -`Bootstrap generated MC ... vs In-cluster generated MC ... diffs:` block: the granular -summary line (`&{osUpdate:... kargs:... fips:... passwd:... files:... units:... ...}`) -says which categories diverge, and the raw diff that follows shows the content. Read the -diff to determine WHAT diverged, then trace which rendering input differed between the -bootstrap and in-cluster renders — the two should be identical given identical inputs, so -the divergent content points at the input or component that skewed. This is an OS-layer -signal per the SKILL's evidence check: read -[operating-system-changes.md](../operating-system-changes.md) alongside (the granular -summary's `osUpdate` field records whether the OS image itself diverged). - -### Other / Unknown Failures - -**Failure mode**: `other` in junit_install.xml - -Catch-all for failures that don't match a known pattern. - -**What to check**: -- Comprehensive analysis of ALL available logs -- Installer log for any error/fatal messages -- Log bundle if available -- Unusual patterns or timeout messages -- Whether the failure matches a known pattern but was misclassified - ---- - -## Common Install Failure Categories - -### API Server Not Coming Up - -**Symptoms**: `"Waiting for Kubernetes API"` timeout, kube-apiserver crash loops in bootkube.log - -**Investigation**: -- Check `clusterapi/kube-apiserver.log` for startup errors -- Check etcd health — API server depends on etcd -- Verify certificates and TLS configuration -- Check bootstrap network connectivity -- Look for resource exhaustion (OOM kills) - -### etcd Bootstrap Failures - -**Symptoms**: etcd container crashes in bootkube.log, "etcd cluster unhealthy" messages - -**Investigation**: -- Check `clusterapi/etcd.log` for member formation errors -- Verify network connectivity between bootstrap and master nodes -- Check disk I/O performance (etcd is latency-sensitive) -- Look for DNS resolution failures preventing member discovery - -### Machine Provisioning Failures - -**Symptoms**: "machine has not yet been provisioned", machine stuck in provisioning state - -**Investigation**: -- Installer log for CAPI/machine-api errors -- Cloud provider API errors (quota, capacity) -- For metal: Ironic provisioning logs (see [metal.md](metal.md)) -- Security group / network configuration issues - -### Certificate Issues - -**Symptoms**: TLS handshake errors, x509 certificate verification failures, "certificate has expired" - -**Investigation**: -- Check time synchronization between nodes -- Look for cert generation errors in bootkube.log -- Check if FIPS mode is affecting certificate handling -- Review cert-manager or cert rotation operator logs - -### DNS Resolution Failures - -**Symptoms**: "no such host", "DNS lookup failed", unresolved service names - -**Investigation**: -- Check CoreDNS pod status -- Verify DNS operator health -- Check `bootstrap/network/` for DNS configuration -- For disconnected environments: verify local DNS resolver setup -- For IPv6: check AAAA record availability - -### Network Connectivity Issues - -**Symptoms**: "connection refused", "connection timed out", "no route to host" - -**Investigation**: -- Check `bootstrap/network/ip-addr.txt` and `ip-route.txt` -- Verify security group / firewall rules on cloud platform -- Check for MTU mismatches (especially with overlay networks) -- For IPv6/dualstack: verify IPv6 addressing and routing -- For proxy environments: check proxy configuration - -### Cloud Credential Issues - -**Symptoms**: "AccessDenied", "AuthFailure", "Forbidden", IAM errors - -**Investigation**: -- Check installer log for cloud API authentication errors -- Verify service account / IAM role configuration -- Check if credentials expired during long-running installation -- See [cloud-provider-errors.md](../cloud-provider-errors.md) for platform-specific patterns - -### Image Pull Failures During Install - -**Symptoms**: `ImagePullBackOff`, `ErrImagePull`, "failed to pull image" - -**Investigation**: -- Check if the image repository is accessible -- For disconnected environments: verify mirror registry configuration -- Check for image tag resolution failures -- Verify pull secrets are correctly configured -- Check registry certificate trust - -### Operator Timeout During Install - -**Symptoms**: "context deadline exceeded", operators not reaching available=True - -**Investigation**: -- Identify which specific operators failed (from installer log final messages) -- Check operator-specific logs in must-gather (if available) -- Look for dependency chains: operator A waiting on operator B -- Check for resource exhaustion preventing pod scheduling -- Verify all required images are pullable - ---- - -## Symptom Labels - -The CI system may attach **symptom labels** — machine-detected patterns stored as JSON artifacts -in `job_labels/` that provide environmental context. - -### Locating Symptom Labels - -Download the `job_labels/*.json` files with the commands in the -[`job_labels/` section of artifacts.md](../artifacts.md#job_labels--symptom-labels). - -### Using Symptom Labels - -Each JSON file describes a detected symptom with a summary and explanation. - -**IMPORTANT**: symptoms are **environmental observations**, NOT definitive root causes. They -inform your investigation but you must still do thorough root cause analysis. For example, "test -failures during high CPU events" means CPU pressure was detected during the failure window — it -could explain the failure absent another root cause, but is not proof of causation. - -Include symptom labels in the "Known Symptoms Seen" section of your report. - ---- - -## Must-Gather Analysis for Install Failures - -Must-gather provides cluster-state diagnostics captured after the install attempt. It is only -relevant to `cluster creation` and `cluster operator stability` failures, where the cluster came -up far enough to be partially operational; for `configuration`, `infrastructure`, and `cluster -bootstrap` failures it is normally absent. If no `must-gather*.tar` exists, collection failed -because the cluster was too unstable — say so rather than suggesting a download. - -For locating, downloading, and extracting must-gather, and what to look for inside it, use the -must-gather reference: [artifacts.md § Must-Gather Archives](../artifacts.md#must-gather-archives). - ---- - -## Log Bundle Analysis Workflow - -### When Log Bundles Are Available - -Log bundles are produced once the install reaches the bootstrap phase or later. **Not available** -for `configuration` or early `infrastructure` failures. - -| Failure Mode | Log Bundle Available? | -|--------------|----------------------| -| `configuration` | No | -| `infrastructure` | Partial or No | -| `cluster bootstrap` | Yes (primary diagnostic source) | -| `cluster creation` | Yes | -| `cluster operator stability` | Yes | -| `other` | Varies | - -### Downloading and Extracting the Log Bundle - -```bash -# Exploded form (recent jobs): copy the whole log-bundle-*/ directory — no extraction needed -gcloud storage cp -r "{gcs-path-to-log-bundle-dir}" \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - -# Legacy tarball form: download, then extract (GCS stores it decompressed as .tar) -gcloud storage cp {gcs-path-to-log-bundle} \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled -tar -xf .work/prow-job-analysis/{build_id}/logs/log-bundle-*.tar \ - -C .work/prow-job-analysis/{build_id}/logs/ -``` - -### Analysis by Failure Mode - -Per-mode procedures live in [§ Stage-Specific Analysis](#stage-specific-analysis). The one -log-bundle-specific step: for **`cluster bootstrap`**, read `bootkube.log` and build a timeline, -cross-reference `etcd.log` / `kube-apiserver.log` / `kubelet.log`, check serial consoles for -kernel panics / Ignition failures, and check `failed-units.txt`. - ---- - -## Distinguishing Install Failure from Test Failure - -Distinguish "installation failed" from "installation succeeded but tests failed": - -| Signal | Installation Failed | Installation Succeeded, Tests Failed | -|--------|--------------------|------------------------------------| -| JUnit XML | `install should succeed: ` test is **failed** | `install should succeed` test is **passed** (or absent) | -| Build log | "Install failed" or fatal installer error | Install completes, test steps execute and fail | -| Installer log | Contains `level=fatal` | Contains `level=info msg="Install complete!"` | -| CI step | Failure in `ipi-install-install` or similar install step | Failure in `openshift-e2e-test` or similar test step | - -If installation **succeeded** but tests failed, use the test failure analysis workflow instead -(see this skill's test failure references, e.g. `test-extension-binaries.md`). - ---- - -## Routing to Metal-Specific Analysis - -When the job name contains `metal` or `baremetal`, consult [metal.md](metal.md), which covers: - -- **OFCIR host acquisition** — whether a bare metal host was acquired -- **Dev-scripts logs** — setup before OpenShift install (dev-scripts invoke the installer, so - installer logs appear in devscripts directories) -- **Ironic/Metal3 provisioning** — BMC communication, node registration, power management - - Bootstrap Ironic logs (master provisioning): `bootstrap/journals/ironic.log` - - Control-plane Ironic logs (worker provisioning): `control-plane/{ip}/containers/metal3-ironic-*.log` -- **libvirt console logs** — VM boot sequence, kernel panics, ignition failures -- **sosreport** — hypervisor-level system diagnostics -- **squid proxy logs** — CI inbound access to the cluster (NOT outbound cluster access) - -**Disconnected metal network architecture**: the hypervisor (dev-scripts host) HAS full internet -access; only the cluster VMs/nodes are disconnected. If downloads fail during dev-scripts setup, -check whether the remote service is down — it's not a network restriction. - -This document still applies to metal jobs for the cluster-level failure; metal.md adds the -infrastructure layer below the installer. - ---- - -## Root Cause Determination - -### Synthesizing Evidence - -Combine all sources to determine the root cause: - -1. **Installer log** — what the installer saw and reported -2. **Log bundle** — node-level detail of what happened during bootstrap -3. **Must-gather** — cluster state after partial install (if available) -4. **Symptom labels** — environmental context (informational, not causal) -5. **JUnit XML** — the classified failure stage - -### Prioritizing Evidence - -- **Final errors are most important** — what was still broken at timeout -- **Persistent errors matter** — errors appearing throughout the log -- **Transient early errors usually don't matter** — eventual consistency -- **Infrastructure errors are usually not product bugs** — quota, rate limiting, outages -- **Operator dependency chains** — trace from symptomatic operator to root-cause operator - -### Common Root Cause Patterns - -| Observation | Likely Root Cause | Verification | -|-------------|-------------------|-------------| -| Cloud API errors in installer log | Quota/capacity/outage | Check cloud provider status, retry the job | -| etcd won't form cluster | Network issues between nodes | Check bootstrap network config, DNS | -| Multiple operators degraded | Upstream dependency failed | Find the root operator in the dependency chain | -| Image pull failures | Registry unreachable or image missing | Check pull secret, registry connectivity | -| Pods evicted or OOMKilled | Resource exhaustion | Check node resource utilization | -| TLS/cert errors in FIPS job | FIPS crypto incompatibility | Check FIPS-specific code paths | -| Bootstrap timeout, no errors | Possible network/DNS issue | Check serial logs, network config | -| Single operator degraded | Operator-specific bug or config | Check operator logs, recent code changes | - ---- - -## Analysis Report Template - -Use this structure when generating install failure analysis reports: - -```text -OpenShift Installation Failure Analysis -======================================== - -Job: {job-name} -Build ID: {build_id} -Platform: {aws/gcp/azure/etc.} -Job Type: {IPI/UPI/SNO/etc.} -Prow URL: {original-url} - -Failure Stage: {stage from junit_install.xml} - -Known Symptoms Seen ---------------------- -(Only include if symptom labels were found — omit section entirely if none) -- {symptom summary}: {symptom explanation} -Note: Symptoms are machine-detected environmental observations, not -definitive causes. - -Summary -------- -{High-level summary of the failure — 2-3 sentences} - -Installer Log Analysis ----------------------- -{Key findings from installer log} - -Final Error: -{Last error/fatal message with timestamp} - -Last Waiting Message: -{Final "Still waiting for..." or "Cluster operators X, Y, Z are not available"} - -Context: -{10-20 lines surrounding the final error} - -Log Bundle Analysis -------------------- -{Findings from log bundle analysis} - -Failed Units: -{List from failed-units.txt, if any} - -Key Journal Errors: -{Important errors from bootkube.log, etcd.log, kube-apiserver.log} - -Root Cause ----------- -{The identified root cause with supporting evidence} - -Recommended Next Steps ----------------------- -{Actionable debugging steps based on failure mode} - -Artifacts Location ------------------- -All artifacts downloaded to: -.work/prow-job-analysis/{build_id}/logs/ - -- Installer logs: .openshift_install*.log -- Log bundle: log-bundle-*/ -- Must-gather: must-gather/ (if available) -``` - ---- - -## Tips and Best Practices - -### Critical Rules - -1. **ALWAYS work backwards from the end of the installer log** — not forwards. Early errors are - usually transient (eventual consistency). -2. **Never suggest downloading must-gather unless you verified the .tar file exists.** If absent, - the cluster was too unstable to collect diagnostics. -3. **Bootstrap failures REQUIRE log bundle analysis** — the installer log alone is insufficient. -4. **Distinguish infrastructure failures from product bugs.** Quota, rate limiting, and API - outages are transient. If the same job succeeds on retry, it was likely infrastructure. -5. **Separate install failures from test failures.** If the `install should succeed` case passed, - the install succeeded and the failure is in the test phase. - -### Investigation Efficiency - -- **Start with `junit_install.xml`** to identify the failure stage — it guides which logs to prioritize -- **Use grep liberally** on large logs — don't read multi-thousand-line logs sequentially -- **Cache artifacts** in `.work/prow-job-analysis/{build_id}/` for re-analysis -- **Exclude deprovision logs/bundles** — they're from teardown, not install -- **Check `failed-units.txt`** — a strong, quick indicator of what went wrong - -### Job Name Clues - -- **FIPS**: check crypto-related errors first -- **IPv6/disconnected**: check mirror registry accessibility, DNS resolution -- **Single-node (SNO)**: check resource exhaustion (all workloads on one node) -- **TechPreview**: bootstrap failures may be in TechPreview-gated code paths that won't reproduce - in Default clusters -- **Upgrade**: if install fails, the upgrade never started — analyze as a pure install failure - -### File Format Notes - -| File Type | Format | Notes | -|-----------|--------|-------| -| Installer log | Structured text (`time=... level=... msg=...`) | Primary analysis source | -| Journal logs | systemd journal format (plain text export) | From log bundle `bootstrap/journals/` | -| Serial logs | Raw console output | Boot sequence, kernel messages | -| YAML files | Kubernetes resource definitions | From `clusterapi/` in log bundle | -| `.gz` files | gzip compressed | Some journal logs are gzipped | -| `.tar` files | tar archive (NOT gzip) | Log bundles are `.tar`, not `.tar.gz` | - ---- - -## Failure Mode Quick Reference - -| JUnit Failure Mode | Installation Stage | Where to Look First | Log Bundle? | Must-Gather? | -|-------------------|-------------------|---------------------|-------------|-------------| -| `configuration` | Pre-installation | Installer log only | No | No | -| `infrastructure` | Infrastructure creation | Installer log — cloud API errors | Partial/No | No | -| `cluster bootstrap` | Bootstrap | Log bundle — bootkube, etcd, kube-apiserver | Yes (primary) | No | -| `cluster creation` | Operator initialization | Must-gather (if available), installer log | Yes | Maybe | -| `cluster operator stability` | Operator stabilization | Must-gather (if available), operator logs | Yes | Maybe | -| `other` | Unknown | All available logs | Varies | Varies | diff --git a/plugins/ci/skills/prow-job-analysis/references/install/metal.md b/plugins/ci/skills/prow-job-analysis/references/install/metal.md deleted file mode 100644 index d94116616..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/install/metal.md +++ /dev/null @@ -1,971 +0,0 @@ -# Install Failure Analysis — Metal (Bare Metal) - -**Use when** the job name contains `metal`, `baremetal`, or `baremetalds` (a bare-metal `sno`/`single-node` job also belongs here, but only when a metal keyword is *also* present — a cloud SNO job like `e2e-aws-ovn-single-node` is not metal; the SKILL.md routing table is authoritative when the signals conflict), the job uses dev-scripts with Metal3/Ironic for provisioning, or you need console-log or sosreport analysis. Metal-specific supplement to [general.md](general.md), covering bare metal install failures that use dev-scripts, Metal3, and Ironic. Use **both** files for metal jobs: [general.md](general.md) owns the standard installer workflow (reading installer logs, log-bundle analysis, failure-stage identification); this file owns the metal layers on top. - -## When to Use This Reference - -Use when ANY of these are true: -- Job name contains `metal`, `baremetal`, or `baremetalds` (a `sno`/`single-node` job qualifies only when a metal keyword is also present; a cloud SNO job routes to Single-node per the SKILL.md table, not here) -- Job uses dev-scripts with Metal3/Ironic for provisioning -- Debugging OFCIR host acquisition, Ironic provisioning, or dev-scripts setup -- Console logs or sosreport analysis is needed - -If the job name has no metal keyword, use [general.md](general.md) only. - ---- - -## Metal Installation Stack - -Metal IPI (Installer-Provisioned Infrastructure) jobs have distinct layers; failures can occur at any of them. Identifying which layer failed is the first step in analysis. - -```text -┌─────────────────────────────────────────────────────┐ -│ Layer 3: OpenShift Installation │ -│ Standard installer runs on provisioned nodes │ -│ → Use general.md techniques for this layer │ -├─────────────────────────────────────────────────────┤ -│ Layer 2: Ironic / Metal3 Provisioning │ -│ Provisions bare metal nodes (or VMs acting as BM) │ -│ → Ironic logs, BareMetalHost resources │ -├─────────────────────────────────────────────────────┤ -│ Layer 1: dev-scripts Setup │ -│ Configures hypervisor, networking, Ironic, builds │ -│ the installer binary │ -│ → Dev-scripts numbered logs (01-05) │ -├─────────────────────────────────────────────────────┤ -│ Layer 0: OFCIR Host Acquisition │ -│ Acquires a physical or virtual host from the CI │ -│ resource pool before anything else runs │ -│ → OFCIR build-log.txt, junit_metal_setup.xml │ -└─────────────────────────────────────────────────────┘ -``` - -### Key Components - -- **dev-scripts** ([openshift-metal3/dev-scripts](https://github.com/openshift-metal3/dev-scripts)): - Framework for setting up and installing OpenShift on bare metal. Configures the - hypervisor, sets up networking, starts Ironic/Metal3 services, builds the installer, - and orchestrates the install. - -- **Metal3**: Kubernetes-native interface to Ironic. Manages `BareMetalHost` custom - resources representing physical or virtual machines. - -- **Ironic**: OpenStack bare metal provisioning service. Handles node enrollment, - inspection, and deployment lifecycle. - -- **OFCIR** (OpenShift Fleeting CI Resources): Acquires baremetal hosts from - infrastructure provider pools for CI jobs. - ---- - -## Metal CI Job Identification - -### Job Naming Patterns - -| Pattern in Job Name | Meaning | -|---------------------|---------| -| `metal` | Generic bare metal job | -| `baremetal` | Bare metal job (synonym) | -| `baremetalds` | Bare metal using dev-scripts | -| `metal-ipi` | Metal Installer-Provisioned Infrastructure | -| `metal-ipi-ovn-ipv6` | Metal IPI with IPv6-only networking (disconnected) | -| `metal-ipi-ovn-dualstack` | Metal IPI with dual-stack networking | -| `sno` | Single-Node OpenShift (often on metal) | -| `metal-assisted` | Metal using Assisted Installer | -| `metal-ipi-serial` | Metal IPI with serial console capture | - -**SNO Bootstrap-in-Place** (`sno`/`sno_bip`): the single node installs with **no Ironic and -no bootstrap VM** — it boots directly from the RHCOS image. Skip the Ironic-provisioning and -bootstrap-node branches below when analyzing these jobs. - -### Examples - -```text -periodic-ci-openshift-release-master-ci-4.19-e2e-metal-ipi-ovn-ipv6 -periodic-ci-openshift-release-master-nightly-4.19-e2e-metal-ipi-ovn-dualstack -periodic-ci-openshift-release-master-ci-4.19-e2e-metal-ipi-serial-ovn-ipv6 -``` - ---- - -## Network Architecture - -**CRITICAL**: "disconnected" in metal CI jobs refers to the **cluster nodes**, NOT the -hypervisor. This is a common misunderstanding in metal failure analysis. - -### Hypervisor (dev-scripts host) - -- **HAS** full internet access -- Downloads packages, container images, and dependencies from the public internet -- Runs dev-scripts Ansible playbooks that download tools (Go, installer, etc.) -- Hosts a local mirror registry to serve images to cluster nodes -- Runs the Squid proxy for inbound CI access to the cluster - -### Cluster VMs/Nodes - -- Run in a **private network** (often IPv6-only when `IP_STACK=v6`) -- **NO** direct internet access (truly disconnected) -- Pull container images from the hypervisor's local mirror registry -- Access to hypervisor services only (registry, DNS, DHCP, etc.) - -### Network Topology Diagram - -```text -┌──────────────────────────────────────────────────────────────┐ -│ CI Infrastructure (Prow) │ -│ │ -│ ┌──────────┐ Squid Proxy ┌──────────────────────┐ │ -│ │ CI Tests ├──────(inbound)──────►│ Cluster API │ │ -│ └──────────┘ │ (on cluster nodes) │ │ -│ └──────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────────┐ │ -│ │ Hypervisor (dev-scripts host) │ │ -│ │ ✅ Full internet access │ │ -│ │ │ │ -│ │ ┌──────────────┐ ┌──────────────┐ ┌─────────────┐ │ │ -│ │ │ Mirror │ │ Ironic │ │ dnsmasq │ │ │ -│ │ │ Registry │ │ Services │ │ (DHCP/DNS) │ │ │ -│ │ └──────┬───────┘ └──────┬───────┘ └──────┬──────┘ │ │ -│ │ │ │ │ │ │ -│ │ ═══════╪═════════════════╪══════════════════╪═══════ │ │ -│ │ │ Provisioning Network │ │ │ -│ │ ═══════╪═════════════════╪══════════════════╪═══════ │ │ -│ │ │ Baremetal Network │ │ │ -│ │ │ │ │ │ │ -│ │ ┌──────┴──┐ ┌──────────┴──┐ ┌───────────┴──┐ │ │ -│ │ │Bootstrap│ │ Master-0 │ │ Master-1 │ │ │ -│ │ │ Node │ │ │ │ │ │ │ -│ │ └─────────┘ └─────────────┘ └──────────────┘ │ │ -│ │ ❌ No internet (disconnected) │ │ -│ └────────────────────────────────────────────────────────┘ │ -└──────────────────────────────────────────────────────────────┘ -``` - -### Common Misconception — AVOID - -- ❌ **WRONG**: "The hypervisor cannot access the internet, so downloads fail" -- ✅ **CORRECT**: "The hypervisor has internet access. If downloads fail, it's likely - the remote service is unavailable, not network restrictions." - -### Implications for Failure Analysis - -| Failure Location | What's Happening | Root Cause Direction | -|------------------|-----------------|---------------------| -| Dev-scripts (steps 01-05), download failures | Hypervisor can't download from internet | Remote service/URL is down or resource removed upstream | -| Dev-scripts, HTTP 403/404 | External resource not found | Check the resource upstream — moved, removed, or access denied (not a network restriction) | -| Installation (step 06+), image pull failures | Cluster nodes can't pull images | Check local mirror registry on hypervisor | -| Cluster nodes, DNS failures | Nodes can't resolve names | Check dnsmasq on hypervisor, not internet DNS | - ---- - -## OFCIR (OpenShift Fleeting CI Resources) - -OFCIR manages the pool of bare metal hosts for CI jobs. Before installation begins, the -job must acquire a host from an OFCIR pool. - -### How OFCIR Works - -1. Job requests a host from a specific pool (e.g., `cipool-ironic-cluster-el9`) -2. OFCIR checks pool availability and allocates a host -3. Host details (name, provider, IP) are returned to the job -4. Dev-scripts then provisions OpenShift on the acquired host - -### OFCIR Artifact Locations - -```text -{target}/ofcir-acquire/ -├── build-log.txt # Acquire step stdout: pool, provider, host -└── artifacts/ - └── junit_metal_setup.xml # JUnit test result for host acquisition -``` - -### Checking OFCIR Acquisition - -```bash -# Download OFCIR logs -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/ofcir-acquire/build-log.txt" \ - ./ofcir-build-log.txt --no-user-output-enabled 2>&1 - -# Download JUnit result -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/ofcir-acquire/artifacts/junit_metal_setup.xml" \ - ./junit_metal_setup.xml --no-user-output-enabled 2>&1 -``` - -### Interpreting OFCIR Results - -**Check `junit_metal_setup.xml` first** — look for the test case: -```text -[sig-metal] should get working host from infra provider -``` - -If this test **failed**, OFCIR could not acquire a host and **installation never -started**. Stop here — the failure is purely infrastructure acquisition. - -**Read `build-log.txt`** — this is the acquire step's stdout, not a JSON document. Grep it -for the pool the job requested, the backing provider, and the host it received: - -```bash -grep -iE "pool|provider|acquired|cir" ./ofcir-build-log.txt -``` - -Key facts to extract: -- **Pool** — the OFCIR pool the job requested (e.g., `cipool-ironic-cluster-el9`, - `cipool-ibmcloud`), set by the job's ci-operator config -- **Provider** — infrastructure provider backing the pool (`ironic`, `equinix`, `aws`, - `ibmcloud`) -- **Host** — the specific host allocated (referenced later in dev-scripts and Ironic logs) - -### Common OFCIR Failure Patterns - -| Pattern | Symptoms | Likely Cause | -|---------|----------|-------------| -| Pool exhaustion | `junit_metal_setup.xml` fails, no host allocated | All hosts in pool are busy or unhealthy | -| Host acquisition timeout | Test fails after waiting | Pool capacity issue or OFCIR service problem | -| Stale host | Host acquired but immediately unusable | Previous job's deprovision left the host dirty/unusable | -| Provider error | Provider-specific error in build log | Infrastructure provider issue (e.g., Equinix API down) | - -**If OFCIR fails**: Report that installation never started. Include pool name and -provider. Suggest checking OFCIR pool health and provider status. - ---- - -## Dev-Scripts Environment - -### What dev-scripts Does - -Dev-scripts sets up the entire bare metal environment as a series of numbered steps on -the hypervisor: - -| Step | Script | Purpose | -|------|--------|---------| -| 01 | `01_install_requirements.sh` | Install packages, dependencies | -| 02 | `02_configure_host.sh` | Configure hypervisor networking, storage, libvirt | -| 03 | `03_build_installer.sh` | Build or download the OpenShift installer binary | -| 04 | `04_setup_ironic.sh` | Start Ironic/Metal3 services, configure BMC | -| 05 | `05_create_install_config.sh` | Generate install-config.yaml | -| 06 | `06_create_cluster.sh` | Run the installer to create the cluster | - -**Critical distinction**: -- Failures in steps **01-05** = dev-scripts setup problem (not an OpenShift install failure) -- Failures in step **06** = actual cluster installation failure (analyze with - [general.md](general.md) techniques plus metal-specific artifacts) - -### Dev-Scripts Log Location - -```text -{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/ -``` - -One log file per step. **Dev-scripts invokes the installer**, so `.openshift_install*.log` -files also appear in this directory tree. - -### How to Download Dev-Scripts Logs - -```bash -gcloud storage cp -r \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/" \ - ./devscripts-logs/ --no-user-output-enabled -``` - -### Dev-Scripts Setup Failure Patterns - -| Step | Common Failures | What to Look For | -|------|----------------|------------------| -| 01 (requirements) | Package install failures, dependency conflicts | `yum`/`dnf` errors, missing repos | -| 02 (host config) | Network bridge setup, libvirt configuration | Bridge errors, IP conflicts, libvirt XML errors | -| 03 (build installer) | Go build errors, download failures | Compiler errors, HTTP 404/403 on downloads | -| 04 (Ironic setup) | BMC connectivity, Ironic container startup | Container pull failures, port conflicts, cert errors | -| 05 (install-config) | Validation errors, missing credentials | Config validation messages | -| 06 (create cluster) | Full installation failure | See installer log analysis ([general.md](general.md)) | - -### VM-Based Metal Simulation - -In CI, "bare metal" nodes are actually **VMs managed by libvirt/QEMU** on the -hypervisor: - -- **libvirt** manages VM lifecycle (create, start, stop, destroy) -- **QEMU** provides hardware emulation -- **Virtual BMC (vBMC)** simulates IPMI/Redfish BMC for each VM -- VMs are given virtual disks, NICs, and console output - -Some failures originate in this simulation layer, not in OpenShift or Ironic. See -[Hardware Simulation Issues](#hardware-simulation-issues) below. - -### Network Architecture in Dev-Scripts - -Dev-scripts configures two networks on the hypervisor: - -1. **Provisioning network** (`provisioning` bridge): - - Used by Ironic for PXE boot and node inspection - - dnsmasq provides DHCP and TFTP on this network - - Nodes PXE boot from this network during initial provisioning - -2. **Baremetal network** (`baremetal` bridge): - - Primary cluster network - - Used for all cluster communication after provisioning - - dnsmasq provides DHCP and DNS on this network - - External bridge for cluster connectivity - -3. **External bridge** (optional): - - Connects the baremetal network to the hypervisor's external network - - Enables inbound CI access to the cluster - -### Squid Proxy - -Dev-scripts runs a Squid proxy on the hypervisor for **inbound** CI access — the CI test -framework connecting to the cluster API through the proxy. - -**Important**: Squid is for **inbound** access (CI → cluster), NOT outbound -(cluster → registry). Cluster nodes access the mirror registry directly on the -hypervisor's internal network. - ---- - -## Ironic and Metal3 Provisioning - -### Ironic Service Architecture - -Ironic runs as a set of containers on the bootstrap node (for master provisioning) -and later on the control plane (for worker provisioning): - -| Service | Role | -|---------|------| -| `ironic` | Main provisioning service — manages node lifecycle | -| `ironic-inspector` | Hardware inspection service — discovers node hardware | -| `ironic-httpd` | HTTP server for serving boot images and configs | -| `dnsmasq` | DHCP/TFTP server for PXE boot on provisioning network | -| `metal3-baremetal-operator` | Kubernetes operator managing BareMetalHost resources | - -### BareMetalHost Provisioning Lifecycle - -Ironic manages nodes through a state machine. These states pinpoint where provisioning -stalled: - -```text -enroll → manageable → inspecting → available → provisioning → deploying → active - │ │ │ │ │ │ - │ │ │ │ │ └─ Node is running - │ │ │ │ └─ Writing image to disk - │ │ │ └─ Ready for deployment - │ │ └─ Discovering hardware (CPU, RAM, disks, NICs) - │ └─ BMC credentials verified, node is manageable - └─ Initial registration with Ironic -``` - -**Failure transitions** (any state can go to): -```text -→ inspect failed (inspection couldn't complete) -→ deploy failed (image write or boot failed) -→ clean failed (disk cleaning failed) -→ error (generic error state) -``` - -### Where to Find Ironic Logs - -**CRITICAL**: There are TWO sets of Ironic logs in different locations. Check the right -set based on what failed. - -#### Bootstrap Ironic Logs (Master Provisioning) - -During bootstrap, Ironic runs on the bootstrap node and provisions the masters. - -```text -log-bundle-*/bootstrap/journals/ironic.log -log-bundle-*/bootstrap/journals/metal3-baremetal-operator.log -``` - -**Check these when**: Master nodes failed to provision, bootstrap phase failures. - -#### Control-Plane Ironic Logs (Worker Provisioning) - -After bootstrap, Ironic moves to the control plane and provisions workers. - -```text -log-bundle-*/control-plane/{node-ip}/containers/metal3-ironic-*.log -log-bundle-*/control-plane/{node-ip}/containers/metal3-baremetal-operator-*.log -``` - -**Check these when**: Worker nodes failed to provision, post-bootstrap failures. - -#### Decision Matrix - -| What Failed | Which Ironic Logs to Check | -|-------------|---------------------------| -| Masters failed to provision | `bootstrap/journals/ironic.log` | -| Workers failed to provision | `control-plane/{ip}/containers/metal3-ironic-*.log` | -| Unsure which failed | Check ALL Ironic logs | -| Bootstrap timeout with no masters | `bootstrap/journals/ironic.log` first | - -### How to Find and Read Ironic Logs - -```bash -# Download and extract log bundle — matches the exploded dir and the legacy tarball -# (CI gunzips everything, so tarballs are .tar, never .tar.gz; recent jobs upload the -# bundle exploded as a log-bundle-*/ directory — download it with gcloud storage cp -r) -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 \ - | grep "log-bundle" -gcloud storage cp {log-bundle-path} ./log-bundle.tar --no-user-output-enabled -tar -xf ./log-bundle.tar - -# Find bootstrap Ironic logs (master provisioning) -find . -path "*/bootstrap/journals/ironic.log" -find . -path "*/bootstrap/journals/metal3-baremetal-operator.log" - -# Find control-plane Ironic logs (worker provisioning) -find . -path "*/control-plane/*/containers/metal3-ironic-*.log" -find . -path "*/control-plane/*/containers/metal3-baremetal-operator-*.log" -``` - -### Mapping Node UUIDs to BareMetalHost Names - -Ironic logs reference nodes by UUID (e.g., `b7fa5b83-91d0-46ee-acd2-e4b33e9ac983`), -but operators and kubectl output use BareMetalHost names. Correlate them: - -```bash -# Search installer log for UUID-to-name mapping -grep -E "uuid|BareMetalHost" .openshift_install*.log - -# Search Ironic logs for node registration -grep "Registering" ironic.log | head -20 - -# Look for mapping in must-gather BareMetalHost YAML -grep -r "uid:" baremetalhosts/ -``` - -### Common Ironic Failure Patterns - -| Pattern | Log Message | Cause | -|---------|------------|-------| -| BMC connection failure | `IPMI Error`, `Redfish connection refused` | BMC unreachable or wrong credentials | -| Inspection timeout | `Timeout waiting for node inspection` | Node didn't respond during hardware discovery | -| Deploy failure | `Deploy failed`, `Image write error` | Disk image couldn't be written to node | -| Power management error | `Failed to get power state`, `Power action failed` | Virtual BMC or IPMI issue | -| SSL/TLS error | `SSL: CERTIFICATE_VERIFY_FAILED` | Certificate mismatch between Ironic components | -| Node stuck registering | `BareMetalHost stuck in registering` | BMC communication failure during registration | -| Provisioning state stuck | `Node {uuid} stuck in {state}` | State machine transition blocked | - -### Key Ironic Log Patterns to Search - -```bash -# BMC errors -grep -i "ipmi\|redfish\|bmc" ironic.log | grep -i "error\|fail\|timeout" - -# Provisioning state changes -grep "provision_state" ironic.log - -# Node registration -grep -i "register\|enroll" ironic.log - -# Deploy operations -grep -i "deploy\|provision" ironic.log | grep -i "error\|fail" - -# SSL/TLS issues -grep -i "ssl\|certificate\|tls" ironic.log | grep -i "error\|fail" - -# Power management -grep -i "power" ironic.log | grep -i "error\|fail" -``` - ---- - -## DHCP, PXE Boot, and Network Boot - -### Boot Sequence - -Metal nodes boot through a network boot sequence managed by dnsmasq on the -provisioning network: - -```text -1. Node powers on (via IPMI/Redfish from Ironic) -2. Node PXE boots from provisioning network -3. dnsmasq provides DHCP lease + TFTP boot file -4. Node downloads and boots iPXE/GRUB -5. iPXE fetches RHCOS kernel + initramfs from Ironic's httpd -6. RHCOS boots and runs Ignition -7. Ignition configures the node (networking, services, etc.) -8. Node joins the cluster -``` - -### dnsmasq Configuration - -dnsmasq serves dual duty in dev-scripts: - -- **On provisioning network**: DHCP + TFTP for PXE boot -- **On baremetal network**: DHCP + DNS for cluster networking - -### PXE Boot Failure Patterns - -| Failure Point | Symptoms | Diagnosis | -|---------------|----------|-----------| -| DHCP timeout | Node never gets an IP | Check dnsmasq logs, provisioning network bridge | -| TFTP error | Node gets IP but can't download boot files | Check dnsmasq TFTP config, boot file paths | -| Boot image download | iPXE starts but fails to fetch kernel | Check Ironic httpd logs, image URLs | -| Wrong boot order | Node boots from disk instead of network | Check VM/node boot configuration | - -### Where to Find Network Boot Logs - -```bash -# dnsmasq logs (in sosreport or dev-scripts logs) -grep -i "dnsmasq" sosreport-*/var/log/messages -grep -i "dhcp\|tftp\|pxe" sosreport-*/var/log/messages - -# Ironic httpd logs (image serving) -grep -i "httpd\|GET\|404\|500" ironic.log -``` - ---- - -## Serial Console Logs - -Serial console logs capture the complete boot output from each VM/node as if watching a -physical serial console. They are **invaluable** for failures that occur before the node -is fully operational. - -### Location and Extraction - -```text -{target}/baremetalds-devscripts-gather/artifacts/libvirt-logs.tar -``` - -```bash -# Download and extract -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/baremetalds-devscripts-gather/artifacts/libvirt-logs.tar" \ - ./libvirt-logs.tar --no-user-output-enabled -tar -xf ./libvirt-logs.tar - -# Find console logs -find . -name "*console*.log" -# Typical names: -# {cluster-name}-bootstrap_console.log -# {cluster-name}-master-0_console.log -# {cluster-name}-master-1_console.log -# {cluster-name}-master-2_console.log -``` - -### What Console Logs Reveal - -The complete boot sequence, as the node would display it on a physical serial port: - -1. **BIOS/UEFI boot** — Hardware initialization -2. **PXE/iPXE boot** — Network boot sequence -3. **Kernel boot** — Linux kernel initialization -4. **initramfs** — Early userspace -5. **Ignition** — CoreOS configuration -6. **systemd services** — Service startup -7. **kubelet/crio** — Container runtime and kubelet startup - -### Key Patterns to Search in Console Logs - -Search `*console*.log` (`grep -i`) for: - -- **Kernel panic/oops**: `panic`, `kernel.*oops`, `BUG:` -- **Ignition** (critical for provisioning): `ignition`, `config fetch failed`, `Ignition failed` -- **Network during boot**: `dhcp`, `network unreachable`, `DNS`, `timeout`, `no carrier` -- **Disk/filesystem**: `mount`, `disk`, `filesystem`, `I/O error`, `readonly` -- **systemd service failures**: `Failed to start`, `service.*failed`, `Unit.*failed` -- **Memory**: `Out of memory`, `oom`, `Cannot allocate` -- **Hardware**: `hardware error`, `MCE`, `machine check` -- **SSH/auth**: `authorized_keys`, `sshd`, `authentication` - -### Console Log vs. Other Logs - -| Scenario | Best Log Source | -|----------|---------------| -| Node never boots at all | Console log — will show where boot stopped | -| Node boots but Ignition fails | Console log — Ignition messages appear here | -| Node boots but doesn't join cluster | Console log + Ironic log + installer log | -| Node joins but operator fails | Installer log + must-gather (not console) | -| Kernel panic during boot | Console log — the only place this appears | - ---- - -## Assisted Installer on Metal - -Some metal CI jobs use the **Assisted Installer** (agent-based installation) instead of -the traditional IPI flow. - -### Agent-Based Installation - -- Uses a **Discovery ISO** booted on target nodes -- An **agent** on each node reports hardware inventory -- The **Assisted Service** validates hosts and orchestrates installation -- Job names may contain `metal-assisted` or `agent` - -### Host Validation Failures - -The Assisted Installer validates each host before installation proceeds: -- Sufficient CPU, memory, and disk -- Network connectivity between hosts -- DNS resolution -- NTP synchronization -- Compatible hardware - -Look for validation errors in the Assisted Service logs or the agent logs on each node. - -### Discovery ISO Issues - -- ISO download failures (check image cache) -- Boot from ISO failures (check console logs) -- Agent registration failures (check agent logs) - ---- - -## Image-Related Issues - -### RHCOS Image Download Failures - -During provisioning, Ironic downloads the RHCOS (Red Hat CoreOS) disk image and writes -it to each node's disk. Failures here block provisioning. Search `ironic.log` for download -and serving errors: `image`, `download`, `rhcos`, `GET.*rhcos`, `404`, `500` (combined with -`error`/`fail`/`timeout`; httpd output is folded into `ironic.log`). - -**Common causes**: -- Image URL changed or was removed upstream -- Image cache on hypervisor is corrupted or full -- Network timeout during large image download -- Disk space exhaustion on hypervisor preventing image caching - -### Image Caching - -Dev-scripts caches RHCOS images on the hypervisor to avoid repeated downloads. If the -cache is corrupted or stale, search `devscripts-logs/*` for: `cache`, `image.*download`, -`pulling`. - -### Disk Image Writing Failures - -After download, Ironic writes the image to the node's virtual disk. Search `ironic.log` for -write failures: `write`, `deploy.*fail`, `disk` (combined with `error`/`fail`). - -**Common causes**: -- Insufficient disk space on target node -- Disk I/O errors (especially in VM environments) -- Image corruption during transfer -- qcow2/raw format mismatch - ---- - -## Hardware Simulation Issues - -Since CI metal jobs use VMs to simulate bare metal, failures can originate in the -virtualization layer. - -### Virtual BMC (vBMC) Failures - -Virtual BMC provides IPMI/Redfish endpoints for each VM, letting Ironic manage them as -physical machines. Search `devscripts-logs/*` for `vbmc`, `virtualbmc`, `virtual.*bmc`; and -`ironic.log` for `ipmi`/`redfish` with `connect`/`refuse`/`timeout`. - -**Common vBMC issues**: -- vBMC service didn't start for one or more VMs -- Port conflicts preventing vBMC from listening -- vBMC crashed or became unresponsive - -### libvirt/QEMU Issues - -Search the hypervisor sosreport: `libvirt`, `qemu`, `kvm` in `sosreport-*/var/log/messages`; -`error`/`fail`/`refuse` in `sosreport-*/var/log/libvirt/*.log`. VM definitions live under -`sosreport-*/etc/libvirt/qemu/`. - -**Common libvirt issues**: -- VM failed to create (insufficient resources) -- VM crashed during boot (QEMU errors) -- Storage pool issues (disk space, permissions) -- CPU model incompatibilities - -### Virtual Disk/Network Device Problems - -Search `*console*.log` for device errors: disks — `virtio.*disk`, `vda`, `sda`, `I/O error`; -network — `virtio.*net`, `eth0`, `ens`, `no carrier`, `link.*down`. - ---- - -## Network-Specific Metal Failures - -### Bridge Configuration Issues - -Dev-scripts creates network bridges (`provisioning`, `baremetal`) on the hypervisor. -Misconfiguration prevents nodes from communicating. - -```bash -# Check bridge configuration in sosreport -grep -i "bridge\|brctl" sosreport-*/sos_commands/networking/* -cat sosreport-*/sos_commands/networking/ip_-d_link 2>/dev/null - -# Check dev-scripts network setup logs -grep -i "bridge\|network\|interface" devscripts-logs/02_* -``` - -### VLAN/Bond Failures - -Some metal jobs test advanced networking configurations: - -```bash -# Check for VLAN errors -grep -i "vlan\|802.1q" *console*.log devscripts-logs/* - -# Check for bond errors -grep -i "bond\|lacp\|aggregat" *console*.log devscripts-logs/* -``` - -### IPv6 SLAAC/DHCPv6 Issues - -IPv6-only metal jobs are particularly prone to addressing issues: - -```bash -# Check IPv6 address configuration -grep -i "ipv6\|slaac\|dhcpv6\|router.*advert\|fe80\|fd00" *console*.log - -# Check for IPv6 connectivity issues -grep -i "network unreachable\|no route\|icmpv6" *console*.log - -# Check dnsmasq DHCPv6 configuration -grep -i "dhcpv6\|ra-param\|enable-ra" devscripts-logs/* -``` - -**Common IPv6 issues on metal**: -- Router advertisements not reaching nodes -- DHCPv6 lease not obtained -- SLAAC address conflicts -- DNS resolution failures over IPv6 -- NDP (Neighbor Discovery Protocol) issues - ---- - -## Metal-Specific Artifact Reference - -The canonical metal artifact layout lives in -[artifacts.md](../artifacts.md#metal-bare-metal-job-artifacts); the map below adds the -metal-specific download and extract recipes. - -### Complete Artifact Map - -```text -artifacts/{target}/ -├── ofcir-acquire/ -│ ├── build-log.txt # OFCIR host acquisition log -│ └── artifacts/ -│ └── junit_metal_setup.xml # Host acquisition JUnit result -│ -├── baremetalds-devscripts-setup/ -│ └── artifacts/root/dev-scripts/logs/ # Dev-scripts setup logs -│ ├── 01_install_requirements.log -│ ├── 02_configure_host.log -│ ├── 03_build_installer.log -│ ├── 04_setup_ironic.log -│ ├── 05_create_install_config.log -│ ├── 06_create_cluster.log -│ ├── installer-status.txt # Installer exit status (0 = success) -│ ├── .openshift_install.log # Installer log (dev-scripts invokes installer) -│ └── .openshift_install_state.json -│ -├── baremetalds-devscripts-gather/ -│ └── artifacts/ -│ ├── libvirt-logs.tar # VM console logs -│ │ └── {cluster}-bootstrap_console.log -│ │ └── {cluster}-master-0_console.log -│ │ └── {cluster}-master-1_console.log -│ │ └── {cluster}-master-2_console.log -│ ├── log-bundle-*.tar # Log bundle with Ironic logs -│ │ └── bootstrap/journals/ -│ │ │ ├── ironic.log # Master provisioning -│ │ │ ├── metal3-baremetal-operator.log -│ │ │ ├── bootkube.log -│ │ │ └── kubelet.log -│ │ └── control-plane/{node-ip}/containers/ -│ │ ├── metal3-ironic-*.log # Worker provisioning -│ │ └── metal3-baremetal-operator-*.log -│ ├── sosreport-*.tar.xz # Hypervisor diagnostics -│ └── squid-logs-*.tar # CI access proxy logs -``` - -### Downloading Metal Artifacts - -The OFCIR log and dev-scripts logs download directly by path: - -```bash -# OFCIR host acquisition log -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/ofcir-acquire/build-log.txt" \ - ./ofcir-build-log.txt --no-user-output-enabled 2>&1 || echo "Not found" - -# Dev-scripts logs (directory) -gcloud storage cp -r \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/" \ - ./devscripts-logs/ --no-user-output-enabled -``` - -The tarball artifacts share one locate → copy → extract recipe. Substitute the filename -pattern for the artifact you need: console logs `libvirt-logs\.tar$`, log bundle -`log-bundle.*\.tar$`, sosreport `sosreport.*\.tar\.xz$`, squid `squid-logs.*\.tar$`: - -```bash -gcloud storage ls -r "gs://test-platform-results/{bucket-path}/artifacts/" 2>&1 | grep "PATTERN" -gcloud storage cp {path-from-ls} ./artifact.tar --no-user-output-enabled -tar -xf ./artifact.tar # use tar -xJf for .tar.xz (sosreport) -``` - ---- - -## sosreport Analysis - -The sosreport captures hypervisor-level diagnostics. Use it when the failure appears to -be in the hypervisor itself rather than in OpenShift. - -### Key sosreport Contents - -```text -sosreport-{hostname}-*/ -├── var/log/ -│ ├── messages # System log — libvirt, kernel, service errors -│ └── libvirt/ # Libvirt-specific logs -├── etc/ -│ └── libvirt/ -│ └── qemu/ # VM definitions (XML) -├── sos_commands/ -│ ├── networking/ # Network configuration dumps -│ │ ├── ip_-d_link # Interface details -│ │ ├── ip_-d_addr # IP addresses -│ │ ├── ip_route_show_table_all # Routing tables -│ │ └── bridge_-t_-s_vlan_show # Bridge/VLAN info -│ ├── libvirt/ # Libvirt diagnostic output -│ ├── process/ # Process lists -│ └── memory/ # Memory diagnostics -``` - -### When to Use sosreport - -- Hypervisor resource exhaustion (CPU, memory, disk) -- libvirt/QEMU crashes or errors -- Network bridge or interface problems on the hypervisor -- Kernel errors or panics on the hypervisor itself -- Storage pool issues - -### Key sosreport Searches - -```bash -# Resource exhaustion -grep -i "out of memory\|oom\|no space left\|disk full" sosreport-*/var/log/messages - -# libvirt errors -grep -i "libvirt\|qemu" sosreport-*/var/log/messages | grep -i "error\|fail" - -# Network issues on hypervisor -grep -i "bridge\|bond\|link.*down\|no carrier" sosreport-*/var/log/messages - -# Kernel errors -grep -i "kernel.*error\|BUG\|oops\|panic" sosreport-*/var/log/messages -``` - ---- - -## Squid Proxy Log Analysis - -The Squid proxy runs on the hypervisor to provide **inbound** access from CI -infrastructure to the cluster under test, especially in IPv6/disconnected environments. - -### What Squid Logs Show - -- CI test runner connections to the cluster API -- HTTP status codes for CI-to-cluster requests -- Connection timeouts or refusals -- Proxy configuration issues - -### Location - -```text -{target}/baremetalds-devscripts-gather/artifacts/squid-logs-*.tar -``` - -### Key Squid Log Patterns - -```bash -# Failed connections -grep -i "TCP_DENIED\|ERR_CONNECT\|TIMEOUT\|503\|502" squid-logs-*/access.log - -# Connection patterns -grep -i "CONNECT\|api\|oauth" squid-logs-*/access.log | tail -20 - -# Configuration errors -grep -i "error\|fail\|denied" squid-logs-*/cache.log -``` - -### Common Squid Issues - -| Pattern | Meaning | -|---------|---------| -| `TCP_DENIED` | Proxy blocked the connection (ACL issue) | -| `ERR_CONNECT_FAIL` | Target unreachable (cluster API down) | -| `TIMEOUT` | Connection timed out (network issue or cluster unresponsive) | -| Many 503s | Cluster API not ready or overloaded | - -Squid logs show CI → cluster access. If cluster nodes can't reach the mirror registry, -that's a different issue (check dnsmasq and network config, not Squid). - ---- - -## Metal Analysis Workflow — Step by Step - -### Step 1: Check OFCIR Acquisition - -Did the job get a host? If `junit_metal_setup.xml` shows a failure, stop here. -Report: "OFCIR host acquisition failed — installation never started." - -### Step 2: Read Dev-Scripts Logs - -Read the numbered logs in order. Identify the **first step that failed**. -- Steps 01-05 failed → dev-scripts setup issue (not an OpenShift bug) -- Step 06 failed → proceed to installation analysis - -### Step 3: Check Console Logs (if available) - -Extract and read `libvirt-logs.tar`. Look for: -- Kernel panics -- Ignition failures -- Network configuration problems during boot -- Disk/filesystem errors - -### Step 4: Check Ironic Logs (from log bundle) - -Determine whether masters or workers failed, then check the matching Ironic logs: -- Bootstrap Ironic → master provisioning issues -- Control-plane Ironic → worker provisioning issues - -Map node UUIDs to BareMetalHost names to identify specific failing nodes. - -### Step 5: Analyze Installer Logs - -Use the techniques from [general.md](general.md): work backwards from the end of the log, -focus on final errors (not early transient ones), and track dependency chains. - -### Step 6: Check Optional Artifacts - -If the root cause is still unidentified: -- **sosreport**: hypervisor-level issues -- **Squid logs**: CI access problems (IPv6/disconnected jobs) - ---- - -## Common Metal Failure Patterns — Quick Reference - -| Issue | Symptoms | Primary Artifact | Resolution Hints | -|-------|----------|------------------|-----------------| -| **OFCIR pool exhaustion** | No host acquired | `junit_metal_setup.xml` | Wait for pool capacity; check pool health | -| **Dev-scripts package failure** | Step 01 fails | Dev-scripts log 01 | Check package repos, dependencies | -| **Network bridge failure** | Step 02 fails, no connectivity | Dev-scripts log 02 | Check libvirt network config | -| **Installer build failure** | Step 03 fails | Dev-scripts log 03 | Check Go toolchain, source availability | -| **Ironic container failure** | Step 04 fails | Dev-scripts log 04 | Check container image availability | -| **Install-config error** | Step 05 fails | Dev-scripts log 05 | Check config template, credentials | -| **BMC connection refused** | Nodes stuck registering | Ironic log | Check vBMC, IPMI port availability | -| **Inspection timeout** | Nodes stuck inspecting | Ironic log | Check provisioning network DHCP | -| **Deploy failure** | Nodes stuck deploying | Ironic log | Check RHCOS image, disk space | -| **Kernel panic** | Node crash during boot | Console log | Check hardware/VM configuration | -| **Ignition failure** | Node boots but misconfigured | Console log | Check Ignition config URL, MCS access | -| **DHCP timeout** | No IP address obtained | Console log + dnsmasq | Check provisioning network, dnsmasq | -| **Mirror registry down** | Image pull failures | Dev-scripts log, installer log | Check registry pod on hypervisor | -| **CI access failure** | Tests can't reach cluster | Squid logs | Check proxy config, network routing | -| **Hypervisor OOM** | Random VM crashes | sosreport | Check hypervisor memory allocation | -| **Disk space exhaustion** | Image downloads fail | sosreport, dev-scripts log | Check hypervisor disk usage | diff --git a/plugins/ci/skills/prow-job-analysis/references/networking.md b/plugins/ci/skills/prow-job-analysis/references/networking.md deleted file mode 100644 index 2dd837bcb..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/networking.md +++ /dev/null @@ -1,294 +0,0 @@ -# Networking Reference - -Network failures in Prow CI jobs in the **cluster under test** and its **disconnected/proxy -environment**: image pull / registry, DNS, OVN-Kubernetes/SDN, service connectivity, load -balancer/ingress, and network policy. For adjacent layers see: -- Build-farm registry, image promotion, `manifest unknown` *from promotion timing* → [ci-infrastructure-changes.md](ci-infrastructure-changes.md) -- Metal hypervisor mirror registry, squid, dnsmasq, SLAAC/DHCPv6 → [install/metal.md](install/metal.md) -- Network failures *during install* → [install/general.md](install/general.md) -- Disruption (OVS-stall fan-out, backend classification) → [disruption.md](disruption.md) -- Cloud API / quota / LB-provisioning quota → [cloud-provider-errors.md](cloud-provider-errors.md) -- Full artifact tree → [artifacts.md](artifacts.md) - -## Fast Triage - -| Symptom / log string | Likely area | Section | -|----------------------|-------------|---------| -| `ImagePullBackOff`, `ErrImagePull`, `failed to pull image` | Image pull | [Image Pull Failures](#image-pull-failures) | -| `manifest unknown`, `manifest unknown: manifest unknown` | Registry / promotion | [Image Pull Failures](#image-pull-failures) · [ci-infrastructure-changes.md](ci-infrastructure-changes.md) | -| Job name has `disconnected`, `proxy`, `ipv6`, `oci-disconnected` | Disconnected/proxy | [Disconnected & Proxy](#disconnected-and-proxy-environments) | -| `x509: certificate signed by unknown authority` (pulls) | Mirror trust | [Disconnected & Proxy](#disconnected-and-proxy-environments) | -| `no such host`, `SERVFAIL`, `i/o timeout` on `:53` | DNS | [DNS Failures](#dns-failures) | -| `CNI request failed`, `failed to create pod network sandbox` | OVN/SDN | [OVN-Kubernetes & SDN](#ovn-kubernetes-and-sdn) | -| `Unreasonably long ... poll interval` (OVSVswitchdLog) | OVS stall | [OVN-Kubernetes & SDN](#ovn-kubernetes-and-sdn) · [disruption.md](disruption.md) | -| `connection refused` to a ClusterIP, `no endpoints available` | Service/endpoints | [Service Connectivity](#service-connectivity) | -| Router/ingress not ready, LB `EXTERNAL-IP` pending | LB / ingress | [Load Balancer & Ingress](#load-balancer-and-ingress) | -| Test passes without a NetworkPolicy, times out with one | Network policy | [Network Policy](#network-policy) | - -## Where to Look - -| Artifact | Path | Use | -|----------|------|-----| -| Cluster events | `gather-extra/artifacts/oc_cmds/events` | `Failed`/`BackOff`/`FailedCreatePodSandBox` events | -| Pod status | `gather-extra/artifacts/oc_cmds/pods` | Pods in `ImagePullBackOff`, `ContainerCreating`, `CrashLoopBackOff` | -| OVN pod logs | `gather-extra/artifacts/pods/openshift-ovn-kubernetes/` | CNI, port-binding, ovn-controller | -| SDN pod logs (legacy) | `gather-extra/artifacts/pods/openshift-sdn/` | sdn/ovs pods | -| DNS pod logs | `gather-extra/artifacts/pods/openshift-dns/` | CoreDNS `dns-default-*` | -| Ingress pod logs | `gather-extra/artifacts/pods/openshift-ingress/` | `router-default-*` | -| Network operator | `gather-extra/artifacts/pods/openshift-network-operator/` | CNO rollout/degraded | -| Node journals | `gather-extra/artifacts/journal_logs/` | OVS stalls, kubelet CNI, link events | -| Operator status | `gather-extra/artifacts/oc_cmds/co` | `network`, `dns`, `ingress` degraded | -| Timeline | `**/e2e-timelines_spyglass_*.json` | `OVSVswitchdLog`, `Disruption`, `NodeMonitor` | -| Proxy config | must-gather `cluster-scoped-resources/config.openshift.io/proxies/cluster.yaml` | HTTP(S)_PROXY, NO_PROXY, trustedCA | -| ICSP/IDMS | must-gather `cluster-scoped-resources/{operator,config}.openshift.io/imagecontentsourcepolicies\|imagedigestmirrorsets` | Mirror routing | - -If must-gather is present, the must-gather-analyzer `analyze_network.py` script summarizes CNI -health, network operator status, and pod networking. It is **not** run automatically — download and -extract must-gather first (see the [artifacts reference](artifacts.md)), then run the script -manually from the [must-gather plugin](../../../../must-gather/skills/must-gather-analyzer/SKILL.md). - ---- - -## Image Pull Failures - -A recurring CI networking failure class, especially in disconnected/proxy/IPv6 jobs. - -### Diagnosis Workflow - -1. **Find the affected pods** — grep `oc_cmds/pods` and `oc_cmds/events` for `ImagePullBackOff`, - `ErrImagePull`, `Back-off pulling image`, `FailedCreatePodSandBox`. -2. **Get the exact pull error** — the *event message* (not just the phase) names the failure. - In must-gather, read the pod YAML `.status.containerStatuses[].state.waiting.{reason,message}`. -3. **Extract the image reference** — note the registry host and whether it's a mirror - (e.g. `virthost.../registry`, `registry.build0x...`) vs. an upstream (`quay.io`, `registry.redhat.io`, `registry.ci.openshift.org`). -4. **Classify** using the taxonomy below, then follow the matching section. -5. **Check breadth** — one pod (image-specific) vs. many pods/nodes (registry/DNS/mirror/proxy outage). - -### Taxonomy (match the error string) - -| Category | Error strings to grep | Meaning / next step | -|----------|----------------------|---------------------| -| **Auth failure** | `unauthorized: authentication required`, `401 Unauthorized`, `pull access denied`, `authentication required` | Pull secret missing/invalid for that registry. Check `pull-secret` in `openshift-config`; for external images check the global pull secret has creds for that host. | -| **Image not found** | `manifest unknown`, `manifest unknown: manifest unknown`, `name unknown`, `blob unknown to registry`, `not found` | Tag/digest doesn't exist. Disconnected: image wasn't mirrored (see mirroring race). Build-farm/promotion timing: see [ci-infrastructure-changes.md](ci-infrastructure-changes.md). | -| **Network connectivity** | `dial tcp ... i/o timeout`, `connection refused`, `no route to host`, `context deadline exceeded` | Node can't reach the registry endpoint. Check DNS to the registry host, node routing, proxy/NO_PROXY, firewall/SG. | -| **DNS** | `dial tcp: lookup ... no such host`, `server misbehaving` | Registry hostname unresolvable — see [DNS Failures](#dns-failures). Disconnected: mirror host must resolve on the cluster network. | -| **Registry TLS/trust** | `x509: certificate signed by unknown authority`, `tls: failed to verify certificate`, `certificate has expired` | Mirror CA not trusted. Check `additionalTrustedCA`/`image.config` and the mirror registry cert. Common in disconnected jobs. | -| **Registry overload / rate limit** | `toomanyrequests`, `pull rate limit`, `503 Service Unavailable`, `error pinging` | Upstream throttling or registry outage; usually transient and widespread. | -| **Registry down** | `error pinging container registry`, `ping attempt failed`, all pulls from one host fail | Mirror/registry pod not serving — check the registry pod (disconnected) or [ci-infrastructure-changes.md](ci-infrastructure-changes.md) for build-farm registry. | - -CRI-O wraps these as `rpc error: code = ... desc = ... reading manifest ...` / -`failed to pull and unpack image` — the underlying cause is the tail of the message. - ---- - -## Disconnected and Proxy Environments - -Disconnected jobs run the cluster with **no direct internet**; all images come from a **mirror -registry**, and image references are rewritten by **ICSP** (`ImageContentSourcePolicy`, older) or -**IDMS** (`ImageDigestMirrorSet`, 4.13+). Proxy jobs route egress through an HTTP/HTTPS proxy. - -**Detect the environment** — job name contains `disconnected`, `proxy`, `ipv6`, `oci-disconnected`, -or `okd-scos` disconnected variants; or `oc get proxy/cluster` / `imagecontentsourcepolicies` / -`imagedigestmirrorsets` are non-empty in must-gather. - -### ICSP/IDMS → Mirror → Pull Verification Chain - -Debug the pull path end to end, in order: - -1. **Mirror routing exists** — confirm ICSP/IDMS is present and covers the failing image's source - repo. Read `imagecontentsourcepolicies` / `imagedigestmirrorsets` YAML; each maps a `source` - (e.g. `quay.io/openshift-release-dev/ocp-v4.0-art-dev`) to `mirrors` (the mirror host). A pull of - an unlisted source repo goes upstream and fails (disconnected → `i/o timeout`). -2. **Mirror host resolves & is reachable** — the mirror hostname must resolve on the cluster network - and the registry port (usually `5000`/`8443`) must accept connections. `no such host` → DNS; - `connection refused`/`i/o timeout` → routing/registry-down. -3. **Mirror trusts** — mirror CA must be in the cluster trust (`image.config.openshift.io/cluster` - `additionalTrustedCA`, or `user-ca-bundle`). `x509: ... unknown authority` = untrusted CA. -4. **Image is actually mirrored** — a present ICSP/reachable/trusted mirror still fails with - `manifest unknown`/`blob unknown` if that specific tag/digest was never mirrored (see race below). - -### Proxy Infrastructure - -- **Config**: `proxies.config.openshift.io/cluster` → `httpProxy`, `httpsProxy`, `noProxy`, - `trustedCA`. Env on nodes/pods: `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`. -- **`NO_PROXY` gaps** — if the mirror/registry, `.svc`, cluster/service CIDRs, or `.cluster.local` - are missing from `noProxy`, in-cluster or mirror traffic is wrongly sent to the proxy → - `connection refused`/`Forbidden`/`403` on internal pulls. -- **SOCKS / proxy pod lifecycle** — CI disconnected/proxy jobs run a proxy (squid/http, or a - SOCKS/`sshuttle` bastion) providing egress or CI→cluster access. If that proxy **pod/service - restarts or crashes mid-run**, connectivity blips: transient `ImagePullBackOff`, `i/o timeout`, - or disruption that self-heals. Correlate the failure window with proxy-pod restarts - (`restartCount`, `lastState.terminated`) and node journals. Symptom = intermittent, not - persistent. On metal the **squid proxy is inbound only** (CI→cluster) — see [install/metal.md](install/metal.md). - -### Image Mirroring Race Conditions - -The mirror step (`oc adm catalog mirror` / `oc-mirror` / `oc image mirror`) and ICSP/IDMS -application race against pods that start pulling: - -- **Pods pull before ICSP/IDMS applies** → briefly hit the upstream source → `i/o timeout`, - then recover once the mirror config lands. Transient `ImagePullBackOff` that clears. -- **Pods pull before the mirror is fully populated** → `manifest unknown`/`blob unknown` for - images not yet pushed; clears after mirroring completes. -- **Verdict**: if pulls succeed on retry / other pods pull the same image fine, it's a race - (infra/timing), not a product bug. If a specific image *never* appears in the mirror, mirroring - of that repo genuinely failed — check the mirror step's build-log. - -### `manifest unknown` from Promotion Timing (build-farm) - -Distinct from the disconnected case: a periodic/consumer job references a payload/component image -whose **postsubmit promotion hasn't finished**, so the tag doesn't exist yet → -`manifest unknown` / `creating_release_images`. Transient CI infra, not a product bug. Detection -in [ci-infrastructure-changes.md](ci-infrastructure-changes.md). - ---- - -## DNS Failures - -Cluster DNS is CoreDNS, served by the `dns-default` **DaemonSet** in `openshift-dns`; the DNS -operator lives in `openshift-dns-operator`. - -### Error Patterns - -- `dial tcp: lookup on :53: no such host` — resolution failure (NXDOMAIN or no answer) -- `i/o timeout` / `server misbehaving` on `:53` — CoreDNS unreachable or upstream timing out -- `SERVFAIL` — upstream/forward resolver failure -- CoreDNS logs: `[ERROR] plugin/errors ... i/o timeout`, `[ERROR] ... unreachable` - -### Common Causes - -- **CoreDNS pod not scheduled on a node** — `dns-default` tolerates all taints and runs everywhere; - if a node is NotReady/cordoned or the DaemonSet pod is missing, pods on that node lose local DNS. - Check `dns-default-*` pod count vs. node count and per-node placement. Node-local DNS means one - bad node causes node-local resolution gaps, not cluster-wide. -- **CoreDNS cascade** — CoreDNS forwards external names upstream; if the upstream (node - `/etc/resolv.conf`, cloud resolver, or disconnected private resolver) is down, *every* external - lookup SERVFAILs, which cascades into operator degradation, image pulls (`no such host`), and - broad test failures that look unrelated. Confirm by checking CoreDNS logs for upstream errors and - whether in-cluster (`.svc.cluster.local`) names still resolve while external ones fail. -- **Disconnected/IPv6** — verify the private/mirror resolver is configured and (IPv6) that AAAA - records exist. See [install/metal.md](install/metal.md) for dnsmasq specifics. - ---- - -## IPv6 and Dual-Stack - -Jobs with `ipv6`, `dualstack`, or metal `ovn-ipv6`/`ovn-dualstack` in the name. IPv6-only jobs are -almost always **disconnected** (mirror + private DNS) — combine this with the sections above. - -- **IPv6-only** — every component and endpoint must be IPv6-capable; a hard-coded IPv4 literal or - missing AAAA record breaks connectivity. Watch for `no route to host` on `fd00::/fc00::` addresses. -- **Dual-stack** — both families active; failures often stem from address-family preference or a - service/endpoint present in only one family. Check `ipFamilies`/`ipFamilyPolicy` on Services and - that EndpointSlices exist for both families. -- **Addressing** (metal) — SLAAC/DHCPv6/NDP/router-advertisement issues; see [install/metal.md](install/metal.md). - ---- - -## OVN-Kubernetes and SDN - -OVN-Kubernetes is the default CNI. Pods in `openshift-ovn-kubernetes`: `ovnkube-node-*` -(DaemonSet; containers `ovn-controller`, `ovnkube-controller`, `nbdb`, `sbdb`, `northd`) and -`ovnkube-control-plane-*` (4.14+; older releases: `ovnkube-master-*`). Legacy OpenShift SDN uses -`openshift-sdn` (`sdn-*`, `ovs-*`). - -### Pod Networking / CNI Failures - -Pods stuck `ContainerCreating` with events: - -- `failed to create pod network sandbox ... plugin type="ovn-k8s-cni-overlay" ... failed to send CNI request` -- `failed to configure pod interface`, `failed to get pod annotation`, `timed out waiting for OVS port binding` -- `error while waiting for OVS ... to be ready` - -Investigate: `ovnkube-node` logs on the pod's node, then `ovnkube-control-plane`/`ovnkube-master` -(central control), then `nbdb`/`sbdb` health. A single-node pattern = node-local OVN; cluster-wide = -control-plane/db. - -### OVS vswitchd Stalls - -OVS is the dataplane; a stall freezes **all** traffic on that node. - -- Timeline source `OVSVswitchdLog`: `Unreasonably long NNNNms poll interval` -- `>500ms` = degraded; `>1000ms` = effectively frozen (no packets forwarded) -- **A classic root cause is CPU starvation** — OVS shares CPU with other processes. Correlate with - `CPUMonitor` (>95%) and disk I/O ([resource-exhaustion.md](resource-exhaustion.md)); the - stall's disruption fan-out is classified in [disruption.md](disruption.md). - -```json -{ "source": "OVSVswitchdLog", "message": { "humanMessage": "Unreasonably long 9000ms poll interval" } } -``` - -Also present in node journals: grep `journal_logs/` for `Unreasonably long`. - ---- - -## Service Connectivity - -Pod-to-pod and pod-to-Service reachability. - -- **No ready endpoints** — `connection refused` / `no endpoints available for service /`. - The backing pods aren't Ready. Check the pods, then whether EndpointSlices list ready addresses - (`oc get endpointslices -n `; audit logs show EndpointSlice churn during disruption). -- **Endpoint propagation lag** — after a readiness change, EndpointSlices update with delay; - connections to the just-removed pod fail transiently. Correlate with `readyz=false` transitions - and EndpointSlice modification audit events. -- **ClusterIP unreachable but pod IP works** — service proxying (OVN services / kube-proxy) issue, - not the app. Check `ovnkube-node` logs for load-balancer/service programming errors. -- **Cross-node pod-to-pod fails, same-node works** — overlay/geneve tunnel or OVS problem on one - node (see OVS stall) or an MTU mismatch on the overlay. -- **Hairpinning (pod reaching its own Service)** — a pod can reach other services but not its own - ClusterIP (self-connection times out) while others reach it fine. OVN handles hairpin via - masquerade; failures point to CNI/masquerade config. Test-only if the workload self-connects. - ---- - -## Load Balancer and Ingress - -- **Router pods** — `router-default-*` in `openshift-ingress`; operator in - `openshift-ingress-operator`. Not-ready routers → `ingress` clusteroperator degraded and - `*.apps.` routes unreachable. -- **Cloud LB provisioning** — `router-default` Service (type `LoadBalancer`) stuck with no - `EXTERNAL-IP`; events `EnsuringLoadBalancer` / `SyncLoadBalancerFailed`. Points at the cloud - API/quota layer — the event message carries the cloud error; see [cloud-provider-errors.md](cloud-provider-errors.md). -- **LB health checks failing** — LB provisioned but marks backends unhealthy → intermittent - ingress/API disruption. Check health-check port/path and router readiness; correlate with - disruption on ingress-routed backends (image-registry, oauth, console) in [disruption.md](disruption.md). -- **Ingress/API DNS** — `api.` / `*.apps.` records must resolve; missing records - present as `no such host` reaching the cluster. The ingress canary route is a quick health signal. - ---- - -## Network Policy - -`NetworkPolicy` / `AdminNetworkPolicy` (and OVN `EgressFirewall`, or SDN `EgressNetworkPolicy`) can -block **legitimate** traffic — either a test's own policy or a leftover from a prior test. - -- **Signature**: connectivity works with no policy, then a specific test times out (never `connection - refused`) after applying/for the duration of a policy; removing it restores traffic. -- **Check**: `oc get networkpolicy,adminnetworkpolicy -A`, `oc get egressfirewall -A` (OVN), - `oc get egressnetworkpolicy -A` (SDN). A default-deny with a missing/incorrect allow rule (wrong - podSelector/namespaceSelector/ports) silently drops traffic. -- **DNS caveat** — an egress policy that omits UDP/TCP `:53` to `openshift-dns` breaks name - resolution for selected pods, surfacing as DNS failures rather than an obvious policy block. -- **Enforcement bug vs. config** — if the policy *should* allow the traffic but it's still dropped, - suspect OVN policy programming: check `ovnkube-node`/`ovnkube-control-plane` logs for ACL errors. - ---- - -## Common Failure Patterns — Quick Reference - -| Pattern | Likely cause | Where to look | -|---------|--------------|---------------| -| Many pods `ImagePullBackOff`, one registry host | Mirror/registry down or untrusted | Events, pod logs, registry pod, ICSP/IDMS + CA | -| `manifest unknown` in disconnected job | Image not mirrored / mirroring race | Mirror step log, ICSP/IDMS coverage | -| `manifest unknown` in periodic after merge | Promotion timing | [ci-infrastructure-changes.md](ci-infrastructure-changes.md) | -| `x509: unknown authority` on pulls | Mirror CA not trusted | `image.config` additionalTrustedCA, proxy trustedCA | -| Intermittent pull/`i/o timeout`, self-heals | Proxy/SOCKS pod restart or mirror race | Proxy pod `restartCount`, timeline window | -| `no such host` cluster-wide | CoreDNS upstream cascade | `openshift-dns` CoreDNS logs, node resolv.conf | -| DNS gaps on one node | `dns-default` not scheduled there | `dns-default-*` per-node placement | -| Pods stuck `ContainerCreating`, `CNI request failed` | OVN pod networking | `ovnkube-node` → control-plane → nbdb/sbdb | -| All backends fail on one node | OVS stall (CPU starvation) | `OVSVswitchdLog`, `CPUMonitor`, journals | -| `no endpoints available` / `connection refused` to svc | No ready endpoints / propagation lag | Pods, EndpointSlices, audit logs | -| Router not ready, `*.apps` unreachable | Ingress operator / router pods | `openshift-ingress`, `co` ingress | -| LB `EXTERNAL-IP` pending | Cloud LB provisioning | Service events, [cloud-provider-errors.md](cloud-provider-errors.md) | -| Test times out only with a NetworkPolicy | Policy blocks legit traffic | `networkpolicy`/`egressfirewall`, `:53` allow | diff --git a/plugins/ci/skills/prow-job-analysis/references/operating-system-changes.md b/plugins/ci/skills/prow-job-analysis/references/operating-system-changes.md deleted file mode 100644 index 2622434c5..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/operating-system-changes.md +++ /dev/null @@ -1,203 +0,0 @@ -# Operating System Changes Reference - -Use when no other failure mode explains the cause and you suspect an operating system -level problem — kernel, cri-o, NetworkManager, systemd, SELinux, rpm-ostree, or bootloader -changes. The node OS is RHCOS (Red Hat CoreOS), the immutable OS every OpenShift node boots. A -single RHCOS bump inside a payload swaps the kernel, cri-o, systemd, NetworkManager, and SELinux -policy across the whole cluster at once, so it breaks many unrelated jobs with no product-code PR -to blame. -Route here when: - -- Containers fail to start or die at runtime (`cri-o`, `crun`, `runc`, OCI/cgroup errors) -- Node networking breaks after boot (NetworkManager, host interface/route/DNS) -- A node kernel-panics, hangs, or never returns Ready after a reboot -- Nodes fail Ignition/first-boot, or `rpm-ostree`/boot fails to apply an OS update -- A failure appears **only** on one RHCOS variant (RHCOS 10 but not RHCOS 9, or the reverse) -- SELinux denials, failed systemd units, or filesystem errors surface in node journals - -**Use a different reference for:** - -- MCO **drain/reboot stalls** and MachineConfigPool mechanics → - [upgrade.md](upgrade.md#mco-drain-failures) (owns the drain cascade) -- In-cluster **networking** (OVN-Kubernetes/SDN, DNS, ingress, IP exhaustion) → - [networking.md](networking.md) -- **OOM / node pressure / disk / PID** exhaustion → [resource-exhaustion.md](resource-exhaustion.md) -- CPU-starvation / OVS-stall **disruption** interpretation → [disruption.md](disruption.md) -- OS problems **during install** (bootstrap, Ignition, serial console) → - [install/general.md](install/general.md); metal console/sosreport → - [install/metal.md](install/metal.md) -- Full artifact tree → [artifacts.md](artifacts.md) - -## What RHCOS Is - -RHCOS is an immutable, `rpm-ostree`-managed RHEL image shipped **as a component of the release -payload** (image-stream tags `rhel-coreos` and `rhel-coreos-10`), not installed per node. Nodes -never `yum install`; the entire OS is a versioned tree that the MCO deploys and the node boots -into. Two variants ship today: - -| Variant | Base | Job-name fragment | `rhcos_version` values | -|---------|------|-------------------|------------------------| -| RHCOS 9 | RHEL 9 | none (4.x default) or `rhcos9` | `rhcos9`, `rhcos9-default` | -| RHCOS 10 | RHEL 10 | `rhcos10` | `rhcos10`, `rhcos10-default` | -| Heterogeneous | both | `rhcos9_10` | `rhcos9_10` | - -RHCOS 10 carries a **different kernel, systemd, SELinux policy, and package set** than RHCOS 9. -When both exist in a payload, a bug can be confined to one variant — see -[Variant Isolation](#variant-isolation-the-strongest-signal). - -## How an RHCOS Change Reaches a Node - -```text -payload bumps rhel-coreos image → machine-config-operator renders a new MachineConfig - → machine-config-daemon (one pod per node) picks it up - → cordon → drain → rpm-ostree deploy new OS tree → reboot into it → uncordon - → repeats one node at a time per MachineConfigPool (master pool, then worker pool) -``` - -An OS change and a config change travel the **same MCO path**, which splits the debugging: - -- The **drain/reboot** half (PDBs, stuck nodes, MCP `Degraded`) is owned by - [upgrade.md](upgrade.md#mco-drain-failures). -- The **OS-content** half — what changed inside the booted tree (kernel, cri-o, NetworkManager, - systemd) — is this reference. - -## When to Suspect the OS Layer - -Work top-down; the first two are the highest-signal. - -### Variant Isolation (the strongest signal) - -A failure that hits RHCOS 10 jobs and **zero** RHCOS 9 jobs (or the reverse) is almost certainly -an OS-variant difference — kernel, systemd, SELinux, or a package that differs between RHEL 9 and -RHEL 10. Heterogeneous (`rhcos9_10`) jobs count toward both variants; when only their RHCOS-10 -nodes fail, treat it as RHCOS-10-isolated. This narrows the search at once and points at the -platform/RHCOS team, not the product PR under test. - -### Payload-Boundary, Cross-Job Blast Radius - -An RHCOS bump lands at a **payload boundary** and affects every job that consumes that payload. -Signature: a class of failures that begins on one nightly/CI payload, spans unrelated components, -and matches no product PR. Confirm the onset aligns with a payload on -[search.ci](https://search.ci.openshift.org/) or [Sippy](https://sippy.dptools.openshift.org/), -then compare the RHCOS RPM diff for that payload ([next section](#comparing-rhcos-between-payloads)). - -### Host-Level Symptom - -The failing component runs on the host, below Kubernetes: cri-o/crun/runc, the kernel, -NetworkManager, systemd, SELinux, or the filesystem/bootloader. These speak in **node journals -and serial consoles**, not in operator pod logs — see [Failure Patterns](#failure-patterns) and -[Artifact Locations](#artifact-locations). - -## Comparing RHCOS Between Payloads - -For payload debugging, the release controller and the payload snapshot expose a **deep RPM-level -changelog** between an RHCOS version and its predecessor — the fastest way to see what actually -changed in the OS. - -- **Release-controller changelog** — the payload diff (`changelog.json`) carries a top-level - `nodeImageStreams` block: per-variant RPM diffs (added/removed/changed packages) between this - payload and the previous one. -- **Payload snapshot** ([payload-snapshot](../../payload-snapshot/SKILL.md)) persists this as - `summary.json` → `payloads[].rhcos_changes[]` (per-variant `rpmDiff`, keyed by tag - `rhel-coreos` / `rhel-coreos-10`) and extracts the full RPM database to - `rpmdb//rpmdb.sqlite`: - - ```bash - # every RPM in a variant, at exact NEVRA - rpm -qa --dbpath "$(pwd)/payload////rpmdb/rhel-coreos-10" - ``` - -- Map the failing job to its variant (`rhcos_version` in the snapshot's job entry), then read - that variant's `rpmDiff` for the job's originating payload. Overlap between the diff and the - failure's subsystem (kernel / systemd / cri-o / NetworkManager / SELinux) is the correlation. - -**RHCOS RPM changes are not revert candidates** — they cannot be pulled out of a payload like a -PR. Surface them as **RHCOS RPM suspects** for the RHCOS/platform team, with the variant, the -package NEVRA delta, and a rationale. Full suspect-scoring workflow: -[payload-analysis](../../payload-analysis/SKILL.md). - -## Identifying the RHCOS Version on a Node - -From gathered cluster state (no live cluster needed): - -- `gather-extra/artifacts/oc_cmds/nodes` — the node `OSImage` (e.g. - `Red Hat Enterprise Linux CoreOS 9.x`) plus `KernelVersion` and `ContainerRuntimeVersion` under - `.status.nodeInfo`. -- must-gather `cluster-scoped-resources/core/nodes/*.yaml` — the same fields per node. - -The `OSImage` string is the quickest confirmation of which RHEL base (9 vs 10) a node booted. - -## Failure Patterns - -| Symptom / log string | OS subsystem | Where to look | -|----------------------|--------------|---------------| -| `CreateContainerError`, `RunContainerError`, `failed to create OCI runtime`, `error reserving ctr name`, crun/runc errors | cri-o / crun / runc | crio journal + `crio_service.log`; pod events | -| Runtime ↔ kubelet mismatch after a bump, `CRI-O` version errors, cgroup v1/v2 errors | cri-o / kubelet | `crio_service.log`, `kubelet_service.log`, journal | -| Node loses connectivity **after boot**, missing IP/route, `NetworkManager-wait-online` unit fails | NetworkManager | node journal (`NetworkManager` unit), serial console | -| `Kernel panic - not syncing`, `BUG:`, `Oops`, `soft lockup`, `Call Trace:`; node never returns Ready after reboot | kernel | serial console; journal (previous boot) | -| systemd unit `failed` / `start-limit-hit`; degraded first boot | systemd | `failed-units.txt`, journal, serial console | -| `avc: denied`, `SELinux is preventing` after an OS bump | SELinux policy | node journal (`audit` / `setroubleshoot`) | -| `read-only file system`, xfs/ext4 errors, mount or `rpm-ostree` deploy failures | storage / rpm-ostree | journal, serial console | -| Ignition first-boot failure (fresh install) | ignition / afterburn | serial console, log bundle → [install/general.md](install/general.md) | - -Container-start and NetworkManager issues dominate day-to-day OS debugging; kernel panics and -Ignition failures usually keep a node from ever becoming Ready — check the serial console first. - -## Artifact Locations - -| Artifact | Path | OS signal | -|----------|------|-----------| -| Node journals | `gather-extra/artifacts/nodes//journal` (some jobs: `gather-extra/artifacts/journal_logs/`) | kernel, NetworkManager, crio, kubelet, systemd per node | -| crio/kubelet host logs | must-gather `host_service_logs/masters/{crio,kubelet}_service.log` | runtime & kubelet decisions at the host level | -| Failed units | log bundle / must-gather `failed-units.txt` | which systemd units failed (fast first look) | -| Serial console | `log-bundle-*/serial/*-serial.log`; metal `libvirt-logs.tar` | kernel panic, boot hang, Ignition — the only place a pre-Ready node speaks | -| Node OS/kernel/runtime | `gather-extra/artifacts/oc_cmds/nodes` (`.status.nodeInfo`) | `OSImage`, `KernelVersion`, `ContainerRuntimeVersion` | -| MCO / MCD logs | `gather-extra/artifacts/pods/openshift-machine-config-operator/` | which OS/config version a node is on; rollout errors | -| MachineConfig(Pool) | `oc_cmds/machineconfigpool`, `oc_cmds/machineconfig` | rendered config, per-pool update/degraded state | - -**The per-node `journal` files are gzip-compressed without a `.gz` extension** — a plain -`grep` on them matches nothing. List `gather-extra/artifacts/nodes/` for the node names, -then read each journal with `zcat`/`zgrep`: - -```bash -# kernel panics / OOM / NetworkManager / SELinux across every node journal -for j in gather-extra/artifacts/nodes/*/journal; do - echo "== $j" - zgrep -Ei "kernel panic|BUG:|Oops|Call Trace|out of memory|avc: +denied" "$j" -done - -# Runtime versions per boot. End-of-run snapshots (oc_cmds/nodes, nodes.json) -# show only the final version; changes within the run are visible only here. -zgrep -E "Starting CRI-O, version|Container runtime initialized" \ - gather-extra/artifacts/nodes/*/journal -``` - -Journals and serial consoles are the OS layer's primary evidence; operator pod logs sit above it -and show only the downstream symptom. Never clear the OS layer from end-of-run node -snapshots alone — compare versions across boots in the journals. - -## Quick Triage Checklist - -1. **Variant?** Read `rhcos_version` (snapshot) or `OSImage` (`oc_cmds/nodes`). Failure on only - one variant → OS-variant-specific; go straight to the RPM diff. -2. **Payload boundary?** Did the failure class start on a specific payload across unrelated jobs - with no product PR? → compare `rhcos_changes[]` / `nodeImageStreams` RPM diff. -3. **Which subsystem?** Match the symptom in [Failure Patterns](#failure-patterns) to - cri-o / kernel / NetworkManager / systemd / SELinux / storage. -4. **Read host evidence.** Node journal + `host_service_logs` for runtime/NM/systemd; serial - console for kernel panic / boot / Ignition; `failed-units.txt` for a fast unit list. -5. **Correlate.** Overlap between the RPM diff and the failing subsystem = RHCOS suspect (not a - revert). Record variant, package NEVRA delta, and rationale for the platform team. -6. **Drain vs content.** A node that will not drain/reboot is an MCO **rollout** problem - ([upgrade.md](upgrade.md#mco-drain-failures)); a node that boots the new - OS and then misbehaves is an OS-**content** problem (this reference). - -## See Also - -- [upgrade.md](upgrade.md) — MCO drain/reboot cascade, MachineConfigPool status, version skew -- [networking.md](networking.md) — in-cluster OVN/DNS/ingress (vs host NetworkManager here) -- [resource-exhaustion.md](resource-exhaustion.md) — OOM, node pressure, kernel OOM killer -- [disruption.md](disruption.md) — OVS stalls, CPU starvation, interval interpretation -- [install/general.md](install/general.md) · [install/metal.md](install/metal.md) — Ignition, serial console, sosreport at install time -- [artifacts.md](artifacts.md) — full artifact tree (journals, serial, must-gather, log bundle) -- [payload-snapshot](../../payload-snapshot/SKILL.md) · [payload-analysis](../../payload-analysis/SKILL.md) — RHCOS RPM diffs, rpmdb, suspect scoring diff --git a/plugins/ci/skills/prow-job-analysis/references/resource-exhaustion.md b/plugins/ci/skills/prow-job-analysis/references/resource-exhaustion.md deleted file mode 100644 index d30db9451..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/resource-exhaustion.md +++ /dev/null @@ -1,413 +0,0 @@ -# Resource Exhaustion Reference - -CPU, memory, disk, ephemeral-storage, PID, etcd-space, and PVC exhaustion in Prow CI — -OOM kills, node pressure and eviction, unschedulable pods, and the cascades they trigger. - -First question is always **which cluster ran out**: the ephemeral **cluster under test**, -or the **CI build-farm** running the job. Different failure domains, owners, and fixes — -see [Two Failure Domains](#two-failure-domains) first. - -## When to Use - -- Pods/containers show `OOMKilled`, exit code `137`, or high `restartCount` -- Nodes report `MemoryPressure`, `DiskPressure`, or `PIDPressure`; pods `Evicted` -- Pods stuck `Pending` with `FailedScheduling` (`Insufficient cpu/memory`, taints, PVC) -- Pods stuck `ContainerCreating` on storage/IP allocation; PVCs stuck `Pending` -- etcd logs `mvcc: database space exceeded` / `NOSPACE` alarm -- Job dies in `build-log.txt` with `pod_pending` or a truncated log (sidecar OOM) - -**Use a different reference for:** - -- CPU starvation / disk-I/O latency driving disruption (`CPUMonitor`, `CloudMetrics`, - `EtcdDiskWalFsyncDuration`, OVS stalls) → [disruption.md](disruption.md#key-signal-sources) -- Build-farm capacity, `pod_pending` mechanics, lease exhaustion, `openshift/release` - correlation → [ci-infrastructure-changes.md](ci-infrastructure-changes.md#pod_pending) -- Cloud **quota** at install time (vCPU, EBS/PD volume limits, EIP) → - [cloud-provider-errors.md](cloud-provider-errors.md) -- Resource pressure *during install* (bootstrap OOM, undersized control plane) → - [install/general.md](install/general.md) -- OVN node-subnet / CNI IP exhaustion mechanics → [networking.md](networking.md#ovn-kubernetes-and-sdn) - -## Two Failure Domains - -Every Prow CI run involves two clusters; exhaustion means opposite things in each. - -| | **CI build-farm cluster** | **Cluster under test** | -|---|---|---| -| What runs there | ci-operator pod, the multi-container test pod (`test` + `sidecar`), build pods | The ephemeral OpenShift cluster being installed and exercised | -| Exhaustion looks like | `pod_pending`, `test`/`sidecar` container `OOMKilled`, evicted ci-op pod, truncated `build-log.txt` | Product pods `OOMKilled`, nodes `NotReady`/pressure, etcd `NOSPACE`, PVCs `Pending` | -| Evidence lives in | top-level `build-log.txt`, `prowjob.json`, `podinfo.json` | `gather-extra/`, must-gather, `e2e-timelines_spyglass_*.json` | -| Owner / fix | Test Platform; retry, adjust step resource requests | Product team or job config; real bug vs undersized cluster | -| Verdict | **CI infrastructure** (build-farm capacity or step resource config) | Product bug **or** infra, must disambiguate | - -**Decision rule:** if the failure is in the **top-level** `build-log.txt` before/around -test start, or in `podinfo.json` container statuses, it is the **build-farm** pod — CI -infra. If it is in gather-extra/must-gather/interval data from the provisioned cluster, it -is the **cluster under test**. See -[ci-infrastructure-changes.md](ci-infrastructure-changes.md#distinguishing-ci-infrastructure-vs-product-failures) -for the full CI-vs-product framework. - -## Fast Triage - -| Symptom / log string | Area | Section | -|----------------------|------|---------| -| exit code `137`, `reason: OOMKilled` | OOM kill | [OOM Kills](#oom-kills) | -| `Out of memory: Killed process`, `oom-kill:`, `Memory cgroup out of memory` | Kernel/cgroup OOM | [OOM Kills](#oom-kills) | -| `System OOM encountered`, `SystemOOM` event | Node-level OOM | [OOM Kills](#oom-kills) | -| node cond `MemoryPressure`/`DiskPressure`/`PIDPressure=True`, pods `Evicted` | Node pressure | [Node Pressure & Eviction](#node-pressure-and-eviction) | -| `The node was low on resource: [memory\|ephemeral-storage]` | Eviction | [Node Pressure & Eviction](#node-pressure-and-eviction) | -| `FailedScheduling`, `Insufficient cpu/memory`, `untolerated taint` | Scheduling | [Pod Scheduling Failures](#pod-scheduling-failures) | -| `Usage of EmptyDir`/`ephemeral local storage usage exceeds` | Ephemeral storage | [Ephemeral Storage](#ephemeral-storage-exhaustion) | -| `mvcc: database space exceeded`, `alarm:NOSPACE`, `etcdserver: no space` | etcd space | [etcd Storage](#etcd-storage-exhaustion) | -| PVC `Pending`, `ProvisioningFailed`, `failed to provision volume` | Storage provisioning | [PVC & Storage](#pvc-and-storage-provisioning) | -| truncated `build-log.txt`, `sidecar` `OOMKilled` in `podinfo.json` | CI sidecar | [CI Pod & Sidecar](#ci-pod-and-sidecar-resources) | -| `pod_pending`, `did not start running within ...` | Build-farm capacity | [ci-infrastructure-changes.md](ci-infrastructure-changes.md#pod_pending) | - -## Where to Look - -Paths are relative to `artifacts/{target}/` (cluster under test). Full form: -`gs://test-platform-results/{bucket-path}/artifacts/{target}/...`. See -[artifacts.md](artifacts.md#gather-extra-artifacts) for the complete tree. - -| Artifact | Path | Use for | -|----------|------|---------| -| Node conditions/capacity | `gather-extra/artifacts/oc_cmds/nodes` | `MemoryPressure`/`DiskPressure`/`PIDPressure`, allocatable | -| Cluster events | `gather-extra/artifacts/oc_cmds/events` | `Evicted`, `SystemOOM`, `FailedScheduling`, `ProvisioningFailed` | -| Pod snapshot | `gather-extra/artifacts/oc_cmds/pods` | `RESTARTS`, `Pending`, `Evicted`, `OOMKilled` status | -| PV/PVC | `gather-extra/artifacts/oc_cmds/pv` | Bound vs available volumes | -| Node journals | `gather-extra/artifacts/journal_logs/` | Kernel OOM (`Out of memory`), kubelet eviction | -| etcd pod logs | `gather-extra/artifacts/pods/openshift-etcd/` | `NOSPACE`, quota exceeded, compaction | -| Pod YAML (container status) | must-gather `namespaces//pods//.yaml` | `lastState.terminated.{reason,exitCode}`, `restartCount` | -| Host service logs | must-gather `host_service_logs/masters/{kubelet,crio}_service.log` | kubelet eviction/OOM decisions | -| CI test pod status | `podinfo.json` | build-farm container exit codes / `OOMKilled` | -| Timeline | `**/e2e-timelines_spyglass_*.json` | `NodeMonitor`, `Alert`, `KubeletLog`, `CPUMonitor` | - -If must-gather is present, the must-gather-analyzer scripts summarize node/pod/event health -(`analyze_nodes.py --problems-only`, `analyze_pods.py --problems-only`, -`analyze_events.py --type Warning`, `analyze_etcd.py`). They are **not** run automatically — -download and extract must-gather first (see the [artifacts reference](artifacts.md)), then run the -scripts manually from the -[must-gather plugin](../../../../must-gather/skills/must-gather-analyzer/SKILL.md). - ---- - -## OOM Kills - -A container is OOM-killed when it exceeds its memory `limit` (cgroup OOM) or the node runs -out of memory and the kernel OOM killer reaps a victim (node OOM). The signals differ. - -### Exit code 137 and the OOMKilled reason - -- **`exitCode: 137`** = `128 + 9` (SIGKILL). Strong hint, not proof — SIGKILL from other - sources (liveness-probe hard kill, node shutdown) also yields 137. -- **`reason: OOMKilled`** in the container's terminated state is the definitive signal. -- `exitCode: 143` = `128 + 15` (SIGTERM) — graceful stop or liveness restart, **not** OOM. - -### Which container was killed - -Read the pod YAML (must-gather) and inspect every container, not just container 0: - -```bash -# must-gather: find OOM-killed containers cluster-wide -grep -rl "OOMKilled" */namespaces/*/pods/*/*.yaml -``` - -In the YAML, the killed container has: - -```yaml -lastState: - terminated: - reason: OOMKilled - exitCode: 137 -restartCount: 5 # repeated OOM → CrashLoopBackOff -``` - -`state.running` with a high `restartCount` and `lastState.terminated.reason: OOMKilled` -means it is OOM-looping now. `gather-extra/artifacts/oc_cmds/pods` shows `RESTARTS` counts to -spot the pod first; the YAML confirms the reason. - -### Container-limit OOM vs node-level OOM - -| | Container-limit (cgroup) OOM | Node-level OOM | -|---|---|---| -| Trigger | One container exceeds its `resources.limits.memory` | Node memory exhausted; kernel picks victims | -| Event | None by default — only `containerStatuses.lastState` | Kubelet node event `SystemOOM` (`System OOM encountered`) | -| Journal | `Memory cgroup out of memory: Killed process ...` | `Out of memory: Killed process ...`, `oom-kill:constraint=CONSTRAINT_NONE` | -| Meaning | That workload needs more memory / has a leak | Node oversubscribed or a noisy neighbor; look at `MemoryPressure` | -| Fix | Raise the container limit or fix the leak | Right-size requests, spread load, add capacity | - -### Where OOM evidence lives - -1. **Pod status** (most reliable): must-gather pod YAML `containerStatuses[].lastState.terminated`. -2. **Events**: `oc_cmds/events` / must-gather events — node-level OOM shows `SystemOOM`; - Node Problem Detector (when present) emits `OOMKilling`. -3. **Node journal**: `journal_logs/` — grep `Out of memory`, `oom-kill:`, `Killed process`, - `Memory cgroup out of memory`. The victim's `(comm)` and RSS are logged here. -4. **Kernel/dmesg**: on nodes that never became Ready, kernel OOM appears in the install - log bundle serial console (see [install/general.md](install/general.md)) rather than the journal. - -```bash -grep -rEi "out of memory|oom-kill|killed process|cgroup out of memory" \ - gather-extra/artifacts/journal_logs/ -``` - ---- - -## Node Pressure and Eviction - -Kubelet sets a node condition and evicts pods when a resource crosses an eviction threshold. - -| Condition | Resource | Downstream | -|-----------|----------|------------| -| `MemoryPressure` | node `memory.available` low | Evict `BestEffort`, then `Burstable` over request; taint `node.kubernetes.io/memory-pressure` | -| `DiskPressure` | `nodefs`/`imagefs` low (`.available`/`.inodesFree`) | Image & dead-container GC, then pod eviction; taint `node.kubernetes.io/disk-pressure` | -| `PIDPressure` | node PIDs exhausted | New pods blocked; existing may be killed; taint `node.kubernetes.io/pid-pressure` | - -### The eviction chain - -```text -resource threshold crossed → node condition True → kubelet evicts lowest-priority pods - → pod .status.phase=Failed, reason=Evicted → rescheduled elsewhere - → if every node is pressured: reschedule storm → more pressure → nodes NotReady -``` - -Evicted pods carry a message naming the resource: - -- `The node was low on resource: memory.` -- `The node was low on resource: ephemeral-storage. Threshold quantity: ...` -- `Pod ephemeral local storage usage exceeds the total limit of containers ...` - -### Where to look - -- `oc_cmds/nodes` → `.status.conditions[]` for `*Pressure` (`status: "True"`) and - `.status.allocatable`; `.spec.taints` for the pressure taints. -- `oc_cmds/events` → `Evicted`, `NodeHasDiskPressure`, `NodeHasInsufficientMemory`, - `EvictionThresholdMet`; recovery shows `NodeHasSufficientMemory`/`NodeHasNoDiskPressure`. -- Timeline `NodeMonitor` intervals → node Ready/NotReady transitions with timing. -- kubelet journal → `attempting to reclaim`, `eviction manager: must evict pod(s)`. - -A node flapping between pressure and recovery evicts and reschedules repeatedly — a root-cause -pattern behind "random" unrelated test failures ([Cascades](#how-exhaustion-cascades)). - ---- - -## Pod Scheduling Failures - -Unschedulable pods stay `Pending` with `PodScheduled=False` and a `FailedScheduling` event. -The scheduler message enumerates why each node was rejected. - -| Message fragment | Cause | -|------------------|-------| -| `Insufficient cpu`, `Insufficient memory` | Requests exceed allocatable on every node | -| `Insufficient ephemeral-storage` | Ephemeral-storage request unsatisfiable | -| `node(s) had untolerated taint {...}` | Taint (incl. pressure/`NotReady` taints) without matching toleration | -| `node(s) didn't match Pod's node affinity/selector` | nodeSelector/affinity too restrictive | -| `node(s) had volume node affinity conflict` | PV zone/topology ≠ where the pod can run | -| `pod has unbound immediate PersistentVolumeClaims` | PVC not bound (see [PVC & Storage](#pvc-and-storage-provisioning)) | -| `too many pods` / `node(s) exceed max volume count` | Node pod-count or per-node volume-attach cap reached | - -```bash -grep -rE "FailedScheduling|Insufficient|untolerated taint|volume node affinity" \ - gather-extra/artifacts/oc_cmds/events -``` - -**Subnet / IP exhaustion** also strands pods, but at `ContainerCreating`, not scheduling: -the node's pod subnet or the cloud ENI/secondary-IP pool is empty (`no IP addresses -available in range`, `failed to assign an IP address`). Mechanics live in -[networking.md](networking.md#ovn-kubernetes-and-sdn); cloud IP/subnet limits in -[cloud-provider-errors.md](cloud-provider-errors.md). - -Distinguish real exhaustion from config: `Insufficient cpu/memory` on **all** nodes = the -cluster is genuinely full (undersized, or a workload requests too much); an `untolerated -taint`/affinity message = placement constraints, not capacity. - ---- - -## Ephemeral Storage Exhaustion - -Ephemeral storage is the node's `nodefs`/`imagefs` (container writable layers, `emptyDir`, -logs, images). Filling it triggers `DiskPressure` → image GC → eviction. - -Common CI causes: - -- **Test artifacts written to the pod filesystem** instead of an artifact dir — large - dumps, cores, or captures balloon the writable layer. -- **Container logs growing unbounded** — a chatty/looping container fills `/var/log/pods`; - kubelet rotates, but burst rate can outrun rotation. -- **Image layer buildup** — many large images pulled onto one node fill `imagefs`. -- **`emptyDir` with no `sizeLimit`** — a workload fills the medium until node-level pressure. - -Signals: - -- Pod evicted with `low on resource: ephemeral-storage` or - `Usage of EmptyDir volume "X" exceeds the limit "Y"`. -- Node `DiskPressure=True`; `oc_cmds/nodes` `.status.allocatable`/`.capacity` - `ephemeral-storage`; kubelet reclaim of `imagefs`/`nodefs`. -- Journal: filesystem-full / GC messages; `free disk space failed`. - -kubelet garbage-collects `imagefs` at its image-GC high threshold before evicting; if -eviction still fires, disk is filling faster than GC frees it. This is distinct from -**disk-I/O saturation** (throughput, not capacity) — that is a disruption signal -(`CloudMetrics`, `EtcdDisk*`) in [disruption.md](disruption.md#key-signal-sources). - ---- - -## etcd Storage Exhaustion - -etcd enforces a backend DB size quota. Hitting it puts the cluster into a **read-only** -alarm state — writes fail cluster-wide, the API returns errors, and operators degrade. -OpenShift raises the quota above the 2 GiB upstream default (commonly ~8 GiB). - -Log/alarm strings (`gather-extra/artifacts/pods/openshift-etcd/`): - -- `mvcc: database space exceeded`, `etcdserver: mvcc: database space exceeded` -- `alarm:NOSPACE`, `etcdserver: no space` -- `database space exceeded` in kube-apiserver logs (writes rejected downstream) -- `apply request took too long`, `took too long ... to execute` (also disk-I/O — correlate) - -Causes: - -- **Object bloat** — an explosion of secrets, configmaps, events, or CRs (a controller - hot-looping create/delete, or a test creating thousands of objects) grows the DB and the - history keyspace faster than compaction reclaims it. -- **Compaction/defrag not keeping up** — space is freed logically by compaction but only - returned to the FS by defragmentation; without defrag the DB stays near quota. -- **Slow disk** amplifies the above (WAL fsync latency) — see - [disruption.md](disruption.md#key-signal-sources). - -Evidence: etcd `current.log`/`previous.log` for the alarm and compaction lines; alerts -`etcdBackendQuotaLowSpace` / `etcdExcessiveDatabaseGrowth` (timeline `Alert`); count objects -via must-gather (`namespaces/*/core/` sizes, event counts) to find the bloated resource. -This is etcd out of **space** — etcd slow due to disk **I/O latency** is separate. - ---- - -## PVC and Storage Provisioning - -Dynamic provisioning can fail at bind time, leaving PVCs `Pending` and their consuming pods -`Pending` (`unbound immediate PersistentVolumeClaims`). - -| PVC/event signal | Cause | -|------------------|-------| -| `ProvisioningFailed`, `failed to provision volume with StorageClass "X"` | CSI driver/provisioner error | -| `waiting for a volume to be created, either by external provisioner ...` | Provisioner not running/responding; missing CSI driver | -| `rpc error: code = ... exceeded`, `VolumeLimitExceeded`, `exceeded quota` | Cloud volume count/size limit or namespace quota | -| `node(s) exceed max volume count`, `Multi-Attach error for volume` | Per-node attach cap; volume already attached elsewhere | -| PVC `Pending` with no events, pod not yet scheduled | `WaitForFirstConsumer` — **normal** until a pod consumes it | - -Notes: - -- **`WaitForFirstConsumer`** binds only once a pod is scheduled; a `Pending` PVC here is - expected, not a failure. Only investigate if the *pod* is also stuck. -- **Topology conflict** — a bound PV in zone A with the pod forced to zone B yields - `volume node affinity conflict` at scheduling ([Pod Scheduling](#pod-scheduling-failures)). -- Cloud-side volume **quota/attach limits** are the storage face of - [cloud-provider-errors.md](cloud-provider-errors.md); CSI-driver crashes are a product/infra - bug — check the driver pods under `gather-extra/artifacts/pods/`. - -Look in `oc_cmds/pv`, `oc_cmds/events` (grep `ProvisioningFailed|Multi-Attach|exceed`), and -the CSI driver/`openshift-cluster-csi-drivers` pod logs. - ---- - -## CI Pod and Sidecar Resources - -The Prow test pod is multi-container: `test` (the workload/ci-operator entrypoint) plus a -Prow **`sidecar`** that streams, censors (redacts secrets from), and uploads logs and -artifacts; init containers include `clonerefs`, `initupload`, and `place-entrypoint`. All -run on the **build-farm**, not the cluster under test. - -- **`sidecar` OOM** — very large artifact/log volume makes the sidecar buffer/censor a lot; - it can be `OOMKilled` (exit `137`). Result: `build-log.txt` ends abruptly, artifacts are - truncated or missing, and the job errors even if tests passed. Confirm in `podinfo.json` - (container `sidecar`, `reason: OOMKilled`). -- **`test` container OOM/evicted** — the ci-operator/test process exceeds its step resource - request, or the build node is under pressure → `pod_pending`, `Process interrupted`, or a - truncated log. - -This is a **build-farm** (CI infrastructure) failure — retry, and reduce artifact size or -adjust the step's resource requests in `openshift/release`. Mechanics and `pod_pending` -detail: [ci-infrastructure-changes.md](ci-infrastructure-changes.md#pod_pending). Do not -mistake it for a cluster-under-test problem. - ---- - -## How Exhaustion Cascades - -Resource exhaustion rarely stays local; one shortage fans out into many unrelated failures: - -```text -memory limit hit → container OOMKilled → CrashLoopBackOff → dependency unavailable - → dependent test times out → many "unrelated" tests fail together - -node pressure → kubelet evicts pods → reschedule onto other nodes → those pressure too - → nodes NotReady → API/etcd on those nodes disrupted → cluster-wide disruption - -etcd NOSPACE → writes rejected → apiserver errors → operators degrade → conformance mass-fails -``` - -**Victim vs cause:** a test that *fails* during pressure is usually a **victim**. A test -that *runs* (and often passes) while consistently present during pressure across multiple -runs is a more likely **cause**. Identify the *first* resource event in time and trace -forward; the earliest OOM/eviction/alarm is typically the trigger, later failures are -fallout. CPU-starvation fan-out (OVS stalls, backend disruption) is analyzed in -[disruption.md](disruption.md). - ---- - -## Diagnosing from Must-Gather and Interval Files - -**Interval files** (`e2e-timelines_spyglass_*.json`) give a time-ordered view. Relevant -sources ([disruption.md](disruption.md#key-signal-sources) has the full catalog): - -- `NodeMonitor` / `MachineMonitor` — node Ready↔NotReady, pressure transitions -- `Alert` — `KubeletTooManyPods`, `etcdBackendQuotaLowSpace`, memory/disk alerts firing -- `KubeletLog` / `PodLog` — eviction and OOM lifecycle events -- `CPUMonitor`, `CloudMetrics`, `EtcdDisk*` — CPU/disk-I/O pressure (disruption territory) - -```bash -grep -oE '"(NodeMonitor|Alert|KubeletLog)"[^}]*"humanMessage":"[^"]*"' \ - *e2e-timelines_spyglass_*.json -``` - -**Must-gather** gives cluster state at collection time: - -1. `analyze_nodes.py --problems-only` → node conditions, allocatable, pressure/taints. -2. `analyze_pods.py --problems-only` → OOMKilled/Evicted/Pending pods, restart counts. -3. `analyze_events.py --type Warning` → `Evicted`, `FailedScheduling`, `ProvisioningFailed`, `SystemOOM`. -4. `analyze_etcd.py` → etcd DB size, alarms, compaction health. -5. Read the flagged pod YAMLs for `containerStatuses[].lastState.terminated`. - -Correlate the earliest resource event's timestamp against the failed test's window (from -`E2ETest` intervals) to separate cause from fallout. - -> Must-gather only exists if the cluster was stable enough to run `oc adm must-gather`. If -> absent, the cluster likely collapsed under the exhaustion — rely on `gather-extra`, -> journals, and interval files. See [artifacts.md](artifacts.md#must-gather-availability). - ---- - -## Quick Triage Checklist - -1. **Which domain?** Top-level `build-log.txt` / `podinfo.json` → build-farm (CI infra, - retry). gather-extra / must-gather / intervals → cluster under test. - ([Two Failure Domains](#two-failure-domains)) -2. **Which resource?** memory (OOM), disk/inodes (DiskPressure/ephemeral), PIDs, etcd space, - PVC — grep the strings in [Fast Triage](#fast-triage). -3. **Container-limit or node-level?** `containerStatuses` reason only → cgroup limit; node - `SystemOOM`/`MemoryPressure` → node oversubscribed. ([OOM Kills](#oom-kills)) -4. **Earliest event.** Find the first OOM/eviction/`FailedScheduling`/`NOSPACE` in time; - trace the cascade forward from there. ([Cascades](#how-exhaustion-cascades)) -5. **Cause or victim?** A failing test during pressure is usually a victim; a heavy test - present across runs during pressure is a candidate cause. -6. **Undersized or leak?** `Insufficient` on all nodes / SNO → capacity; steadily growing - memory/DB with restarts → leak. Retry-passes ⇒ transient infra; reproduces ⇒ real. - -## See Also - -- [disruption.md](disruption.md) — CPU starvation, disk-I/O latency, etcd I/O, OVS stalls, backend disruption -- [ci-infrastructure-changes.md](ci-infrastructure-changes.md) — build-farm capacity, `pod_pending`, CI-vs-product framework -- [cloud-provider-errors.md](cloud-provider-errors.md) — cloud quota, volume/EIP limits at install time -- [networking.md](networking.md) — OVN node-subnet / CNI IP exhaustion -- [install/general.md](install/general.md) — resource pressure during installation -- [artifacts.md](artifacts.md) — full artifact tree, gather-extra, must-gather layout diff --git a/plugins/ci/skills/prow-job-analysis/references/test-extension-binaries.md b/plugins/ci/skills/prow-job-analysis/references/test-extension-binaries.md deleted file mode 100644 index a57d0689c..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/test-extension-binaries.md +++ /dev/null @@ -1,814 +0,0 @@ -# Test Extension Binaries Reference - -Covers OpenShift test extension binaries (OTE): discovery and invocation by `openshift-tests`, -build and packaging in CI images, and how their failures surface in artifacts. Use this when a -test failure originates from an extension binary rather than the core `openshift-tests` binary. - -## When to Use - -- A test failure comes from a component-owned extension binary (not origin's own tests) -- Extension binary extraction or discovery fails during CI job startup -- Tests that should run are missing or not being registered -- Extension binary crashes, panics, or produces protocol errors -- Build failures result in missing or broken extension binaries in payload images -- Need to understand the relationship between extension binaries and `openshift-tests` - ---- - -## What Are Test Extension Binaries? - -Standalone executables implementing the **openshift-tests-extension (OTE) protocol**. They let -component teams ship their own e2e tests alongside their component images in the release payload, -without merging test code into `openshift/origin`. - -### The Extension Model - -Traditionally, all OpenShift e2e tests lived in `openshift/origin`, compiled into a single -`openshift-tests` binary. The extension model decentralizes this: - -| Aspect | Traditional (origin) | Extension Binary | -|--------|---------------------|-----------------| -| **Test code lives in** | `openshift/origin` | Component repo (e.g., `openshift/cluster-etcd-operator`) | -| **Binary name** | `openshift-tests` | `-tests-ext` (e.g., `cluster-etcd-operator-tests-ext`) | -| **Shipped in image** | `tests` payload image | Component's own payload image | -| **Test registration** | Compiled into binary | Discovered at runtime via OTE protocol | -| **Ownership** | Origin / TRT team | Component team | - -### Standard vs Extension Tests - -**Standard tests** (origin-native): -- Compiled directly into `openshift-tests` -- Registered via Ginkgo `Describe`/`It` blocks at import time -- Always present — no extraction or discovery needed - -**Extension tests**: -- Live in separate binaries shipped inside payload images -- Must be extracted from images and discovered at runtime -- Communicate with `openshift-tests` via a JSON-based protocol over stdout -- Can fail at multiple stages: extraction, discovery, registration, or execution - ---- - -## The OTE Protocol - -A command-line protocol that `openshift-tests` (origin) uses to discover and run tests, defined by -the [openshift-tests-extension](https://github.com/openshift-eng/openshift-tests-extension) framework. - -### Protocol Commands - -Every extension binary must support these subcommands: - -#### `info` — Report extension metadata - -```bash - info -``` - -Returns a JSON object describing the extension: -```json -{ - "apiVersion": "v1.1", - "source": { - "commit": "abc123...", - "build_date": "2026-01-15T10:30:00Z", - "git_tree_state": "clean", - "source_url": "https://github.com/openshift/cluster-etcd-operator" - }, - "component": { - "product": "openshift", - "type": "payload", - "name": "cluster-etcd-operator" - } -} -``` - -- Timeout: 10 minutes -- Errors here prevent the binary from being used at all - -#### `list` — Enumerate available tests - -```bash - list -o jsonl [--env-flag=VALUE...] -``` - -Returns one JSON object per line (JSONL format), each describing a test: -```jsonl -{"name":"[sig-etcd] etcd should start successfully","lifecycle":"blocking","labels":["sig-etcd"],"codeLocations":["/path/to/test.go:42"]} -{"name":"[sig-etcd] etcd leader election","lifecycle":"informing","labels":["sig-etcd"],"codeLocations":["/path/to/test.go:87"]} -``` - -- Timeout: 10 minutes -- Lines not starting with `{` are silently skipped (allows stderr/debug output) -- Environment flags are passed to let the binary determine platform-specific tests - -#### `run-test` — Execute specific tests - -```bash - run-test -n "test name 1" -n "test name 2" -o jsonl -``` - -Returns JSONL results at the **end** of stdout: -```jsonl -{"name":"[sig-etcd] etcd should start successfully","lifecycle":"blocking","result":"passed","duration":12500000000,"output":"..."} -{"name":"[sig-etcd] etcd leader election","lifecycle":"blocking","result":"failed","duration":45200000000,"error":"Error: ...","output":"..."} -``` - -`result` is one of `passed`/`failed`/`skipped`; `duration` is an int64 in **nanoseconds** -(12.5s = `12500000000`), not a string. - -- Results are parsed **backwards** from end of output to avoid stray JSON from logs -- Exit code is ignored (non-zero is expected when tests fail) -- `EXTENSION_ARTIFACT_DIR` environment variable tells the binary where to write artifacts - -#### `images` — List required test images (optional) - -```bash - images -``` - -Returns JSON describing container images the tests need. - -### JSON Extraction - -Origin's parser (`extractJSON`) is tolerant of non-JSON output before the actual JSON payload. -It scans stdout line-by-line looking for the first line that starts with `{` or `[`. This means -extension binaries can emit log lines to stdout before the JSON — but they must not emit -invalid JSON that looks like it starts a JSON object. - ---- - -## Extension Binary Lifecycle in CI - -### How Extension Binaries Are Built - -Built in each component's CI pipeline via a Dockerfile multi-stage build. Typical pattern: - -```dockerfile -# Build stage for test extension -FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS test-extension-builder -RUN mkdir -p /go/src/github.com/openshift/ -WORKDIR /go/src/github.com/openshift/ -COPY . . -RUN make tests-ext-build && \ - cd bin && \ - tar -czvf -test-extension.tar.gz -tests-ext && \ - rm -f -tests-ext - -# Final image (component's operator image) -FROM ... -COPY --from=test-extension-builder /go/src/github.com/openshift//bin/-test-extension.tar.gz /usr/bin/ -``` - -The extension binary is: -1. Compiled from Go source using the OTE framework -2. Compressed with gzip (`.gz` suffix in the registry) -3. Placed at a well-known path inside the component's payload image (typically `/usr/bin/`) - -### Image Layering - -Extension binaries live **inside the component's own release payload image**, not in the -`tests` image. For example: - -| Extension Binary | Payload Image Tag | Binary Path in Image | -|-----------------|-------------------|---------------------| -| `cluster-etcd-operator-tests-ext` | `cluster-etcd-operator` | `/usr/bin/cluster-etcd-operator-tests-ext.gz` | -| `machine-config-tests-ext` | `machine-config-operator` | `/usr/bin/machine-config-tests-ext.gz` | -| `ovn-kubernetes-tests-ext` | `ovn-kubernetes` | `/usr/bin/ovn-kubernetes-tests-ext.gz` | -| `k8s-tests-ext` | `hyperkube` | `/usr/bin/k8s-tests-ext.gz` | -| `oc-tests-ext` | `cli` | `/usr/bin/oc-tests-ext.gz` | -| `olmv1-tests-ext` | `olm-operator-controller` | `/usr/bin/olmv1-tests-ext.gz` | - -### The Extension Binary Registry - -Origin maintains a **hardcoded registry** of known extension binaries in -`pkg/test/extensions/binary.go`, mapping each payload image tag to the binary path within it: - -```go -var extensionBinaries = []TestBinary{ - // Self reference for origin's own internal extension - {imageTag: "tests", binaryPath: os.Args[0]}, - - // Extensions in other payload images - {imageTag: "aws-machine-controllers", binaryPath: "/machine-api-provider-aws-tests-ext.gz"}, - {imageTag: "cli", binaryPath: "/usr/bin/oc-tests-ext.gz"}, - {imageTag: "cluster-etcd-operator", binaryPath: "/usr/bin/cluster-etcd-operator-tests-ext.gz"}, - {imageTag: "cluster-ingress-operator", binaryPath: "/usr/bin/cluster-ingress-operator-tests-ext.gz"}, - {imageTag: "machine-config-operator", binaryPath: "/usr/bin/machine-config-tests-ext.gz"}, - {imageTag: "ovn-kubernetes", binaryPath: "/usr/bin/ovn-kubernetes-tests-ext.gz"}, - // ... 30+ more entries -} -``` - -**Important**: A new extension binary must be **registered** in origin's `extensionBinaries` -list to be discovered. Without registration, the binary ships in the image but is never used. - -### Non-Payload Extension Discovery - -Origin also supports **non-payload extensions** — test binaries in images outside the release -payload, discovered dynamically via ImageStreamTag annotations in the cluster: - -1. Origin checks for `TestExtensionAdmission` CRDs that define permitted patterns -2. It scans ImageStreamTags across namespaces for `ComponentAnnotation` -3. Permitted extensions are extracted and run alongside payload extensions -4. Unpermitted extensions generate synthetic skip tests - -### Extraction Process (`ExtractAllTestBinaries`) - -When `openshift-tests` starts a test run, it extracts all extension binaries: - -1. **Filter by tags**: Apply `EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS` / - `EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS` environment variables -2. **Determine release image**: Find the payload image from `RELEASE_IMAGE_LATEST` or cluster -3. **Create temp directory**: `/tmp/external-binary-*` for extracted binaries -4. **Extract in parallel**: Use `oc image extract` to pull each binary from its payload image - - Self-reference (`tests` image) is skipped — origin is already running - - Binaries are decompressed from `.gz` format - - Made executable -5. **Discover non-payload extensions**: Scan ImageStreamTags for permitted extensions -6. **Return all binaries**: Both payload and non-payload extensions - -### Environment Variable Controls - -| Variable | Purpose | -|----------|---------| -| `OPENSHIFT_SKIP_EXTERNAL_TESTS` | If set, skip ALL external extension binaries (use only origin's built-in tests) | -| `EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS` | Comma-separated image tags — use ONLY these extensions | -| `EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS` | Comma-separated image tags — exclude these extensions | -| `EXTENSION_ARTIFACT_DIR` | Set per-extension — where the binary should write its test artifacts | -| `ARTIFACT_DIR` | Base artifact directory for the CI job | -| `TEST_PROVIDER` | Cluster provider config (passed to extensions via environment flags) | -| `TEST_JUNIT_DIR` | Where JUnit XML results should be written | - ---- - -## Common Extension Binary Failure Patterns - -### 1. Binary Not Found / Missing from Image - -**Symptoms:** -```text -failed to extract binary from image: file not found: /usr/bin/cluster-foo-operator-tests-ext.gz -``` -or -```text -encountered errors while extracting binaries: couldn't extract /usr/bin/component-tests-ext.gz from image ... -``` - -**Causes:** -- Component repo's Dockerfile doesn't include the test extension build stage -- Binary path in the registry doesn't match the actual path in the image -- Image build failed silently — the extension stage compiled but didn't copy the artifact -- Image was built from a branch that doesn't have the OTE integration yet - -**Diagnosis:** -```bash -# Check if the binary exists in the payload image -oc image extract --path /usr/bin/: --confirm -ls -la /*tests-ext* - -# Check the Dockerfile in the component repo for test extension stages -grep -n "test-extension\|tests-ext" Dockerfile* -``` - -### 2. Binary Fails to Execute - -**Symptoms:** -```text -failed running ' info': exec: "": permission denied -failed running ' info': exec format error -failed running ' info': no such file or directory (dynamic linker) -``` - -**Causes:** -- Binary not marked executable after extraction -- Architecture mismatch (e.g., binary built for amd64, running on arm64) -- Dynamic linking failure — binary depends on shared libraries not in the test pod -- Corrupted binary (incomplete download or extraction) -- Binary was gzipped but path in registry doesn't have `.gz` suffix (or vice versa) - -**Diagnosis:** -```bash -# Check binary properties -file -ldd # Check dynamic linking -chmod +x && ./ info # Test execution -``` - -### 3. Binary Starts but Fails to Register Tests (Protocol Errors) - -**Symptoms:** -```text -no valid JSON found in output from ' info' command -couldn't unmarshal extension info from : ... -failed running ' list': ... -``` -or -```text -no valid JSON found in output: -``` - -**Causes:** -- Binary writes non-JSON to stdout before the JSON payload and it starts with `{` or `[` -- Binary crashes during `info` or `list` with a panic that precedes JSON output -- Binary built with incompatible OTE framework version (API version mismatch) -- Binary requires cluster connectivity for `list` but cluster is unavailable at that point -- Initialization code (e.g., `framework.AfterReadingAllFlags(&framework.TestContext)`) was never - called before the binary emitted output, leaving test context unpopulated - -**Diagnosis:** -```bash -# Run the binary manually to see raw output -./ info 2>&1 -./ list -o jsonl 2>&1 | head -20 - -# Check API version compatibility -./ info 2>&1 | jq .apiVersion -``` - -**Build log patterns:** -```text -Raw output from : panic: runtime error: invalid memory address or nil pointer dereference -No valid JSON found in output from info command -``` - -### 4. Binary Registers Tests but They Don't Run - -**Symptoms:** -- Extension binary extracts and lists tests successfully -- But tests are absent from JUnit results -- Test count in build log shows 0 tests selected from extension - -**Causes:** -- Test suite filtering (`--suite`) excludes extension tests -- Environment flags cause extension to report 0 applicable tests -- Test names don't match selection regex -- Extension tests are all set to `LifecycleInforming` and the suite only runs blocking tests -- Extension binary's `list` command returned tests, but none matched the current environment - -**Diagnosis:** -```bash -# Check what tests the extension advertises -./ list -o jsonl | wc -l -./ list -o jsonl | jq -r '.name' | head -20 - -# Check test lifecycle -./ list -o jsonl | jq -r '.lifecycle' | sort | uniq -c -``` - -### 5. Extension Binary Panics During Test Execution - -**Symptoms in build log:** -```text -panic: runtime error: invalid memory address or nil pointer dereference -goroutine 1 [running]: -github.com/openshift/cluster-foo-operator/test/e2e.TestSomething(...) - /go/src/github.com/openshift/cluster-foo-operator/test/e2e/test.go:42 +0x1a4 -``` - -**Common panic causes:** -- Missing `framework.AfterReadingAllFlags(&framework.TestContext)` initialization -- Missing `util.WithCleanup()` in BeforeAll (leaves `testsStarted = false`) -- Nil kubeconfig or missing KUBECONFIG environment variable -- Test expects cluster resources that don't exist in the test environment -- OTE framework version mismatch — binary built with old framework, run by new origin - -**Diagnosis:** -- Look for `panic:` in the step-level build log -- Get the full goroutine dump — the crashing function and file:line -- Check if the component's `cmd/extension/main.go` has proper initialization - -### 6. Extension Binary Timeout Issues - -**Symptoms:** -```text -context deadline exceeded -signal: killed -``` - -**Causes:** -- `info` command: Timeout is 10 minutes — binary hangs during initialization -- `list` command: Timeout is 10 minutes — binary attempts cluster operations during listing -- `run-test` command: Timeout matches the test timeout — test itself runs too long -- Binary enters an infinite loop or deadlock - -**Diagnosis:** -- Check build log for timing — did the binary run for the full timeout period? -- Check if the binary requires cluster access for `info`/`list` (it shouldn't) - ---- - -## How Extension Binary Failures Appear in Artifacts - -### JUnit XML Output - -Extension test results appear in JUnit XML alongside standard tests: - -**Standard test in JUnit:** -```xml - - -``` - -**Extension test in JUnit:** -```xml - - -``` - -Both appear under `classname="openshift-tests"` because origin orchestrates all test execution. -To identify which binary owned a test, check the test name prefix (e.g., `[sig-etcd]`) or the -build log's binary-to-test mapping. - -**Extension binary extraction failure in JUnit:** - -When an extension binary fails to extract or initialize, origin may generate a **synthetic -test failure**: -```xml - - - encountered errors while extracting binaries: couldn't extract /usr/bin/cluster-etcd-operator-tests-ext.gz - - -``` - -**Unpermitted non-payload extension in JUnit:** -```xml - - - -``` - -### Build-Log Patterns - -#### Successful extension loading: -```text -level=info msg="Fetching info for cluster-etcd-operator-tests-ext.gz" -level=info msg="Fetched info for cluster-etcd-operator-tests-ext.gz in 2.1s" -level=info msg="Listing tests" binary=cluster-etcd-operator-tests-ext.gz -level=info msg="OTE API version is: v1.1" binary=cluster-etcd-operator-tests-ext.gz -level=info msg="Listed 42 tests in 3.5s" binary=cluster-etcd-operator-tests-ext.gz -``` - -#### Extension binary extraction failure: -```text -level=error msg="Failed to extract binary from image tag 'cluster-foo-operator': file not found" -encountered errors while extracting binaries: couldn't extract /usr/bin/cluster-foo-operator-tests-ext.gz from image -``` - -#### Extension binary info/protocol failure: -```text -level=error msg="Failed to fetch info for cluster-foo-operator-tests-ext.gz: exit status 1" -level=error msg="Command output for cluster-foo-operator-tests-ext.gz: panic: runtime error: ..." -level=error msg="No valid JSON found in output from cluster-foo-operator-tests-ext.gz info command" -level=error msg="Raw output from cluster-foo-operator-tests-ext.gz: " -``` - -#### Extension binary list failure: -```text -failed running 'cluster-foo-operator-tests-ext.gz list': exit status 2 -Output: Error: unable to load kubeconfig: ... -``` - -#### Skipping external tests: -```text -level=warning msg="Using built-in tests only due to OPENSHIFT_SKIP_EXTERNAL_TESTS being set" -``` - -#### Tag filtering: -```text -level=info msg="Including extension binary with image tag: cluster-etcd-operator" -level=info msg="Excluding extension binary with image tag: ovn-kubernetes (not in include list)" -``` - -### How to Distinguish "Extension Binary Broken" from "Test Actually Failed" - -| Signal | Extension Binary Issue | Test Actually Failed | -|--------|----------------------|---------------------| -| Error during `info` or `list` | ✅ Binary is broken | ❌ | -| Panic before test output | ✅ Binary initialization failed | ❌ | -| Test runs and produces pass/fail result | ❌ | ✅ Test logic failed | -| "no valid JSON found" in build log | ✅ Protocol error | ❌ | -| Extension not found in JUnit at all | ✅ Extraction failed | ❌ | -| Test listed but never ran | ⚠️ Possibly filtering issue | ⚠️ Possibly skipped | -| Test ran and produced failure message | ❌ | ✅ Check the failure | - ---- - -## Debugging Workflow - -### Step 1: Check if the Extension Binary Exists in the Test Image - -```bash -# Find the binary's image tag in origin's registry -# (see the Known Extension Binary Implementations table below) - -# Extract and check the payload image contents -oc adm release info --image-for= -oc image extract --path /usr/bin/: --confirm -ls -la /*tests-ext* -``` - -**In CI artifacts**: Look at the build-log.txt for extraction messages: -```bash -# Search build log for extension extraction -grep -i "extract\|extension\|tests-ext" build-log.txt -``` - -### Step 2: Check if it Was Discovered/Loaded - -Search the build log for the binary name: -```bash -grep "" build-log.txt -# e.g., grep "cluster-etcd-operator-tests-ext" build-log.txt -``` - -Expected success pattern: -```text -Fetching info for .gz -Fetched info for .gz in Xs -``` - -Failure pattern: -```text -Failed to fetch info for .gz: ... -Failed to extract binary from image tag '': ... -``` - -### Step 3: Check if Tests Were Registered - -```bash -grep "Listed.*tests" build-log.txt -# Expected: "Listed 42 tests in 3.5s" -# Problem: "Listed 0 tests in 0.1s" or absence of this line -``` - -If tests were registered, they'll appear in the test selection output: -```bash -grep "Found.*test specs" build-log.txt -``` - -### Step 4: Check Test Execution Output - -Look at the step-level build log for the test step (e.g., `openshift-e2e-test`): -```bash -# Download the step build log -gcloud storage cp \ - "gs://test-platform-results/{bucket-path}/artifacts/{target}/openshift-e2e-test/build-log.txt" \ - ./step-build-log.txt --no-user-output-enabled - -# Search for extension test output -grep -A5 "run-test\|RunTests" step-build-log.txt -``` - -### Step 5: Check for Infrastructure vs Test Issues - -Determine whether the failure is: - -1. **Infrastructure** — binary missing, extraction failed, protocol error - - Fix: Check the component's Dockerfile, rebuild, or update origin's registry -2. **Framework initialization** — panic during setup, missing config - - Fix: Check `cmd/extension/main.go` initialization code -3. **Test logic** — test ran but assertion failed - - Fix: Investigate the test code and cluster state (normal test debugging) -4. **Environment mismatch** — tests not applicable to this platform - - Fix: Check environment flags and test filtering logic - ---- - -## Extension Binary Naming — Pattern and Exceptions - -Most binaries follow **`` → `/usr/bin/-tests-ext.gz`**. Don't rely on a -memorized list — the registry (`extensionBinaries` in `pkg/test/extensions/binary.go`, see -[The Extension Binary Registry](#the-extension-binary-registry) above) grows as components are -added. Read the current list straight from origin: - -```bash -curl -s https://raw.githubusercontent.com/openshift/origin/master/pkg/test/extensions/binary.go \ - | grep -oE '(imageTag|binaryPath): "[^"]*"' -``` - -**Watch for these naming exceptions** — they break the `-tests-ext.gz` convention, so a -pattern-based guess of the binary path will be wrong: - -| Image Tag | Binary Path | Deviation | -|-----------|-------------|-----------| -| `cluster-node-tuning-operator` | `/usr/bin/cluster-node-tuning-operator-test-ext.gz` | `-test-ext` (singular) | -| `cluster-control-plane-machine-set-operator` | `/cluster-control-plane-machine-set-operator-ext.gz` | `-ext.gz`, no `/usr/bin` | -| `cluster-version-operator` | `/usr/bin/cluster-version-operator-tests.gz` | `-tests.gz`, no `-ext` | -| `machine-api-operator` | `/machine-api-tests-ext.gz` | no `/usr/bin` prefix | -| `aws-machine-controllers` | `/machine-api-provider-aws-tests-ext.gz` | binary name ≠ image tag | -| `hyperkube` | `/usr/bin/k8s-tests-ext.gz` | upstream Kubernetes | - -An image can ship **two** binaries (e.g. `cluster-cloud-controller-manager-operator` has both -a generic and an AWS binary) — expect a tag to appear more than once in the registry. - ---- - -## Packaging and Build Issues - -### Build Steps and Entry Point - -Extension binaries are built from their component repos using the OTE framework: - -1. **Source code**: Test files live in `test/e2e/` within the component repo -2. **Entry point**: `cmd/extension/main.go` initializes the OTE framework -3. **Build**: `make tests-ext-build` compiles the binary to `bin/-tests-ext` -4. **Package**: Dockerfile compresses and copies into the component image - -#### Typical `main.go` structure: -```go -package main - -import ( - g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" - et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" - // Import test packages to register Ginkgo tests - _ "github.com/openshift//test/e2e" -) - -func main() { - // Build test specs from Ginkgo suite - allSpecs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() - - // Filter to only local tests (exclude vendored tests) - componentSpecs := allSpecs.Select(func(spec *et.ExtensionTestSpec) bool { - for _, loc := range spec.CodeLocations { - if strings.Contains(loc, "/test/e2e/") && - !strings.Contains(loc, "/go/pkg/mod/") && - !strings.Contains(loc, "/vendor/") { - return true - } - } - return false - }) - - // Set lifecycle (informing = won't block CI) - componentSpecs.Walk(func(spec *et.ExtensionTestSpec) { - spec.Lifecycle = et.LifecycleInforming - }) - - ext.AddSpecs(componentSpecs) - // ... register with extension framework and serve -} -``` - -### Version Skew Between Extension Binaries and openshift-tests - -Version skew across the OTE protocol can cause issues: - -| Issue | Symptom | Resolution | -|-------|---------|------------| -| Extension uses newer API version | Origin filters out unsupported env flags | Update origin | -| Extension uses older API version | Missing required fields in JSON | Rebuild extension with newer OTE framework | -| OTE framework mismatch | `undefined: ginkgo.NewWriter` build errors | Align Ginkgo version with OTE framework | -| Go version mismatch | Build fails in Dockerfile | Use matching builder image version | - -**API version compatibility**: Origin's `filterToApplicableEnvironmentFlags` checks the -extension's reported `apiVersion` and only passes environment flags that the extension's -version supports. This provides forward compatibility — newer origin versions won't break -older extensions with flags they don't understand. - -### Build Failures That Result in Missing/Broken Extensions - -1. **Go dependency conflicts**: - ```text - undefined: otelgrpc.UnaryClientInterceptor - cannot use v6 as net.IP - ``` - Fix: Update Go dependencies, especially `openshift/kubernetes` fork - -2. **Missing Ginkgo APIs**: - ```text - undefined: ginkgo.NewWriter - spec.Labels undefined - ``` - Fix: Update Ginkgo to the version aligned with OTE framework - (v2.27.2-openshift-4.22 branch of `openshift/onsi-ginkgo`) - -3. **Go version too old for dependencies**: - ```text - go: go.mod requires go >= 1.25.0 (running go 1.23; GOTOOLCHAIN=local) - ``` - Fix: Use `GOTOOLCHAIN=auto` or update builder image - -4. **k8s.io/kms requires newer Go**: - ```text - k8s.io/kms requires go >= 1.25.0 - ``` - Fix: Add k8s.io/kms replace directive to use OpenShift fork - -5. **Test filtering not working — 1000+ tests registered**: - ```text - Found 1247 test specs (expected ~50) - ``` - Fix: Check `main.go` filesystem path filtering — ensure it excludes - `/go/pkg/mod/` and `/vendor/` paths - ---- - -## Test Selection with Extensions - -### How Suite and Name Filtering Interact with Extensions - -After discovery, extension tests are integrated into origin's test selection: - -1. **All extension binaries list their tests** via the `list` command -2. **Origin merges** all extension tests with its own built-in tests -3. **Suite selection** (`--suite openshift/conformance/parallel`) filters the combined set -4. **Name matching** (`--run "etcd"`) further filters by test name regex -5. **Environment selectors** exclude tests not applicable to the current platform - -Extension binaries can define their own suites: -```text -/conformance/parallel -/conformance/serial -/disruptive -/all -``` - -These suite names are prefixed with the component name to avoid collisions. - -### How Extension Tests Are Named and Categorized - -Extension tests follow standard OpenShift test naming conventions: - -```text -[sig-] [] -``` - -Examples: -```text -[sig-etcd] etcd cluster health should be recoverable [Disruptive] -[sig-network] OVN pods should be running [Serial] -[sig-auth] OAuth server tokens should be refreshed -``` - -Labels that affect categorization: -- `[Serial]` — must run sequentially, not in parallel -- `[Disruptive]` — may affect cluster state, run in isolation -- `[Skipped:...]` — conditions under which the test is skipped -- `[Level0]` — conformance test (used by OTE-migrated tests) -- `[OTP]` — test ported from openshift-tests-private (tracking marker) - -### How to Run Only Extension Tests or Exclude Them - -**Run only specific extension's tests:** -```bash -# Set environment to include only one extension -export EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS=cluster-etcd-operator -openshift-tests run ... -``` - -**Exclude specific extensions:** -```bash -# Exclude a known-broken extension -export EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS=cluster-foo-operator,ovn-kubernetes -openshift-tests run ... -``` - -**Skip all external extensions:** -```bash -# Run only origin's built-in tests -export OPENSHIFT_SKIP_EXTERNAL_TESTS=1 -openshift-tests run ... -``` - ---- - -## Artifact Locations for Extension Tests - -Stored under a component-specific subdirectory: - -```text -$ARTIFACT_DIR//// -``` - -For example: -```text -artifacts/e2e-aws/openshift-e2e-test/artifacts/openshift/payload/cluster-etcd-operator/ -artifacts/e2e-aws/openshift-e2e-test/artifacts/openshift/payload/ovn-kubernetes/ -``` - -Each extension's artifact directory contains: -- JUnit XML results (if the extension writes them) -- Custom artifacts (logs, dumps, screenshots) -- Whatever the extension binary chooses to write to `EXTENSION_ARTIFACT_DIR` - ---- - -## Cross-References - -- [CI Infrastructure Changes](ci-infrastructure-changes.md) — CI step script and infrastructure - analysis, including changes to ci-operator and Prow test infrastructure -- [Aggregated Jobs](aggregated.md) — Statistical test analysis across multiple runs, including - extension test result aggregation -- [Artifacts Reference](artifacts.md) — Complete artifact directory structure -- [OTE Framework Repository](https://github.com/openshift-eng/openshift-tests-extension) — - Source code for the extension framework -- [OTE Migration Plugin](../../../../ote-migration/README.md) — Automated migration tool for - adding OTE support to component repos -- Origin extension registry: `openshift/origin` → `pkg/test/extensions/binary.go` diff --git a/plugins/ci/skills/prow-job-analysis/references/test-failure.md b/plugins/ci/skills/prow-job-analysis/references/test-failure.md deleted file mode 100644 index f6a997179..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/test-failure.md +++ /dev/null @@ -1,237 +0,0 @@ -# Test Failure Root-Cause Analysis - -Use when [flaky-test-identification.md](flaky-test-identification.md) has classified a failure -as a real product regression in a plain e2e test (non-extension, non-install, non-upgrade) and -you need the root cause — for example, a conformance test like -`[sig-network] services should serve endpoints` that was stable on Sippy until it began failing -in `e2e-aws-ovn`. - -This assumes the three-way triage is already done: the failure is not a ci-operator `reason`, -not a fail+pass flake twin, not a ``, and not shared infra across 3+ jobs. If that is -not yet settled, start at [flaky-test-identification.md](flaky-test-identification.md). - -Use a different reference when the failing test is: an extension binary (`*-tests-ext`) → -[test-extension-binaries.md](test-extension-binaries.md); `install should succeed` → -[install/general.md](install/general.md); an upgrade-phase regression → -[upgrade.md](upgrade.md); an `aggregated-` job verdict → [aggregated.md](aggregated.md). - ---- - -## The method - -**Never stop at a high-level symptom.** "The pod is crash-looping", "the operator went -Degraded", "the request timed out" are restatements of the failure, not causes. Each one has -an originating error one or two hops upstream. The job is to follow that chain: - -```text -JUnit assertion → test source (what it asserted, on what resource) - → cluster state in that namespace at the failure window - → the failing container's own error (previous.log, exitCode) - → the specific root cause (config, code, dependency) -``` - -Work it in that order. Each step narrows *where* and *when* to look for the next. - ---- - -## Step 1 — Read the JUnit failure - -The `` message and its stack trace are the entry point: they name the assertion, the -resource, and the moment of failure. JUnit is the source of truth over any alarming build-log -line (see [flaky-test-identification.md](flaky-test-identification.md#junit-interpretation)). - -```bash -# openshift-tests results -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/openshift-e2e-test/artifacts/junit/junit_e2e_*.xml" \ - .work/prow-job-analysis/{build_id}/ --no-user-output-enabled -``` - -Extract three things from the failing ``: - -| Extract | From | Feeds | -|---------|------|-------| -| **Assertion** — expected vs actual, resource name/namespace | `` | Step 2 (source), Step 3 (scope) | -| **Stack trace** — the code path that failed | `` body | Step 2 (source file/line) | -| **Failure window** — when the test ran and failed | `time=` attr + interval files (Step 4) | Step 4 (correlation) | - -Read the stack-trace shape — it tells you the failure *kind* before you read a single log: - -| Trace shape | Kind | Where the cause lives | -|-------------|------|-----------------------| -| `Expected to equal ` / Gomega matcher | Assertion on cluster state | The resource the assertion names — Steps 3–5 | -| `timed out waiting for ... condition` / `context deadline exceeded` | The awaited state never arrived | The component that should have produced it — Steps 3–5 | -| `panic:` + goroutine stack | Test-code or client-go bug | Test source (Step 2); a nil/response from the API | -| `connection refused` / `no route to host` / `EOF` | Endpoint unreachable | Networking or the backend pod — Steps 3–5 | - ---- - -## Step 2 — Locate the test source from the test name - -The test name is the concatenation of its Ginkgo node texts, so it maps back to source. Read -the source to learn **what the test actually asserts** and **which resources it touches** — -that is what you correlate in Steps 3–5. - -- The `[sig-*]` prefix names the owning SIG. OpenShift-authored e2e tests live in - [`openshift/origin`](https://github.com/openshift/origin) under `test/extended/`; upstream - Kubernetes conformance tests are vendored there from `k8s.io/kubernetes/test/e2e/`. -- Find the source by grepping a distinctive **literal fragment** of the name (the `It(...)` - text, not the full `[sig-x]` string) in `openshift/origin`: - -```bash -# In a local openshift/origin checkout — quote a stable, unique phrase from the test name -grep -rn "should serve endpoints" test/extended/ vendor/k8s.io/kubernetes/test/e2e/ -``` - -- For an extension test (`*-tests-ext`), the source repo and file are reported directly: the - `codeLocations` field in the binary's `list`/`run-suite` metadata gives `file:line`. See - [test-extension-binaries.md](test-extension-binaries.md). - -Read the assertion and the setup it depends on (namespaces created, images pulled, services -or CRs applied). A test failing at a `Consistently`/`Eventually` on some resource is telling -you *that resource* misbehaved — go check it, do not re-debug the test harness. - ---- - -## Step 3 — Scope to namespace and component from the test name - -The SIG prefix and the assertion's target namespace point at the operator and pod logs worth -reading. Pull those, not the whole cluster. - -| Test signal | Component / namespaces to inspect | Domain reference | -|-------------|-----------------------------------|------------------| -| `[sig-network]`, service/endpoint/DNS/ingress | `openshift-ovn-kubernetes`, `openshift-dns`, `openshift-ingress`, `openshift-network-operator` | [networking.md](networking.md) | -| `[sig-storage]`, PV/PVC/CSI | `openshift-cluster-csi-drivers`, `openshift-cluster-storage-operator` | [resource-exhaustion.md](resource-exhaustion.md) (capacity) | -| `[sig-api-machinery]`, apiserver/CRD/webhook | `openshift-kube-apiserver`, `openshift-apiserver` | [disruption.md](disruption.md) (apiserver availability) | -| `[sig-etcd]`, quorum/leader | `openshift-etcd` | [resource-exhaustion.md](resource-exhaustion.md) (etcd disk/space) | -| `[sig-node]`, kubelet/scheduling/eviction | node journals, `openshift-machine-config-operator` | [resource-exhaustion.md](resource-exhaustion.md) | -| `[sig-auth]`, RBAC/oauth/SCC | `openshift-authentication`, `openshift-oauth-apiserver` | — | -| `[sig-arch]` / monitor / invariant | Cluster-wide post-run invariant, not one component | [disruption.md](disruption.md) | - -Cluster state for these namespaces is in `gather-extra` and must-gather (paths below; -full tree in [artifacts.md](artifacts.md)): - -```bash -# Cluster operator status and events at gather time -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/co" .work/prow-job-analysis/{build_id}/ --no-user-output-enabled -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/events" .work/prow-job-analysis/{build_id}/ --no-user-output-enabled -``` - ---- - -## Step 4 — Pin the failure window and correlate cluster events - -A cluster event is a cause only if it **precedes and overlaps** the failure. Establish the -window, then look for an operator/event transition inside it. - -```bash -# Interval (timeline) files — nested at unpredictable depth -gcloud storage ls 'gs://test-platform-results/{bucket-path}/**/e2e-timelines_spyglass_*.json' -``` - -1. **Window** — find the interval with `source="E2ETest"` and - `message.annotations.status="Failed"` for your test. Its `from`/`to` timestamps are the - failure window. -2. **Overlap** — filter intervals to `level="Error"|"Warning"` and `source="OperatorState"` - that overlap that window. An operator going Degraded/Unavailable *before* the test failed, - in the namespace from Step 3, is a prime suspect. -3. **Direction of causation** — a transition that starts before the window and clears after is - a cause; one that starts only after the test already failed is a downstream effect. State - the timing explicitly, e.g. "test failed at 10:23:45; network operator went Degraded at - 10:23:12 (reason: `OVNKubernetesController`)". - -`oc_cmds/events` and must-gather per-namespace events give the same signal with reasons and -counts. Cross-check apiserver audit logs when the assertion involves API calls -([artifacts.md](artifacts.md#audit_logs--api-server-audit-logs)). For deep timeline -interpretation — cause vs symptom vs noise — see [disruption.md](disruption.md). - ---- - -## Step 5 — Trace crash-looping / failing containers to the originating error - -When Steps 3–4 point at a pod that restarted or never became ready, the cause is in that -pod's own status and prior log — not in the phrase "crash-looping". - -**Container status** — the richest single source is the pod YAML in must-gather -(`namespaces/{ns}/pods/{pod}/{pod}.yaml`), field `status.containerStatuses[]`: - -| Field | Read as | -|-------|---------| -| `lastState.terminated.exitCode` | **137** = SIGKILL — OOMKilled or liveness-probe kill; **143** = SIGTERM (graceful); **1/2** = application error/panic; **0** + restart = clean exit then restart | -| `lastState.terminated.reason` | `OOMKilled` → memory ([resource-exhaustion.md](resource-exhaustion.md)); `Error` → read the log; `ContainerCannotRun` → image/command | -| `restartCount` | High and climbing = CrashLoopBackOff; correlate the restart times with Step 4's window | -| `state.waiting.reason` | `ImagePullBackOff`/`ErrImagePull` → pull path ([networking.md](networking.md)); `CreateContainerError` → config/secret/mount | - -**Previous log** — the current log shows the healthy restart; the crash is in the *previous* -container. That log holds the originating error: - -```bash -# gather-extra keeps current.log and previous.log per container -gcloud storage cp -r "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/{namespace}/" \ - .work/prow-job-analysis/{build_id}/pods/ --no-user-output-enabled -# read {pod}/{container}/previous.log — the panic/fatal/config error that caused the restart -``` - -**Follow the dependency chain.** A crashing pod usually blames something upstream: a config it -read, a dependency it dialed, a volume it mounted, a CR it reconciled. Trace one hop further -until you reach an error that is actionable (a specific bad value, a missing object, a code -path), not another "X is unavailable". - ---- - -## Common e2e failure categories → typical root cause - -| Category | Signature in JUnit / logs | Typical root cause | Route / next step | -|----------|---------------------------|--------------------|-------------------| -| Endpoint/service unreachable | `connection refused`, `no endpoints available`, `i/o timeout` | No ready backend, OVN programming lag, DNS | [networking.md](networking.md) | -| Eventual-condition timeout | `timed out waiting for condition` on a resource | The producing controller/operator stalled | Step 3 operator → Step 5 pod | -| Pod never Ready | `CrashLoopBackOff`, non-zero `exitCode` in status | The container's own `previous.log` error | Step 5 | -| OOM / node pressure | `exitCode 137`, `OOMKilled`, `NodeNotReady`, evictions | Memory/disk/PID exhaustion | [resource-exhaustion.md](resource-exhaustion.md) | -| API flakiness during test | `the server was unable to return a response`, audit gaps | apiserver/etcd disruption in the window | [disruption.md](disruption.md), [resource-exhaustion.md](resource-exhaustion.md) | -| Image pull failure | `ImagePullBackOff`, `manifest unknown`, `x509` | Registry/mirror/trust | [networking.md](networking.md) | -| Test-code panic | `panic:` + goroutine stack in the test binary | Test bug (nil deref, bad type-assert) | Step 2 source; likely a test-repo fix | -| Cloud API error mid-test | `RequestLimitExceeded`, quota, `InsufficientInstanceCapacity` | Cloud throttling/capacity | [cloud-provider-errors.md](cloud-provider-errors.md) | - -A `panic:` in the *test binary* points at the test repo; a `panic:` or fatal in a *product -pod* points at the product. Distinguish by which process the trace belongs to (Step 1 shape, -Step 5 pod log). - ---- - -## Root-cause synthesis - -Before writing the conclusion, confirm the chain holds end to end: - -1. **Assertion identified** — the exact expected-vs-actual and the resource it names (Step 1). -2. **Source read** — what the test asserted and depended on, not a guess from the name - (Step 2). -3. **Window pinned** — the failure's `from`/`to`, with the correlated event *preceding* it and - in the right namespace (Steps 3–4). -4. **Originating error reached** — a specific `previous.log` line, `exitCode`+`reason`, or - config value — not "pod crash-looping" or "operator degraded" (Step 5). -5. **Domain confirmed** — if the root cause landed in networking/storage/resource/cloud, - the matching reference agrees and owns the fix. - -State the primary cause, the evidence (with timestamps), any contributing factors, and where -the fix lives (product repo, test repo, or a domain reference). If the chain breaks — no -correlated event, source doesn't match the symptom — re-check the flake/infra classification -in [flaky-test-identification.md](flaky-test-identification.md) before concluding. - -## See Also - -- [flaky-test-identification.md](flaky-test-identification.md) — the triage that routes here; - JUnit interpretation, Sippy baselines, infra-vs-flake-vs-regression -- [artifacts.md](artifacts.md) — full artifact tree: `gather-extra`, must-gather, pod YAMLs, - interval and audit paths -- [networking.md](networking.md) — service/DNS/OVN/ingress and image-pull root causes -- [resource-exhaustion.md](resource-exhaustion.md) — OOM (`exitCode 137`), node pressure, - etcd/disk exhaustion behind timeouts -- [disruption.md](disruption.md) — interval/timeline interpretation; apiserver availability in - the failure window -- [cloud-provider-errors.md](cloud-provider-errors.md) — cloud API/quota/capacity errors - during a test -- [test-extension-binaries.md](test-extension-binaries.md) — `*-tests-ext` source - (`codeLocations`) and extension-specific failures -- [hypershift.md](hypershift.md) — plain test failures in HCP jobs; correlate management vs - hosted cluster -- [aggregated.md](aggregated.md) — statistical verdict when the failing test is in an - `aggregated-` job diff --git a/plugins/ci/skills/prow-job-analysis/references/upgrade.md b/plugins/ci/skills/prow-job-analysis/references/upgrade.md deleted file mode 100644 index 8ed30a6fe..000000000 --- a/plugins/ci/skills/prow-job-analysis/references/upgrade.md +++ /dev/null @@ -1,1188 +0,0 @@ -# Upgrade Failure Analysis Reference - -Use when the job name contains `upgrade` and the failure occurs during or after the upgrade -phase — the CVO (Cluster Version Operator) reports errors or gets stuck, operators go degraded -or unavailable, the MCO (Machine Config Operator) fails to drain or reboot nodes, ClusterVersion -is stuck `Progressing=True`, or tests fail in the upgrade window or post-upgrade conformance. -This is the most complex failure category: multi-phase orchestration, rolling node updates, -operator reconvergence, and version skew between components. - -## Upgrade Job Classification - -### Identifying Upgrade Jobs from Name - -Parse the job name for upgrade type and environment: - -| Pattern in Job Name | Upgrade Type | Description | -|---------------------|-------------|-------------| -| `upgrade-from-stable-4.X` | Minor upgrade | Installs GA 4.X, upgrades to 4.(X+1) nightly/CI build | -| `upgrade-from-stable` (no version) | Minor upgrade | Installs previous GA minor, upgrades to current | -| `upgrade-micro` | Micro (z-stream) | Installs previous z-stream of same minor, upgrades to current build | -| `upgrade` (bare, no `from-stable`) | Micro (z-stream) | Usually same-minor upgrade within the version stream | -| `upgrade-from-stable-4.X-to-4.Y` | Multi-minor | Installs 4.X GA, upgrades through 4.Y (may skip versions) | - -### Minor vs Micro vs Major Upgrades - -**Micro (z-stream) upgrades** (e.g., 4.18.3 → 4.18.5): -- Components within the same minor version — API changes are rare -- Machine config changes are typically minimal (no OS-level changes) -- MCO may skip node drain/reboot if no machine config changes exist -- Fastest upgrade type, fewest moving parts -- Failures here often indicate regressions in specific operators - -**Minor upgrades** (e.g., 4.17 → 4.18): -- All operators update to new versions with potentially new APIs -- Machine config changes are common — new kubelet config, OS updates, crio config -- MCO will drain and reboot every node (control plane and workers) -- Takes significantly longer than micro upgrades (30-60+ minutes) -- Version skew between components is expected during the transition -- Most complex upgrade path — highest failure rate - -**Major upgrades** (e.g., 3.x → 4.x): -- Not tested in current CI. If encountered, treat as a special case and investigate manually. - -**EUS-to-EUS upgrades** (e.g., 4.14 → 4.16 via 4.15): the worker MachineConfigPool is -deliberately **paused** (`oc patch mcp/worker --type merge -p '{"spec":{"paused":true}}'`) while -the control plane makes two minor hops, then unpaused once so workers drain/reboot a single time. -A worker MCP stuck `Paused=True` mid-EUS is **expected** — confirm the job is EUS before -diagnosing it as a drain stall. - -### Single-Node (SNO) Upgrade Jobs - -Jobs with `single-node` or `sno` in the name: - -- One node serves as both control plane and worker — no rolling update possible; it must be drained and rebooted in place -- All workloads disrupted during the reboot window -- etcd runs as a single member — no quorum concerns but no redundancy -- MCO drain evicts ALL pods simultaneously; every PDB will be violated (universal and expected) -- Upgrade timeout is typically shorter (one node to update) -- Recovery depends on the single node coming back healthy - -### HyperShift Upgrade Jobs - -HyperShift jobs have two distinct upgrade targets: - -**Management plane upgrade** — HyperShift operator and control plane components on the management cluster: -- HostedControlPlane pods (kube-apiserver, etcd, kas, etc.) -- HyperShift operator itself -- Control plane infrastructure in the management cluster's namespace - -**Hosted cluster upgrade** — the hosted cluster's data plane: -- Worker nodes get new machine configs -- Hosted cluster operators reconverge -- NodePool rolling update - -Key differences from standalone upgrades: -- Control plane runs as pods in the management cluster — no node drain for control plane -- etcd runs as a StatefulSet, not static pods -- Worker node updates are driven by NodePool, not MachineConfigPool -- Two separate ClusterVersion objects exist (management and hosted) — failures can originate in either; always check both - -See [hypershift reference](hypershift.md) for HyperShift-specific diagnostics. - -## Upgrade Phases and How to Identify Which Failed - -Determining which phase failed is the critical first step — it determines the entire -investigation path. - -### Phase 1: Pre-Upgrade Installation - -The cluster is installed with the base version before upgrade begins. - -**How to identify**: JUnit XML contains `install should succeed` — if this test fails, -the job never reached the upgrade phase. - -**Routing**: This is an **install failure**, not an upgrade failure. Use the install guide -that matches the job type: - -- [install failure reference](install/general.md) for cloud jobs -- [metal install reference](install/metal.md) for bare-metal jobs (job name contains `metal`) - -### Phase 2: Pre-Upgrade Health Checks - -Before initiating the upgrade, the test framework verifies: -- All ClusterOperators are Available, not Degraded, not Progressing -- All nodes are Ready -- All MachineConfigPools are updated and not degraded - -**How to identify**: Build log shows `Waiting for cluster operators to stabilize` -or `cluster is not healthy` before any `ClusterVersion` update. - -**Common causes**: Flaky install that left an operator degraded, node that didn't -fully join, or a preexisting condition from the base version. - -### Phase 3: Upgrade Initiation - -The test framework patches the ClusterVersion to set the desired update: - -```bash -oc adm upgrade --to-image= -``` - -**How to identify**: Build log shows `Starting upgrade to ` or -`Upgrading cluster to `. The ClusterVersion object transitions to -`Progressing=True`. - -**Common causes**: Invalid payload image, precondition failures (e.g., admin ack -required for minor upgrades), ClusterVersion webhook rejections. - -### Phase 4: Control Plane Rollout - -The CVO drives operator updates in dependency order. Control plane components first: - -1. **etcd** — Rolling restart of etcd members (one at a time, maintaining quorum) -2. **kube-apiserver** — Rolling restart with new binary and config -3. **kube-controller-manager** — Depends on kube-apiserver -4. **kube-scheduler** — Depends on kube-apiserver -5. **openshift-apiserver** — Depends on kube-apiserver -6. **openshift-controller-manager** — Depends on openshift-apiserver -7. **Other operators** — Authentication, console, monitoring, etc. - -**How to identify**: Timeline/interval data shows `ClusterOperator` source events -with `condition/Progressing changed: True` for control plane operators. Build log -shows CVO applying manifests. - -**Common causes**: -- etcd member fails to restart (disk I/O, corrupt WAL) -- kube-apiserver rollout stuck (cert rotation failure, webhook blocking) -- Operator manifest incompatible with new version -- Resource exhaustion on control plane nodes during rolling restart - -### Phase 5: MCO Node Updates (Drain, Apply, Reboot) - -After control plane operators converge, the MCO updates nodes: - -1. **Cordon** the node (mark unschedulable) -2. **Drain** all pods from the node (respecting PDBs) -3. **Apply** new machine config (kubelet, crio, OS config) -4. **Reboot** the node with new config -5. **Uncordon** the node (mark schedulable) -6. Repeat for next node (one at a time per pool) - -Control plane nodes update first (master MachineConfigPool), then workers. - -**How to identify**: Timeline data shows `MachineConfigPool` events with -`condition/Updating changed: True`. Build log shows MCO drain operations. -Node events show `Draining`, `Rebooting`, `NodeNotReady` transitions. - -**Common causes**: See the dedicated [MCO Drain Failures](#mco-drain-failures) -section below. - -**The upgrade also swaps the OS** (kernel, cri-o, systemd, NetworkManager, SELinux). -If a node misbehaves after its upgrade reboot, weigh RPM package changes between the -two releases alongside operator explanations: node journals record each boot's -package versions, and the release changelog lists the RHCOS diff. If the implicated -subsystem's packages changed across the boundary, read -[operating-system-changes.md](operating-system-changes.md). - -### Phase 6: Operator Reconvergence - -After all nodes are updated, operators stabilize on the new version: -- All ClusterOperators should reach `Available=True, Progressing=False, Degraded=False` -- CVO marks `Progressing=False` on ClusterVersion -- MachineConfigPools show `Updated=True, Updating=False` - -**How to identify**: ClusterVersion transitions from `Progressing=True` to -`Progressing=False`. If it stays `Progressing=True` with `Failing=True`, -an operator failed to reconverge. - -**Common causes**: -- Operator pod crash-looping on new version -- New operator version incompatible with existing custom resources -- Webhook or admission controller blocking operator reconciliation -- Storage or network driver not yet updated, blocking dependent operators - -### Phase 7: Post-Upgrade Conformance Testing - -After the upgrade completes, the test framework runs e2e conformance tests -against the upgraded cluster. - -**How to identify**: This is the second timeline file (sorted by filename). -Build log shows `Running e2e tests` or `openshift-tests run` after the upgrade -completes. JUnit XML shows individual test failures. - -**Common causes**: Tests fail due to post-upgrade cluster issues (operator still -degraded, nodes not fully ready, version-skew related API issues). - -### Determining Which Phase Failed from Artifacts - -Follow this decision tree: - -```text -1. Does JUnit XML contain "install should succeed" failure? - YES → Phase 1 (install failure) → use install reference - NO → continue - -2. Does build-log.txt show upgrade never started? - (No "Starting upgrade" or "Upgrading cluster" message) - YES → Phase 2 (pre-upgrade health check) → check operator health before upgrade - NO → continue - -3. Does build-log.txt show "Cluster did not complete upgrade" - or ClusterVersion stuck at Progressing=True? - YES → Phase 4, 5, or 6 — determine which: - - 3a. Are ClusterOperator events showing control plane operators - stuck at Progressing=True? - YES → Phase 4 (control plane rollout) - - 3b. Are MachineConfigPool events showing Updating=True stuck? - YES → Phase 5 (MCO node updates) - - 3c. Are all nodes updated but operators not stabilizing? - YES → Phase 6 (reconvergence) - - NO → continue - -4. Did the upgrade complete but conformance tests fail? - (ClusterVersion shows Progressing=False but tests fail) - YES → Phase 7 (post-upgrade conformance) -``` - -## MCO Drain Failures - -A failed or stuck drain halts the node-update sequence and, with it, the upgrade. - -### How MCO Drain Works - -To update a node's configuration, the MCO follows this sequence: - -```text -1. Cordon node (SchedulingDisabled) -2. Start draining the node (MCD retries eviction until it succeeds or the node update fails) - - For each pod on the node: - a. Check if pod is managed by a controller (DaemonSet pods are skipped) - b. Check PodDisruptionBudget — if eviction would violate PDB, wait and retry - c. Send eviction request to the API server - d. Wait for pod to terminate - - If drain doesn't complete within timeout → drain failure -3. Apply new MachineConfig (write files, update systemd units) -4. Reboot node -5. Wait for node to come back Ready -6. Uncordon node (SchedulingDisabled removed) -``` - -### PDB (PodDisruptionBudget) Conflicts - -PDBs protect workloads from voluntary disruption (like drains). A drain stalls if -evicting a pod would violate the budget. - -**How PDB blocking works**: -- A PDB specifies `minAvailable` or `maxUnavailable` for a set of pods -- During drain, the API server checks: "If I evict this pod, does the PDB still hold?" -- If eviction would violate the PDB, the request is rejected (HTTP 429) -- MCO retries the eviction, but if the pod can't be rescheduled elsewhere (node - affinity, resource constraints, or the only replica), the drain stalls - -**Common PDB culprits in CI**: - -| Component | PDB | Why It Blocks | -|-----------|-----|---------------| -| `prometheus-k8s` | `minAvailable: 1` | 2 replicas, both may be on the same node being drained | -| `alertmanager-main` | `minAvailable: 1` | Similar to prometheus — small replica count | -| `image-registry` | `minAvailable: 1` | Single replica in some CI configurations | -| `router-default` | `maxUnavailable: 25%` | With 2 replicas, can't evict one if other is down | -| E2E test pods | Various | Tests may create PDBs that aren't cleaned up | -| `openshift-monitoring/*` | Various | Monitoring stack PDBs during node drain | - -### Disruption Poller Pods That Block Their Own Node's Drain - -An insidious failure mode unique to CI. The test framework deploys **disruption -poller pods** on every node to monitor API backend availability, managed by a -DaemonSet-like controller: - -1. Poller pod runs on Node A -2. MCO tries to drain Node A -3. Poller pod is part of the drain target -4. If the poller pod has a PDB, or the framework fails to evict it before the drain, - the drain can stall -5. While the drain stalls, the poller pod reports disruption (node is cordoned, - services disrupted) -6. Feedback loop: drain blocked → disruption detected → drain still blocked - -**How to identify**: Look for drain timeout messages mentioning pods in -`openshift-e2e-*` namespace or pods with names containing `disruption` or `sampler`. - -### The Cascading Failure Pattern - -MCO drain failures trigger a cascade that explains most mass test failures in -upgrade jobs. Additionally the drained node is cordoned (workloads can't schedule -there); if it's a control plane node, etcd/apiserver run on fewer nodes; remaining -nodes bear extra load → potential resource exhaustion; and PDB violations propagate -(pod can't evict → other node picks up load → that node overloads → more failures). - -```text -MCO drain timeout on node X - → MachineConfigPool stuck at Updating=True - → CVO sees MCP not progressing - → ClusterVersion stays Progressing=True, eventually Failing=True - → machine-config operator reports Degraded=True - → Operators gated on node/MCP health report Degraded - → kube-apiserver may restart due to revision changes - → API disruption during restart - → Tests see API errors → mass test failures -``` - -**How to identify the cascade**: - -1. **Check MachineConfigPool status**: - ```bash - gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/machineconfigpool" \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - ``` - Look for pools with `UPDATED=False UPDATING=True DEGRADED=True` - -2. **Check machine-config-daemon logs**: - ```bash - gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-machine-config-operator/" - ``` - Search for `drain` and `evict` messages in the MCD pod logs - -3. **Check node events in interval data**: - Look for `NodeMonitor` and `MachineMonitor` source events showing the drain - and reboot sequence. A gap between `Draining` and `Rebooting` longer than - 30 minutes suggests a drain stall. - -4. **Check for PDB violation messages**: - Search the MCD logs and build log for: - ```text - error when evicting pods - Cannot evict pod as it would violate the pod's disruption budget - pod disruption budget - ``` - -### MCO Drain Artifacts to Examine - -| Artifact | Path | What to Look For | -|----------|------|------------------| -| MachineConfigPool status | `gather-extra/artifacts/oc_cmds/machineconfigpool` | Pool update status, degraded reason | -| Machine-config-daemon logs | `gather-extra/artifacts/pods/openshift-machine-config-operator/machine-config-daemon-*` | Drain progress, eviction errors | -| Node events | Timeline JSON `NodeMonitor` source | Drain start/end, reboot, NotReady transitions | -| MachineConfig status | `gather-extra/artifacts/oc_cmds/machineconfig` | Current vs desired config per pool | -| Node list | `gather-extra/artifacts/oc_cmds/nodes` | SchedulingDisabled status | -| PDB list | `gather-extra/artifacts/oc_cmds/poddisruptionbudgets` | PDB definitions and current disruptions allowed | - -## Operator Degradation Cascades - -Operators update in a dependency chain, so one failure often cascades to others. -Trace the chain to separate root cause from symptoms. - -### Common Degradation Chains - -**Authentication → API Server → Everything**: -```text -authentication operator degraded - → oauth-apiserver cannot serve tokens - → kube-apiserver health checks fail (webhook auth) - → openshift-apiserver reports degraded - → console, monitoring, registry lose API access - → mass operator degradation -``` - -**Machine-API → Machine-Config → Node Lifecycle**: -```text -machine-api-operator unable to reconcile machines - → new machines don't join the cluster - → machine-config-operator can't update nodes that don't exist - → MachineConfigPool stuck at Updating - → CVO timeout waiting for MCP -``` - -**etcd → kube-apiserver → Everything**: -```text -etcd member restart during upgrade - → temporary quorum loss (if unlucky timing) - → kube-apiserver loses etcd backend - → all API requests fail - → all operators report degraded (can't read/write to API) - → mass test failures -``` - -**Ingress → Route-dependent services**: -```text -ingress operator updating router pods - → router pods temporarily unavailable - → console route unreachable - → oauth route unreachable → authentication disrupted - → monitoring routes disrupted -``` - -### Reading ClusterOperator Conditions Timeline - -The interval/timeline data contains `ClusterOperator` events showing condition -transitions. To trace a cascade: - -1. **Find the first operator to become Degraded or Unavailable**: - ```bash - # In the timeline JSON, search for ClusterOperator events - jq '[.[] | select(.source == "ClusterOperator" and .level == "Warning")] | - sort_by(.from) | .[:20]' timeline.json - ``` - -2. **Check the time ordering**: The first operator to degrade is usually the root - cause. Later degradations are often cascading effects. - -3. **Look at the degradation reason**: The `humanMessage` field contains the - condition details: - ```json - { - "source": "ClusterOperator", - "locator": { "keys": { "name": "authentication" } }, - "message": { - "humanMessage": "condition/Degraded changed: False -> True (OAuthServerDeploymentDegraded: deployment/oauth-openshift: 1/2 pods are available)" - }, - "from": "2026-03-21T22:15:00Z" - } - ``` - -4. **Map the cascade**: List all operator degradation events in time order and - identify which operators degraded before vs after the suspected root cause. - -### Distinguishing Transient vs Persistent Degradation - -**Transient degradation (expected during upgrade)**: -- Operator Degraded or Unavailable for < 5 minutes during its own rollout -- Progressing=True while updating; brief Degraded=True while old pods terminate and new pods start -- Returns to Available=True, Degraded=False after its rollout completes - -**Persistent degradation (real problem)**: -- Operator stays Degraded for > 10 minutes after its rollout should have completed -- Never transitions back to Available=True -- Pods crash-looping on the new version -- Degraded condition message shows a specific error, not just "rolling update in progress" - -**How to tell the difference**: -```text -1. Check the ClusterOperator's Progressing condition: - - Progressing=True + Degraded=True → may be transient (still updating) - - Progressing=False + Degraded=True → persistent (update finished but broken) - -2. Check pod status for the operator: - - Pods in CrashLoopBackOff → persistent problem - - Pods in ContainerCreating → may be transient (image pull, startup) - - Multiple restarts in last 10 minutes → persistent problem - -3. Check the timeline for condition oscillation: - - Degraded True → False → True rapidly → flapping (persistent problem) - - Degraded True → False (stays) → transient (resolved) -``` - -## ClusterVersion Analysis - -The ClusterVersion (CV) object is the central resource for upgrade status; the CVO -watches it and drives the entire upgrade. - -### ClusterVersion Object Structure - -```yaml -apiVersion: config.openshift.io/v1 -kind: ClusterVersion -metadata: - name: version -spec: - channel: stable-4.18 - desiredUpdate: - image: quay.io/openshift-release-dev/ocp-release@sha256:... - version: 4.18.5 -status: - availableUpdates: [...] - conditions: - - type: Available - status: "True" - lastTransitionTime: "2026-03-21T20:00:00Z" - - type: Failing - status: "False" - - type: Progressing - status: "True" - message: "Working towards 4.18.5: 85% complete" - - type: RetrievedUpdates - status: "True" - desired: - image: quay.io/openshift-release-dev/ocp-release@sha256:... - version: 4.18.5 - history: - - completionTime: null # null = still in progress - image: quay.io/openshift-release-dev/ocp-release@sha256:... - startedTime: "2026-03-21T21:00:00Z" - state: Partial # Partial = in progress, Completed = done - version: 4.18.5 - - completionTime: "2026-03-21T20:30:00Z" - image: quay.io/openshift-release-dev/ocp-release@sha256:... - startedTime: "2026-03-21T20:00:00Z" - state: Completed - version: 4.17.12 - observedGeneration: 3 -``` - -### ClusterVersion Conditions - -| Condition | Status | Meaning | -|-----------|--------|---------| -| `Progressing=True` | Normal during upgrade | CVO is actively reconciling operators to new version | -| `Progressing=True` + `Failing=True` | Problem | CVO is trying to upgrade but encountering errors | -| `Progressing=False` + `Available=True` | Normal post-upgrade | Upgrade completed successfully | -| `Progressing=True` + stale message | Problem | CVO stuck at a particular percentage | - -### Progressing=True Stuck — Common Causes - -When ClusterVersion stays `Progressing=True` for too long (>60 minutes for minor upgrade): - -1. **Operator not converging**: The message field shows which operator is blocking: - ```text - "Working towards 4.18.5: 85% complete, waiting on kube-apiserver" - ``` - Check that operator's ClusterOperator conditions and pod status. - -2. **MachineConfigPool not completing**: MCPs must finish updating all nodes: - ```text - "Working towards 4.18.5: 95% complete, waiting on machine-config" - ``` - Check MCP status for drain failures or stuck nodes. - -3. **Precondition not met**: Some upgrades require admin acknowledgments: - ```text - "Precondition: admin-acks/ack-4.17-kube-1.31-api-removals-in-4.18 not found" - ``` - The CI test framework should handle this — if it doesn't, it's a test bug. - -4. **Manifest application failure**: CVO can't apply a particular manifest: - ```text - "Unable to apply ... : admission webhook denied the request" - ``` - A webhook is blocking a required change. - -### Failing=True During Upgrade - -`Failing=True` means the CVO encountered errors while reconciling. The `message` -field contains the specific error: - -```yaml -- type: Failing - status: "True" - message: | - ClusterOperator kube-apiserver is degraded: - StaticPodsDegraded: nodes/ip-10-0-0-5.ec2.internal pods - "kube-apiserver-ip-10-0-0-5.ec2.internal" is not ready -``` - -**Key**: `Failing=True` during upgrade does NOT always mean permanent failure. The -CVO retries, and transient failures may resolve. The upgrade is only considered -failed if `Failing=True` persists until the test framework timeout. - -### Reading ClusterVersion status.history - -The `status.history` array tracks all upgrade attempts: - -- **`state: Completed`** — upgrade finished successfully -- **`state: Partial`** — upgrade is in progress or failed before completing -- **`completionTime: null`** — upgrade hasn't finished (in progress or stalled) -- **`startedTime`** — when the upgrade began - -Check the history to determine: -- How long the upgrade has been running (`now - startedTime`) -- Whether previous upgrades completed (base version should show `Completed`) -- Whether the upgrade ever reached `Completed` or stayed at `Partial` - -### Partial Upgrade States - -A partial upgrade means some components are on the new version and some on the old. -This is the most dangerous state because: - -- API compatibility is not guaranteed between versions -- etcd may have been upgraded but apiserver hasn't (or vice versa) -- Some operators are running new code against old CRD schemas -- Workloads may see inconsistent behavior - -**Identifying partial upgrade state**: -```bash -# Check ClusterOperator versions -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/co" \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - -# In the output, check VERSION column — mixed versions indicate partial upgrade -``` - -## Version Skew During Upgrade - -During a minor upgrade, components update at different times, creating expected -version skew where some run the new version while others still run the old. - -### Expected N/N+1 Skew - -During a 4.17 → 4.18 upgrade: - -```text -Time T0: All components at 4.17 -Time T1: etcd updated to 4.18, apiserver still at 4.17 -Time T2: apiserver updated to 4.18, controllers still at 4.17 -Time T3: All control plane at 4.18, workers still at 4.17 -Time T4: Workers updating (MCO drain/reboot cycle) -Time T5: All components at 4.18 -``` - -Operators are designed to handle this skew, but bugs in skew handling are a common -source of upgrade failures. - -### Unexpected Stuck Skew - -If the upgrade stalls, you can end up with persistent skew where: -- Control plane is at 4.18 but workers are at 4.17 (MCO drain failure) -- Most operators are at 4.18 but one is stuck at 4.17 (operator bug) -- etcd is at 4.18 but kube-apiserver is still at 4.17 (rollout failure) - -**Detecting skew from must-gather**: - -Files under `must-gather/cluster-scoped-resources/` are YAML — parse them with `yq` (or -`yq -o=json … | jq`), never `jq` directly. The plain-text `gather-extra/artifacts/oc_cmds/` -dumps are often faster for a version scan. - -1. ClusterOperator versions (spot a CO lagging the rest): - ```bash - cat gather-extra/artifacts/oc_cmds/co # VERSION column, plain text - # From must-gather YAML instead: - yq '.items[] | {name: .metadata.name, versions: .status.versions}' \ - must-gather/cluster-scoped-resources/config.openshift.io/clusteroperators.yaml - ``` - -2. Node kubelet versions (mixed versions = workers mid-rollout): - ```bash - cat gather-extra/artifacts/oc_cmds/nodes # `oc get nodes -o wide`; VERSION = kubelet - # From must-gather YAML instead: - yq '.items[] | [.metadata.name, .status.nodeInfo.kubeletVersion] | @tsv' \ - must-gather/cluster-scoped-resources/core/nodes.yaml - ``` - -3. MachineConfigPool rendered config (compare `.spec.configuration.name` vs - `.status.configuration.name` — a mismatch means the pool has not finished updating): - ```bash - cat must-gather/cluster-scoped-resources/machineconfiguration.openshift.io/machineconfigpools/worker.yaml - ``` - -### API Compatibility Under Version Skew - -During minor upgrades, API version changes can cause issues: -- APIs deprecated in the new version may still be used by old-version components -- New CRD fields added by updated operators may not be understood by old-version controllers -- Webhook configurations from the new version may reject requests from old-version clients - -**Common skew-related failures**: -- `error: the server doesn't have a resource type "X"` — API removal during upgrade -- `unknown field "newField"` — CRD schema mismatch between versions -- Admission webhook timeouts — new webhook not yet available during rollout - -## Upgrade-Specific Artifact Patterns - -### Key Files to Examine - -| File / Path | What It Contains | When to Check | -|-------------|-----------------|---------------| -| `gather-extra/artifacts/oc_cmds/clusterversion` | ClusterVersion CR with status, conditions, history | Always — first artifact to check | -| `gather-extra/artifacts/oc_cmds/co` | All ClusterOperator statuses | Always — identify degraded operators | -| `gather-extra/artifacts/oc_cmds/machineconfigpool` | MCP status (Updated, Updating, Degraded) | MCO failures, node update issues | -| `gather-extra/artifacts/oc_cmds/machineconfig` | Machine configs and their rendered versions | MCP configuration mismatches | -| `gather-extra/artifacts/oc_cmds/nodes` | Node status, kubelet versions, conditions | Node readiness, version skew | -| `gather-extra/artifacts/oc_cmds/poddisruptionbudgets` | PDB status across all namespaces | Drain failures, PDB blocking | -| `gather-extra/artifacts/pods/openshift-machine-config-operator/` | MCO and MCD pod logs | Drain details, reboot timing | -| `gather-extra/artifacts/pods/openshift-cluster-version/` | CVO pod logs | Upgrade orchestration errors | -| Timeline JSON (`e2e-timelines_spyglass_*.json`) | Upgrade events, operator transitions, disruption | Temporal correlation of failures | -| `gather-extra/artifacts/pods/openshift-etcd/` | etcd pod logs | etcd health during upgrade | -| `gather-extra/artifacts/audit_logs/` | API server audit logs | Request failures during upgrade | - -### MCO Logs and Machine-Config-Daemon Logs - -Two key components: -- **machine-config-operator** pod: orchestrates MCP updates -- **machine-config-daemon** pods (one per node): perform the drain/apply/reboot - -Look in the MCD logs for: -```text -# Drain start -I ... msg="initiating drain of node ..." - -# PDB blocking -W ... msg="eviction request failed" error="Cannot evict pod ... it would violate the pod's disruption budget" - -# Drain not completing: MCD retries eviction (~1h) then fails the node update; the -# MCDDrainError alert fires (see disruption.md) -E ... msg="error when evicting pods ... (will retry after 5s)" - -# Config apply -I ... msg="applying machine config ..." - -# Reboot -I ... msg="initiating reboot" - -# Post-reboot -I ... msg="node has rebooted, applying config" -``` - -### Node Events During Upgrade - -The timeline JSON captures node lifecycle events: - -```json -{ - "source": "NodeMonitor", - "locator": { "keys": { "node": "ip-10-0-0-5.ec2.internal" } }, - "message": { "humanMessage": "condition/Ready changed: True -> False (NodeStatusUnknown: Kubelet stopped posting node status)" }, - "from": "2026-03-21T22:30:00Z" -} -``` - -Key `NodeMonitor` transitions to watch for during upgrade: -- `Ready=True → Ready=False` — node going down for reboot -- `Ready=False → Ready=True` — node back after reboot -- `SchedulingDisabled` annotation — node is cordoned for drain - -**Time between `Ready=False` and `Ready=True`**: This is the reboot duration. -Typically 5-10 minutes. If > 20 minutes, the node may have failed to come back. - -### Interval/Timeline Data: Upgrade Phase vs Conformance Phase - -Upgrade jobs produce **two timeline files** (sorted by filename): -1. **First file** = upgrade phase timeline -2. **Second file** = conformance/e2e test phase timeline - -The disruption parser (`parse_disruption.py`) attributes each disruption event to -a phase. Use the `phase_breakdown` output to separate upgrade-phase disruption -(expected within bounds) from conformance-phase disruption (unexpected — a -persistent problem). - -```bash -python3 plugins/ci/skills/analyze-disruption/parse_disruption.py \ - .work/prow-job-analysis/{build_id}/logs/e2e-timelines_spyglass_*.json \ - --window 60 --format text -``` - -The parser output includes phase attribution for each event: -- `phase: upgrade` — disruption during upgrade (expected within thresholds) -- `phase: conformance` — disruption after upgrade completed (unexpected) - -## The "Eventual Consistency" Pattern - -OpenShift upgrades are eventually consistent — errors and transient failures during -the upgrade are expected and most resolve on their own. Recognize this to avoid -chasing false positives. - -### Expected Transient Errors - -These errors during the first 30-60 minutes of an upgrade are **normal and expected**: - -| Error | Why It's Expected | When It's a Problem | -|-------|-------------------|---------------------| -| `connection refused` to API server | API server pod restarting during rollout | Persists > 10 min after rollout | -| Operator `Degraded=True` | Operator transitioning between versions | Persists > 15 min after operator pods are running | -| `etcd leader changed` | etcd member restart during rolling update | More than 3 leader changes, or happens during conformance | -| `node NotReady` | Node rebooting for MCO update | Node doesn't return to Ready within 20 min | -| `pod eviction` events | Node drain during MCO update | Evictions happening outside drain windows | -| `503 Service Unavailable` | Endpoints updating during pod rollout | Persists after all pods are running | -| `certificate not yet valid` | Cert rotation during upgrade | Persists > 5 min | -| `webhook timeout` | Webhook pod restarting | Persists > 5 min after pod is running | - -### Time-Window Analysis - -Correlate errors with the upgrade timeline: - -```text -Upgrade started: T0 -Control plane update: T0 to T0+20min -MCO node drain/reboot: T0+20min to T0+60min -Operator reconvergence: T0+60min to T0+70min -Upgrade completed: T0+70min (approximate) -Conformance tests start: T0+70min -Conformance tests end: T0+140min (approximate) - -Errors in [T0, T0+70min] → Likely transient upgrade noise -Errors in [T0+70min, T0+140min] → Likely real post-upgrade issues -Errors persisting across both windows → Real problem -``` - -**Key rule**: If an error appears during the upgrade window but **resolves by -the time conformance tests start**, it was transient. If it persists into -conformance testing, it's a real issue. - -### Why Some Test Failures During Upgrade Are Expected - -The CI framework runs disruption monitoring during the upgrade phase. Some -disruption is expected: -- API backends briefly stop responding during server restarts (typically < 30s) -- Node-local connections fail when nodes reboot (MCO update) -- In-flight requests fail during pod rollover (zero-downtime rollout) - -Each backend has a **disruption threshold** — an allowed amount of disruption -during upgrade. Only disruption exceeding the threshold is a failure: - -```text -# Example thresholds (from openshift/origin test framework) -# kube-api new connections: 10s allowed during upgrade -# openshift-api reused connections: 5s allowed during upgrade -# image-registry: 30s allowed during upgrade (longer because of image pulls) -``` - -Disruption within thresholds → test passes (expected upgrade behavior) -Disruption exceeding thresholds → test fails (regression) - -## Common Upgrade Failure Categories - -### 1. Infrastructure Failures During Upgrade - -Cloud provider issues that manifest during upgrade. - -**Symptoms**: -- Node fails to reboot (instance terminated by cloud provider) -- New node fails to join after reboot (cloud networking issue) -- etcd disk I/O spike during upgrade (cloud disk performance) -- API throttling during upgrade (too many cloud API calls from MCO) - -**Diagnosis**: -- Check `CloudMetrics` events in timeline data -- Check cloud provider status page for region issues -- Look for `Machine` object status changes in must-gather -- Check for cloud-specific error messages in MCO logs - -**Key artifacts**: -```bash -# Machine objects -gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/oc_cmds/machines" \ - .work/prow-job-analysis/{build_id}/logs/ --no-user-output-enabled - -# Cloud controller manager logs -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-cloud-controller-manager*/" -``` - -### 2. MCO Drain/Reboot Failures - -See the dedicated [MCO Drain Failures](#mco-drain-failures) section above. - -**Quick identification**: -- MachineConfigPool shows `UPDATING=True DEGRADED=True` -- MCD logs show repeated `error when evicting pods ... (will retry after 5s)` and the `MCDDrainError` alert fires -- Build log shows `timeout waiting for MachineConfigPool` - -### 3. Operator Incompatibility with New Version - -An operator's new version crashes or can't reconcile. - -**Symptoms**: -- Operator pods in CrashLoopBackOff after update -- New version logs show panic, nil pointer, or schema errors -- ClusterOperator stays Degraded after its pods were updated - -**Diagnosis**: -- Check the operator's pod logs for the new version: - ```bash - gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-{operator-namespace}/" - ``` -- Look for Go panics, nil pointer dereferences, or `unknown field` errors -- Check if the operator's CRDs were updated before or after the operator pods -- Use `fetch-new-prs-in-payload` skill to identify what PRs changed the operator - -### 4. etcd Member Issues During Control Plane Rollout - -etcd is the most sensitive component during upgrade. - -**Symptoms**: -- `etcd leader changed` events in timeline data -- `slow fdatasync` and `apply took too long` in etcd logs -- kube-apiserver health checks failing (etcd backend unavailable) -- Quorum loss (2 of 3 members unavailable simultaneously) - -**Diagnosis**: -- Check etcd logs for member restart sequencing: - ```text - # Healthy: one member restarts at a time, quorum maintained - member A: stopped → started → healthy - (pause) - member B: stopped → started → healthy - (pause) - member C: stopped → started → healthy - - # Problematic: overlapping restarts - member A: stopped → started - member B: stopped ← overlap, quorum may be lost - ``` -- Check `EtcdDiskCommitDuration` and `EtcdDiskWalFsyncDuration` events -- Look for `EtcdLog` events with `"apply request took too long"` or `"waiting for ReadIndex response took too long"` - -**Key artifacts**: -```bash -# etcd pod logs -gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-etcd/" -``` - -### 5. Webhook/Admission Failures Blocking Upgrades - -Webhooks can block CVO manifest application. - -**Symptoms**: -- CVO logs show `admission webhook denied the request` -- ClusterVersion Failing=True with webhook-related message -- CVO stuck at a particular manifest in the upgrade graph - -**Diagnosis**: -- Check CVO logs for the specific webhook and manifest: - ```bash - gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-cluster-version/" - ``` -- Identify which webhook is blocking and why -- Check if the webhook pod itself is healthy - -**Common culprits**: -- OLM webhooks blocking CRD updates -- Policy webhooks (Gatekeeper, Kyverno) rejecting new resource versions -- Conversion webhooks for CRD version migration - -### 6. Custom Resource Migration Failures - -When CRD schemas change between versions, existing CRs may need migration. - -**Symptoms**: -- Operator logs show `unable to decode` or `unknown field` errors -- CRD conversion webhook failures -- Operator can't read existing CRs after CRD update - -**Diagnosis**: -- Check if CRD was updated before operator pods (schema mismatch) -- Check conversion webhook logs if the CRD has multiple stored versions -- Look for `StoredVersions` mismatches in CRD status - -### 7. Storage/CSI Driver Upgrade Issues - -CSI drivers updating during upgrade can cause storage disruption. - -**Symptoms**: -- PVCs stuck in `Pending` state after upgrade -- Pods can't mount volumes post-upgrade -- CSI driver pods not running on all nodes post-reboot - -**Diagnosis**: -- Check CSI driver pod status across nodes -- Verify CSI node plugin DaemonSet has pods on all nodes -- Check for `FailedAttachVolume` or `FailedMount` events -- Check the storage operator's ClusterOperator status - -### 8. Networking (OVN/SDN) Upgrade Issues - -Network plugin upgrades are particularly sensitive. - -**Symptoms**: -- Pod-to-pod connectivity loss after node reboot -- OVN-Kubernetes pods crash-looping on updated nodes -- Network policy not enforced on some nodes -- DNS resolution failures (CoreDNS pods disrupted) - -**Diagnosis**: -- Check OVN pod status on each node: - ```bash - gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-extra/artifacts/pods/openshift-ovn-kubernetes/" - ``` -- Look for `OVSVswitchdLog` events in timeline data -- Check node-to-node connectivity in disruption data (host-to-host backends) -- Verify CoreDNS pods are running and responding - -**OVN/OVS upgrade sequence**: -During node reboot, OVS and OVN restart with new versions. If the new OVN version -is incompatible with the old OVN on not-yet-updated nodes, networking can break for -that node. - -### 9. Certificate Rotation Failures - -Certificates may rotate during upgrade windows. - -**Symptoms**: -- `x509: certificate has expired` or `certificate is not yet valid` -- API server rejects connections with TLS errors -- Service-to-service communication fails with cert errors - -**Diagnosis**: -- Check kube-apiserver logs for certificate-related errors -- Check if cert-manager or service-ca operator is degraded -- Verify certificate validity windows overlap the upgrade period -- Look for `certificate not valid before` errors (clock skew or premature rotation) - -### 10. Timeout Failures (Upgrade Took Too Long) - -The test framework has a global timeout for the upgrade phase. - -**Symptoms**: -- Build log shows `timed out waiting for the condition` or `upgrade did not complete within` -- All the individual components may be fine, but the total time exceeded the limit -- ClusterVersion shows `Progressing=True` at the time of timeout - -**Diagnosis**: -- Check how far the upgrade progressed (percentage in ClusterVersion message) -- Identify the slowest component (the bottleneck): - ```text - Check ClusterOperator events in timeline data — which operator was the last - to reach Progressing=False? - ``` -- Check MCO node update times — if each node takes 20 minutes and there are 6 - nodes, that's 2 hours of MCO time alone -- Consider resource pressure: under-provisioned CI clusters take longer for - everything - -**Common timeout causes**: -- Slow MCO drain (PDB blocking) — 5 minutes per node adds up -- Slow image pulls (large payload images, rate limiting) -- Slow etcd compaction after upgrade -- Infrastructure bottleneck (cloud disk I/O limits) - -## Distinguishing Install Failures from Upgrade Failures - -Upgrade jobs install first — **an install-time failure is not an upgrade failure**. -Always determine which phase failed first. - -### How to Tell: Install Phase vs Upgrade Phase - -| Signal | Indicates | -|--------|-----------| -| JUnit: `install should succeed` failed | Install failure | -| JUnit: `Cluster upgrade should succeed` failed | Upgrade failure | -| Build log: no `Starting upgrade` message | Install failure (upgrade never started) | -| Build log: `Starting upgrade` present, then timeout | Upgrade failure | -| ClusterVersion history: only one entry (base version) at `Partial` | Install failure | -| ClusterVersion history: base at `Completed`, target at `Partial` | Upgrade failure | -| ClusterVersion history: base at `Completed`, target at `Completed` | Upgrade succeeded, conformance test failure | - -### When to Route to Install References - -If the failure is during installation (upgrade-specific analysis does not apply): -- [install failure reference](install/general.md) for general install analysis -- [metal install reference](install/metal.md) for bare metal jobs - -### When to Stay in Upgrade Analysis - -If install completed (ClusterVersion history shows base version at `Completed`) but -the upgrade or post-upgrade tests failed, use this document, and cross-reference: -- [disruption reference](disruption.md) for disruption during upgrade -- [resource exhaustion reference](resource-exhaustion.md) for resource issues - -## Upgrade Analysis Workflow — Putting It All Together - -1. **Classify the upgrade type** from the job name: minor (from-stable) vs micro, platform - (aws/gcp/azure/metal), network plugin (ovn/sdn), special config (fips/sno/techpreview). -2. **Determine which phase failed.** Download the top-level `build-log.txt` and grep for the - phase markers: `install should succeed` (install phase); `Starting upgrade` / - `Upgrading cluster` (upgrade initiated); `Cluster upgrade should succeed` / - `upgrade did not complete` (upgrade result). See also "Distinguishing Install Failures from - Upgrade Failures" above. -3. **Gather ClusterVersion, ClusterOperator, MachineConfigPool, and node status.** Download the - artifacts from [Key Files to Examine](#key-files-to-examine) — those paths cover every file - needed for this step and step 5. -4. **Analyze timeline/interval data.** Locate `e2e-timelines_spyglass_*.json` and run the - disruption parser for a phased view: - ```bash - python3 plugins/ci/skills/analyze-disruption/parse_disruption.py \ - .work/prow-job-analysis/{build_id}/logs/e2e-timelines_spyglass_*.json \ - --window 60 --format text - ``` -5. **Drill into the implicated component** (MCO, CVO, etcd, kube-apiserver) using the pod-log - paths in the same table. -6. **Identify the root cause**: which phase failed, which component is at the root (trace - degradation cascades back to the first failure), transient vs persistent (time-window - analysis), infrastructure vs product, and whether a specific PR is implicated - (`fetch-new-prs-in-payload`). -7. **Report**: upgrade type, failed phase, root cause with evidence, event timeline, affected - components, and recommendation (revert PR, file bug, or infrastructure issue). - -## Quick Reference: Log Patterns - -### CVO Log Patterns - -```text -# Upgrade started -"Cluster version operator started ... desiredUpdate=..." - -# Applying manifests -"Running sync for ... (attempt N)" -"Done syncing ... (N seconds)" - -# Manifest failure -"error running apply for ..." -"Unable to apply ...: ..." - -# Operator not available -"Cluster operator ... is not available" -"Cluster operator ... is degraded" - -# Upgrade complete -"Cluster version ... reached completion state" -``` - -### MCO/MCD Log Patterns - -```text -# Drain initiated -"initiating drain of node ..." - -# Drain progress -"drain is ongoing for node ..." -"evicted pod ... from node ..." - -# PDB blocking drain -"Cannot evict pod ... it would violate the pod's disruption budget" -"eviction request failed ... pod disruption budget" - -# Drain not completing (MCD retries eviction until the node update fails) -"error when evicting pods" -"will retry after" - -# Config apply -"applying machine config ... to node ..." -"applying OS update" - -# Reboot -"initiating reboot of node ..." -"node ... has rebooted" - -# Node ready -"node ... has become ready" -"node ... config matches desired" -``` - -### etcd Log Patterns During Upgrade - -```text -# Member restart (expected, one at a time) -"etcdserver: starting member ..." -"rafthttp: started HTTP pipelining with peer ..." - -# Leader change (expected during rolling update, one per member restart) -"raft.node: ... elected leader ..." - -# Disk pressure (potential problem) -"slow fdatasync ... took ..." -"apply request took too long ... took ..." - -# Quorum concern -"etcdserver: request timed out" -"etcdserver: leader changed" -"waiting for ReadIndex response took too long" - -# WAL sync issue -"wal: sync duration ... exceeded threshold ..." -``` - -### ClusterOperator Condition Patterns in Timeline - -Timeline `ClusterOperator` events carry these `humanMessage` patterns (full event structure is -shown under [Reading ClusterOperator Conditions Timeline](#reading-clusteroperator-conditions-timeline)): - -- `condition/Progressing changed: False -> True` — operator starting update (expected) -- `condition/Degraded changed: False -> True` — operator degraded (may be transient) -- `condition/Degraded changed: True -> False` — transient degradation resolved -- `condition/Progressing changed: True -> False` — operator update complete (expected) - -## Advanced: Identifying Suspect PRs After Upgrade Failure - -Once the root cause involves a specific operator or component, use the -`fetch-new-prs-in-payload` skill to identify what changed: - -1. **Get the payload tag** from prowjob.json or ClusterVersion -2. **List PRs new in this payload** compared to the previous accepted payload -3. **Filter to the affected component/repo** -4. **Review those PRs** for changes that could cause the failure - -This is especially powerful for micro-upgrade failures where the delta between -versions is small — you can often pinpoint the exact PR that introduced the -regression. diff --git a/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md b/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md new file mode 100644 index 000000000..ae5677f12 --- /dev/null +++ b/plugins/ci/skills/prow-job-analyze-install-failure/SKILL.md @@ -0,0 +1,711 @@ +--- +name: prow-job-analyze-install-failure +description: Analyze OpenShift installation failures in Prow CI jobs by examining installer logs, log bundles, and sosreports. Use when CI job fails "install should succeed" tests at bootstrap, cluster creation or other stages. +--- + +# Prow Job Analyze Install Failure + +This skill helps debug OpenShift installation failures in CI jobs by downloading and analyzing installer logs, log bundles, and sosreports from Google Cloud Storage. + +## When to Use This Skill + +Use this skill when: +- A CI job fails with "install should succeed" test failure +- You need to debug installation failures at specific stages (bootstrap, install-complete, etc.) +- You need to analyze installer logs and log bundles from failed CI jobs + +## Prerequisites + +Before starting, verify these prerequisites: + +1. **gcloud CLI Installation** + - Check if installed: `which gcloud` + - If not installed, provide instructions for the user's platform + - Installation guide: https://cloud.google.com/sdk/docs/install + +2. **gcloud Authentication (Optional)** + - The `test-platform-results` bucket is publicly accessible + - No authentication is required for read access + - Skip authentication checks + +## Input Format + +The user will provide: +1. **Prow job URL** - URL to the failed CI job + - Example: `https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.21-e2e-aws-ovn-techpreview/1983307151598161920` + - The URL should contain `test-platform-results/` + +## Understanding Job Types from Names + +Job names contain important clues about the test environment and what to look for: + +1. **Upgrade Jobs** (names containing "upgrade") + - These jobs perform a **fresh installation first**, then upgrade + - **Minor upgrade jobs**: Contain "upgrade-from-stable-4.X" in the name - upgrade from previous minor version (e.g., 4.21 job installs 4.20, then upgrades to 4.21) + - Example: `periodic-ci-openshift-release-master-ci-4.21-upgrade-from-stable-4.20-e2e-gcp-ovn-rt-upgrade` + - **Micro upgrade jobs**: Have "upgrade" in the name but NO "upgrade-from-stable" - upgrade within the same minor version (e.g., earlier 4.21 to newer 4.21) + - Example: `periodic-ci-openshift-release-master-nightly-4.21-e2e-aws-ovn-upgrade-fips` + - Example: `periodic-ci-openshift-release-master-ci-4.21-e2e-azure-ovn-upgrade` + - If installation fails, the upgrade never happens + - **Key point**: Installation failures in upgrade jobs are still installation failures, not upgrade failures + +2. **FIPS Jobs** (names containing "fips") + - FIPS mode enabled for cryptographic operations + - Pay special attention to errors related to: + - Cryptography libraries + - TLS/SSL handshakes + - Certificate validation + - Hash algorithms + +3. **IPv6 and Dualstack Jobs** (names containing "ipv6" or "dualstack") + - Using IPv6 or dual IPv4/IPv6 networking stack + - Most IPv6 jobs are **disconnected** (no internet access) + - Use a locally-hosted mirror registry for images + - Pay attention to: + - Network connectivity errors + - DNS resolution issues + - Mirror registry logs + - IPv6 address configuration + +4. **Metal Jobs with IPv6** (names containing "metal" and "ipv6") + - Disconnected environment with additional complexity + - The metal install failure skill will analyze squid proxy logs and disconnected environment configuration + +5. **Single-Node Jobs** (names containing "single-node") + - All control plane and compute workloads on one node + - More prone to resource exhaustion + - Pay attention to CPU, memory, and disk pressure + +6. **Platform-Specific Indicators** + - `aws`, `gcp`, `azure`: Cloud platform used + - `metal`, `baremetal`: Bare metal environment (uses specialized metal install failure skill) + - `ovn`: OVN-Kubernetes networking (standard) + +## Implementation Steps + +### Step 1: Parse and Validate URL + +1. **Extract bucket path** + - Find `test-platform-results/` in URL + - Extract everything after it as the GCS bucket relative path + - If not found, error: "URL must contain 'test-platform-results/'" + +2. **Extract build_id** + - Search for pattern `/(\d{10,})/` in the bucket path + - build_id must be at least 10 consecutive decimal digits + - Handle URLs with or without trailing slash + - If not found, error: "Could not find build ID (10+ digits) in URL" + +3. **Determine job type** + - Check if job name contains "metal" (case-insensitive) + - Metal jobs: Set `is_metal_job = true` + - Other jobs: Set `is_metal_job = false` + +4. **Construct GCS paths** + - Bucket: `test-platform-results` + - Base GCS path: `gs://test-platform-results/{bucket-path}/` + - Ensure path ends with `/` + +### Step 2: Create Working Directory + +1. **Create directory structure** + ```bash + mkdir -p .work/prow-job-analyze-install-failure/{build_id}/logs + mkdir -p .work/prow-job-analyze-install-failure/{build_id}/analysis + ``` + - Use `.work/prow-job-analyze-install-failure/` as the base directory (already in .gitignore) + - Use build_id as subdirectory name + - Create `logs/` subdirectory for all downloads + - Create `analysis/` subdirectory for analysis files + - Working directory: `.work/prow-job-analyze-install-failure/{build_id}/` + +### Step 3: Download prowjob.json and Identify Target + +Use the `fetch-prowjob-json` skill to fetch the prowjob.json for this job. See `plugins/ci/skills/fetch-prowjob-json/SKILL.md` for complete implementation details. + +1. **Fetch prowjob.json** using the Prow job URL (convert to gcsweb URL per the `fetch-prowjob-json` skill) +2. **Save locally** to `.work/prow-job-analyze-install-failure/{build_id}/logs/prowjob.json` +3. **Parse and validate** + - Search for pattern: `--target=([a-zA-Z0-9-]+)` in the ci-operator args + - If not found: + - Display: "This is not a ci-operator job. The prowjob cannot be analyzed by this skill." + - Explain: ci-operator jobs have a --target argument specifying the test target + - Exit skill +4. **Extract target name** + - Capture the target value (e.g., `e2e-aws-ovn-techpreview`) + - Store for constructing artifact paths + +### Step 3b: Confirm RHCOS Version + +Use `prow-job-artifact-search` to look for `nodes.json` in gather-extra: + +``` +artifacts/{target}/gather-extra/artifacts/nodes.json +``` + +If found, parse the JSON and extract `.status.nodeInfo.osImage` from each Node item — the version number after "Red Hat Enterprise Linux CoreOS" indicates the RHEL base: +- Version starts with `9.` → RHCOS 9 (e.g., `Red Hat Enterprise Linux CoreOS 9.8.20260613-0 (Plow)`) +- Version starts with `10.` → RHCOS 10 (e.g., `Red Hat Enterprise Linux CoreOS 10.2.20260521-0 (Coughlan)`) + +Report the confirmed RHCOS version(s) in the analysis output. If nodes show mixed osImage values, the cluster is heterogeneous (RHCOS 9 + 10). + +This file may not exist for early install failures (bootstrap, infrastructure) where the cluster never progressed far enough to have nodes or run gather-extra. If unavailable, skip — this step is informational, not blocking. + +### Step 4: Download JUnit XML to Identify Failure Stage + +**Note on install-status.txt**: You may see an `install-status.txt` file in the artifacts. This file contains only the installer's exit code (a single number). The junit_install.xml file translates this exit code into a human-readable failure mode, so always prefer junit_install.xml for determining the failure stage. + +1. **Find junit_install.xml** + - Use recursive listing to find the file anywhere in the job artifacts: + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep "junit_install.xml" + ``` + - The file location varies by job configuration - don't assume any specific path + +2. **Download junit_install.xml** + - Download from the discovered location: + ```bash + # Download from wherever it was found + gcloud storage cp {full-gcs-path-to-junit_install.xml} .work/prow-job-analyze-install-failure/{build_id}/logs/junit_install.xml --no-user-output-enabled + ``` + - If file not found, continue anyway (older jobs or early failures may not have this file) + +3. **Parse junit_install.xml to find failure stage** + - Look for failed test cases with pattern `install should succeed: ` + - Installation failure modes: + - **`cluster bootstrap`** - Early install failure where we failed to bootstrap the cluster. Bootstrap is typically an ephemeral VM that runs a temporary kube apiserver. Check bootkube logs in the bundle. + - **`infrastructure`** - Early failure before we're able to create all cloud resources. Often but not always due to cloud quota, rate limiting, or outages. Check installer log for cloud API errors. + - **`cluster creation`** - Usually means one or more operators was unable to stabilize. Check operator logs in gather-must-gather artifacts. + - **`configuration`** - Extremely rare failure mode where we failed to create the install-config.yaml for one reason or another. Check installer log for validation errors. + - **`cluster operator stability`** - Operators never stabilized (available=True, progressing=False, degraded=False). Check specific operator logs to determine why they didn't reach stable state. + - **`other`** - Unknown install failure, could be for one of the previously declared reasons, or an unknown one. Requires full log analysis. + - Extract the failure stage for targeted log analysis + - Use the failure mode to guide which logs to prioritize + +### Step 4b: Check for Known Symptom Labels + +The CI system may attach **symptom labels** to job runs — machine-detected patterns (e.g., "test failures during high CPU events") stored as JSON artifacts. These are **not root causes** but provide useful environmental context that may help explain failures when no other cause is found. + +1. **List the job_labels directory** + ```bash + gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/job_labels/" 2>/dev/null + ``` + - If the directory does not exist or returns an error, skip this step silently + +2. **Download any JSON symptom files** (exclude `label-summary.html`) + ```bash + gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/job_labels/*.json" \ + .work/prow-job-analyze-install-failure/{build_id}/logs/job_labels/ --no-user-output-enabled 2>/dev/null || true + ``` + +3. **Parse symptom labels** — each JSON file describes a detected symptom with a summary and explanation. Collect all symptom summaries for inclusion in the report. + +4. **Use symptoms as investigative context** — symptoms are environmental observations, NOT definitive causes. They should inform your investigation but you must still perform thorough root cause analysis. Include them in the "Known Symptoms Seen" section of the report. + +### Step 5: Locate and Download Installer Logs + +1. **List all artifacts to find installer logs** + - Installer logs follow the pattern `.openshift_install*.log` + - **IMPORTANT**: Exclude deprovision logs - they are from cluster teardown, not installation + - Use recursive listing to find all installer logs: + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep -E "\.openshift_install.*\.log$" | grep -v "deprovision" + ``` + - This will find installer logs regardless of which CI step created them + +2. **Download all installer logs found** + ```bash + # For each installer log found in the listing (excluding deprovision) + gcloud storage cp {full-gcs-path-to-installer-log} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + ``` + - Download all installer logs found that are NOT from deprovision steps + - If multiple installer logs exist, download all of them (they may be from different install phases) + - Deprovision logs are from cluster cleanup and not relevant for installation failures + +### Step 6: Locate and Download Log Bundle + +1. **List all artifacts to find log bundle** + - Log bundles are `.tar` files (NOT `.tar.gz`) starting with `log-bundle-` + - **IMPORTANT**: Prefer non-deprovision log bundles over deprovision ones + - Use recursive listing to find all log bundles: + ```bash + # Find all log bundles, preferring non-deprovision + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep -E "log-bundle.*\.tar$" + ``` + - This will find log bundles regardless of which CI step created them + +2. **Download log bundle** + ```bash + # If non-deprovision log bundles exist, download one of those (prefer most recent by timestamp) + # Otherwise, download deprovision log bundle if that's the only one available + gcloud storage cp {full-gcs-path-to-log-bundle} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + ``` + - Prefer log bundles NOT from deprovision steps (they capture the failure state during installation) + - Deprovision log bundles may also contain useful info if no other bundle exists + - If multiple log bundles exist, prefer the one from a non-deprovision step + - If no log bundle found, continue with installer log analysis only (early failures may not produce log bundles) + +3. **Extract log bundle** + ```bash + tar -xf .work/prow-job-analyze-install-failure/{build_id}/logs/log-bundle-{timestamp}.tar -C .work/prow-job-analyze-install-failure/{build_id}/logs/ + ``` + +### Step 7: Invoke Metal Install Failure Skill (Metal Jobs Only) + +**IMPORTANT: Only perform this step if `is_metal_job = true`** + +Metal IPI jobs use **dev-scripts** with **Metal3** and **Ironic** to install OpenShift on bare metal. These require specialized analysis. + +1. **Invoke the metal install failure skill** + - Use the Skill tool to invoke: `ci:prow-job-analyze-metal-install-failure` + - Pass the following information: + - Build ID: `{build_id}` + - Bucket path: `{bucket-path}` + - Target name: `{target}` + - Working directory already created: `.work/prow-job-analyze-install-failure/{build_id}/` + +2. **The metal skill will**: + - Download and analyze dev-scripts logs (setup process before OpenShift installation) + - Download and analyze libvirt console logs (VM/node boot sequence) + - Download and analyze optional artifacts (sosreport, squid logs) + - Determine if failure was in dev-scripts setup or cluster installation + - Generate metal-specific analysis report + +3. **Continue with standard analysis**: + - After metal skill completes, continue with Step 8 (Analyze Installer Logs) + - The metal skill provides additional context about dev-scripts and console logs + - Standard installer log analysis is still relevant for understanding cluster creation failures + +### Step 8: Analyze Installer Logs + +**CRITICAL: Understanding OpenShift's Eventual Consistency** + +OpenShift installations exhibit "eventual consistency" behavior, which means: +- Components may report errors while waiting for dependencies to become ready +- Example: Ingress operator may error waiting for networking, which errors waiting for other components +- These intermediate errors are **expected and normal** during installation +- Early errors in the log often resolve themselves and are NOT the root cause + +**Error Analysis Strategy**: +1. **Start with the NEWEST/FINAL errors** - Work backwards in time +2. Focus on errors that persisted until installation timeout +3. Track backwards from final errors to identify the dependency chain +4. Early errors are only relevant if they directly relate to the final failure state +5. Don't chase errors that occurred early and then disappeared - they likely resolved + +**Example**: If installation fails at 40 minutes with "kube-apiserver not available", an error at 5 minutes saying "ingress operator degraded" is likely irrelevant because it probably resolved. Focus on what was still broken when the timeout occurred. + +1. **Read installer log** + - The installer log is a sequential log file with timestamp, log level, and message + - Format: `time="YYYY-MM-DDTHH:MM:SSZ" level= msg=""` + +2. **Identify key failure indicators (WORK BACKWARDS FROM END)** + - **Start at the end of the log** - Look at final error/fatal messages + - **Error messages**: Lines with `level=error` or `level=fatal` near the end of the log + - **Last status messages**: The final "Still waiting for" or "Cluster operators X, Y, Z are not available" messages + - **Warning messages**: Lines with `level=warning` near the failure time that may indicate problems + - **Then work backwards** to find when the failing component first started having issues + - Ignore errors from early in the log unless they persist to the end + +3. **Extract relevant log sections (prioritize recent errors)** + - For bootstrap failures: + - Search for: "bootstrap", "bootkube", "kube-apiserver", "etcd" in the **last 20% of the log** + - For install-complete failures: + - Search for: "Cluster operators", "clusteroperator", "degraded", "available" in the **final messages** + - For timeout failures: + - Search for: "context deadline exceeded", "timeout", "timed out" + - Look at what component was being waited for when timeout occurred + +4. **Create installer log summary** + - Extract **final/last** error or fatal message (most important) + - Extract the last "Still waiting for..." message showing what didn't stabilize + - Extract surrounding context (10-20 lines before and after final errors) + - Optionally note early errors only if they relate to the final failure + - Save to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/installer-summary.txt` + +### Step 9: Analyze Log Bundle + +**Skip this step if no log bundle was downloaded** + +1. **Understand log bundle structure** + - `log-bundle-{timestamp}/` + - `bootstrap/journals/` - Journal logs from bootstrap node + - `bootkube.log` - Bootkube service that starts initial control plane + - `kubelet.log` - Kubelet service logs + - `crio.log` - Container runtime logs + - `journal.log.gz` - Complete system journal (gzipped) + - `bootstrap/network/` - Network configuration + - `ip-addr.txt` - IP addresses + - `ip-route.txt` - Routing table + - `hostname.txt` - Hostname + - `serial/` - Serial console logs from all nodes + - `{cluster-name}-bootstrap-serial.log` - Bootstrap node console + - `{cluster-name}-master-N-serial.log` - Master node consoles + - `clusterapi/` - Cluster API resources + - `*.yaml` - Kubernetes resource definitions + - `etcd.log` - etcd logs + - `kube-apiserver.log` - API server logs + - `failed-units.txt` - List of systemd units that failed + - `gather.log` - Log bundle collection process log + +2. **Analyze based on failure mode from junit_install.xml** + + **For "cluster bootstrap" failures:** + + Bootstrap failures are varied and complex. You MUST thoroughly examine the log bundle to build a complete timeline — do not guess the root cause from a single error. + + - Read `bootstrap/journals/bootkube.log` thoroughly. Identify every process that started, crashed, or errored, noting timestamps to build a chronological sequence. + - For any crashed process (non-zero exit status, ContainerDied), read its stderr/stdout in the surrounding lines. Exit codes tell you *that* it crashed; the error output tells you *why*. Treat a crash as a potential bug but validate the termination reason (OOM, host restart, killed by signal, resource limits, etc.) by examining container exit status, kernel messages, and host metrics before assigning root cause. Consult surrounding logs and infra signals — exit codes, ContainerDied event details, dmesg/journal entries, and resource utilization — to distinguish software defects from infra/resource-induced terminations. + - Pursue errors: read surrounding context, follow references to other components' logs, and trace the chain of causation back to the originating failure. The first error is often a symptom, not the cause. + - Check supporting logs: `clusterapi/kube-apiserver.log`, `clusterapi/etcd.log`, `bootstrap/journals/kubelet.log`. Cross-reference timestamps with bootkube.log. + - Check `serial/{cluster-name}-bootstrap-serial.log` for kernel panics, ignition failures, disk errors. + - Check `failed-units.txt` for failed systemd units. + + **For "infrastructure" failures:** + - Primary focus on installer log, not log bundle (failure happens before bootstrap) + - Search installer log for cloud provider API errors + - Look for quota exceeded messages (e.g., "QuotaExceeded", "LimitExceeded") + - Look for rate limiting errors (e.g., "RequestLimitExceeded", "Throttling") + - Check for authentication/permission errors + - **Infrastructure provisioning methods** (varies by OpenShift version): + - **Newer versions**: Use **Cluster API (CAPI)** to provision infrastructure + - Look for errors in ClusterAPI-related logs and resources + - Check for Machine/MachineSet/MachineDeployment errors + - Search for "clusterapi" or "machine-api" related errors + - **Older versions**: Use **Terraform** to provision infrastructure + - Look for "terraform" in log entries + - Check for terraform state errors or apply failures + - Search for terraform-related error messages + - Log bundle may not exist or be incomplete for this failure mode + + **For "cluster creation" failures:** + - Check if must-gather was successfully collected: + - Look for `must-gather*.tar` files in the gather-must-gather step directory + - If NO .tar file exists, must-gather collection failed (cluster was too unstable) + - Do NOT suggest downloading must-gather if the .tar file doesn't exist + - If must-gather exists, check for operator logs + - Look for degraded cluster operators + - Check operator-specific logs to see why they couldn't stabilize + - Review cluster operator status conditions + - This indicates cluster bootstrapped but operators failed to deploy + + **For "configuration" failures:** + - Focus entirely on installer log + - Look for install-config.yaml validation errors + - Check for missing required fields or invalid values + - This is a very early failure before any infrastructure is created + - Log bundle will not exist for this failure mode + + **For "cluster operator stability" failures:** + - Similar to "cluster creation" but operators are stuck in unstable state + - Check if must-gather was successfully collected (look for `must-gather*.tar` files) + - If must-gather doesn't exist, rely on installer log and log bundle only + - Check for operators with available=False, progressing=True, or degraded=True + - Review operator logs in gather-must-gather (if it exists) + - Check for resource conflicts or dependency issues + - Look at time-series of operator status changes + + **For "other" failures:** + - Perform comprehensive analysis of all available logs + - Check installer log for any errors or fatal messages + - Review log bundle if available + - Look for unusual patterns or timeout messages + +3. **Extract key information** + - If `failed-units.txt` exists, read it to find failed services + - For each failed service, find corresponding journal log + - Extract error messages from journal logs + - Save findings to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/log-bundle-summary.txt` + +### Step 10: Generate Analysis Report + +1. **Create comprehensive analysis report** + - Combine findings from all sources: + - Installer log analysis + - Log bundle analysis + - sosreport analysis (if applicable) + +2. **Report structure** + ``` + OpenShift Installation Failure Analysis + ======================================== + + Job: {job-name} + Build ID: {build_id} + Job Type: {metal/cloud} + Prow URL: {original-url} + + Failure Stage: {stage from junit_install.xml} + + Known Symptoms Seen (only if symptom labels found in Step 4b — omit if none) + --------------------- + - {symptom summary}: {symptom explanation} + Note: Symptoms are machine-detected environmental observations, not definitive + causes. They add context to help explain failures when correlated with other evidence. + + Summary + ------- + {High-level summary of the failure} + + Installer Log Analysis + ---------------------- + {Key findings from installer log} + + First Error: + {First error message with timestamp} + + Context: + {Surrounding log lines} + + Log Bundle Analysis + ------------------- + {Findings from log bundle} + + Failed Units: + {List from failed-units.txt} + + Key Journal Errors: + {Important errors from journal logs} + + Metal Installation Analysis (Metal Jobs Only) + ----------------------------------------- + {Summary from metal install failure skill} + - Dev-scripts setup status + - Console log findings + - Key metal-specific errors + + See detailed metal analysis: .work/prow-job-analyze-install-failure/{build_id}/analysis/metal-analysis.txt + + Recommended Next Steps + ---------------------- + {Actionable debugging steps based on failure mode: + + For "configuration" failures: + - Review install-config.yaml validation errors + - Check for missing required fields + - Verify credential format and availability + + For "infrastructure" failures: + - Check cloud provider quota and limits + - Review cloud provider service status for outages + - Verify API credentials and permissions + - Check for rate limiting in cloud API calls + + For "cluster bootstrap" failures: + - Review bootkube logs for control plane startup issues + - Check etcd cluster formation in etcd.log + - Examine kube-apiserver startup in kube-apiserver.log + - Review bootstrap VM serial console for boot issues + + For "cluster creation" failures: + - Identify which operators failed to deploy + - Check if must-gather was collected (look for must-gather*.tar files) + - If must-gather exists: Review specific operator logs in gather-must-gather + - If must-gather doesn't exist: Cluster was too unstable to collect diagnostics; rely on installer log and log bundle + - Check for resource conflicts or missing dependencies + + For "cluster operator stability" failures: + - Identify operators not reaching stable state + - Check operator conditions (available, progressing, degraded) + - Check if must-gather exists before suggesting to review it + - Review operator logs for stuck operations (if must-gather available) + - Look for time-series of operator status changes + + For "other" failures: + - Perform comprehensive log review + - Look for timeout or unusual error patterns + - Check all available artifacts systematically + } + + Artifacts Location + ------------------ + All artifacts downloaded to: + .work/prow-job-analyze-install-failure/{build_id}/logs/ + + - Installer logs: .openshift_install*.log + - Log bundle: log-bundle-*/ + - sosreport: sosreport-*/ (metal jobs only) + ``` + +3. **Save report** + - Save to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/report.txt` + +### Step 11: Present Results to User + +1. **Display summary** + - Show the analysis report to the user + - Highlight the most critical findings + - Provide file paths for further investigation + +2. **Offer next steps** + - Based on the failure type, suggest specific debugging actions: + - For bootstrap failures: Check API server and etcd logs + - For install-complete failures: Check cluster operator status + - For network issues: Review network configuration + - For metal job failures: Examine VM console logs + - **IMPORTANT**: Only suggest reviewing must-gather if you verified the .tar file exists + - Don't suggest downloading must-gather if no .tar file was found + - If must-gather doesn't exist, state that the cluster was too unstable to collect it + +3. **Provide artifact locations** + - List all downloaded files with their paths + - Note whether must-gather was successfully collected or not + - Explain how to explore the logs further + - Mention that artifacts are cached for faster re-analysis + +## Installation Stages Reference + +Understanding the installation stages helps target analysis: + +1. **Pre-installation** (Failure mode: "configuration") + - Validation of install-config.yaml + - Credential checks + - Image resolution + - **Common failures**: Invalid install-config.yaml, missing required fields, validation errors + +2. **Infrastructure Creation** (Failure mode: "infrastructure") + - Creating cloud resources (VMs, networks, storage) + - For metal: VM provisioning on hypervisor + - **Common failures**: Cloud quota exceeded, rate limiting, API outages, permission errors + +3. **Bootstrap** (Failure mode: "cluster bootstrap") + - Bootstrap node boots with temporary control plane + - Bootstrap API server and etcd start + - Bootstrap creates master nodes + - **Common failures**: API server won't start, etcd formation issues, bootkube errors + +4. **Master Node Bootstrap** + - Master nodes boot and join bootstrap etcd + - Masters form permanent control plane + - Bootstrap control plane transfers to masters + - **Common failures**: Masters can't reach bootstrap, network issues, ignition failures + +5. **Bootstrap Complete** + - Bootstrap node is no longer needed + - Masters are running permanent control plane + - Cluster operators begin initialization + - **Common failures**: Control plane not transferring, master nodes not ready + +6. **Cluster Operators Initialization** (Failure mode: "cluster creation") + - Core cluster operators start + - Operators begin deployment + - Initial operator stabilization + - **Common failures**: Operators can't deploy, resource conflicts, dependency issues + +7. **Cluster Operators Stabilization** (Failure mode: "cluster operator stability") + - Operators reach stable state (available=True, progressing=False, degraded=False) + - Worker nodes can join + - **Common failures**: Operators stuck progressing, degraded state, availability issues + +8. **Install Complete** + - All cluster operators are available and stable + - Cluster is fully functional + - Installation successful + +## Failure Mode Mapping + +| JUnit Failure Mode | Installation Stage | Where to Look | Artifacts Available | +|-------------------|-------------------|---------------|-------------------| +| `configuration` | Pre-installation | Installer log only | No log bundle | +| `infrastructure` | Infrastructure Creation | Installer log, cloud API errors | Partial or no log bundle | +| `cluster bootstrap` | Bootstrap | Log bundle (bootkube, etcd, kube-apiserver) | Full log bundle | +| `cluster creation` | Operators Initialization | gather-must-gather, operator logs | Full artifacts | +| `cluster operator stability` | Operators Stabilization | gather-must-gather, operator status | Full artifacts | +| `other` | Unknown | All available logs | Varies | + +## Key Files Reference + +### Installer Logs +- **Location**: `.openshift_install-{timestamp}.log` +- **Format**: Structured log with timestamp, level, message +- **Key patterns**: + - `level=error` - Error messages + - `level=fatal` - Fatal errors that stop installation + - `waiting for` - Timeout/waiting messages + +### Log Bundle Structure +- **bootstrap/journals/bootkube.log**: Bootstrap control plane initialization +- **bootstrap/journals/kubelet.log**: Bootstrap kubelet (container orchestration) +- **clusterapi/kube-apiserver.log**: API server logs +- **clusterapi/etcd.log**: etcd cluster logs +- **serial/*.log**: Node console output +- **failed-units.txt**: systemd services that failed + +### Metal Job Artifacts + +Metal installations are handled by the `prow-job-analyze-metal-install-failure` skill. +See that skill's documentation for details on dev-scripts, libvirt logs, sosreport, and squid logs. + +## Tips + +- **CRITICAL**: Work backwards from the end of the installer log, not forwards from the beginning +- Early errors often resolve themselves due to eventual consistency - focus on final errors +- Always start by checking the installer log for the **LAST** error, not the first +- The log bundle provides detailed node-level diagnostics +- Serial console logs show the actual boot sequence and can reveal kernel panics +- Failed systemd units in failed-units.txt are strong indicators of the problem +- Bootstrap failures are often etcd or API server related +- Install-complete failures are usually cluster operator issues +- Metal jobs use a specialized skill for analysis (prow-job-analyze-metal-install-failure) +- Cache artifacts in `.work/prow-job-analyze-install-failure/{build_id}/` for re-analysis +- Use grep with relevant keywords to filter large log files +- Timeline of events from installer log helps correlate issues across logs +- Pay attention to job name clues: fips, ipv6, dualstack, metal, single-node, upgrade +- IPv6 jobs are often disconnected and use mirror registries +- Only suggest must-gather if the .tar file exists; if not, cluster was too unstable +- **TechPreview jobs** (names containing "techpreview") enable additional feature gates not active in Default clusters. Bootstrap failures in TechPreview jobs may be in TechPreview-gated code paths (e.g., on-cluster layering, OS image management) that won't reproduce in Default clusters. Note this in your analysis when relevant. + +## Important Notes + +1. **Bootstrap Failures Require Log Bundle Analysis** + - Bootstrap timeout failures can have many causes — always download and examine the installer log bundle to determine the actual root cause + - Do not classify bootstrap failures without examining the logs + +2. **Eventual Consistency Behavior** + - OpenShift installations exhibit eventual consistency + - Components report errors while waiting for dependencies + - Early errors are EXPECTED and usually resolve automatically + - **Always analyze backwards from the final timeout, not forwards from the start** + - Only errors that persist until failure are relevant root causes + +3. **Upgrade Jobs and Installation** + - Jobs with "upgrade" in the name perform installation FIRST, then upgrade + - If you're analyzing an installation failure in an upgrade job, it never got to the upgrade phase + - "minor" upgrade: Installs 4.n-1 version (e.g., 4.20 for a 4.21 upgrade job) + - "micro" upgrade: Installs earlier payload in same stream + +4. **Log Bundle Availability** + - Not all jobs produce log bundles + - Older jobs may not have this feature + - Installation must reach a certain point to generate log bundle + +5. **Must-Gather Availability** + - Must-gather only exists if a `must-gather*.tar` file is present + - If no .tar file exists, the cluster was too unstable to collect diagnostics + - **Never suggest downloading must-gather unless you verified the .tar file exists** + +6. **Metal Job Specifics** + - Metal jobs are analyzed using the specialized `prow-job-analyze-metal-install-failure` skill + - That skill handles dev-scripts, libvirt console logs, sosreport, and squid logs + - See the metal skill documentation for details + +7. **Debugging Workflow** + - Start with installer log to find **LAST** error (not first) + - Use failure stage to guide which logs to examine + - Log bundle provides node-level details + - For metal jobs, invoke the metal-specific skill for additional analysis + - Work backwards in time to trace dependency chains + +8. **Common Failure Patterns** + - **Bootstrap etcd not starting**: Check etcd.log and bootkube.log + - **API server not responding**: Check kube-apiserver.log + - **Masters not joining**: Check master serial logs + - **Operators degraded**: Check specific operator logs in must-gather (if it exists) + - **Network issues**: Check network configuration in bootstrap/network/ + +9. **File Formats** + - Installer log: Plain text, structured format + - Journal logs: systemd journal format (plain text export) + - Serial logs: Raw console output + - YAML files: Kubernetes resource definitions + - Compressed files: .gz (gzip), .xz (xz) diff --git a/plugins/ci/skills/prow-job-analyze-metal-install-failure/SKILL.md b/plugins/ci/skills/prow-job-analyze-metal-install-failure/SKILL.md new file mode 100644 index 000000000..30f7065cb --- /dev/null +++ b/plugins/ci/skills/prow-job-analyze-metal-install-failure/SKILL.md @@ -0,0 +1,459 @@ +--- +name: prow-job-analyze-metal-install-failure +description: Analyze OpenShift bare metal installation failures in Prow CI jobs using dev-scripts artifacts. Use for jobs with "metal" in name, for debugging Metal3/Ironic provisioning, installation, or dev-scripts setup failures. You may also use the prow-job-analyze-install-failure skill with this one. +--- + +# Prow Job Analyze Metal Install Failure + +This skill helps debug OpenShift bare metal installation failures in CI jobs by analyzing dev-scripts logs, libvirt console logs, sosreports, and other metal-specific artifacts. + +## When to Use This Skill + +Use this skill when: +- A bare metal CI job fails with "install should succeed" test failure +- The job name contains "metal" or "baremetal" +- You need to debug Metal3/Ironic provisioning issues +- You need to analyze dev-scripts setup failures + +This skill is invoked by the main `prow-job-analyze-install-failure` skill when it detects a metal job. + +## Metal Installation Overview + +Metal IPI jobs use **dev-scripts** (https://github.com/openshift-metal3/dev-scripts) with **Metal3** and **Ironic** to install OpenShift: +- **dev-scripts**: Framework for setting up and installing OpenShift on bare metal +- **Metal3**: Kubernetes-native interface to Ironic +- **Ironic**: Bare metal provisioning service + +The installation process has multiple layers: +1. **dev-scripts setup**: Configures hypervisor, sets up Ironic/Metal3, builds installer +2. **Ironic provisioning**: Provisions bare metal nodes (or VMs acting as bare metal) +3. **OpenShift installation**: Standard installer runs on provisioned nodes + +Failures can occur at any layer, so analysis must check all of them. + +## Network Architecture (CRITICAL for Understanding IPv6/Disconnected Jobs) + +**IMPORTANT**: The term "disconnected" refers to the cluster nodes, NOT the hypervisor. + +### Hypervisor (dev-scripts host) +- **HAS** full internet access +- Downloads packages, container images, and dependencies from the public internet +- Runs dev-scripts Ansible playbooks that download tools (Go, installer, etc.) +- Hosts a local mirror registry to serve the cluster + +### Cluster VMs/Nodes +- Run in a **private IPv6-only network** (when IP_STACK=v6) +- **NO** direct internet access (truly disconnected) +- Pull container images from the hypervisor's local mirror registry +- Access to hypervisor services only (registry, DNS, etc.) + +### Common Misconception +When analyzing failures in "metal-ipi-ovn-ipv6" jobs: +- ❌ WRONG: "The hypervisor cannot access the internet, so downloads fail" +- ✅ CORRECT: "The hypervisor has internet access. If downloads fail, it's likely due to the remote service being unavailable, not network restrictions" + +### Implications for Failure Analysis +1. **Dev-scripts failures** (steps 01-05): If external downloads fail, check if the remote service/URL is down or has removed the resource +2. **Installation failures** (step 06+): If cluster nodes cannot pull images, check the local mirror registry on the hypervisor +3. **HTTP 403/404 errors during dev-scripts**: Usually means the resource was removed from the upstream source, not that the network is restricted + +## Prerequisites + +1. **gcloud CLI Installation** + - Check if installed: `which gcloud` + - If not installed, provide instructions for the user's platform + - Installation guide: https://cloud.google.com/sdk/docs/install + +2. **gcloud Authentication (Optional)** + - The `test-platform-results` bucket is publicly accessible + - No authentication is required for read access + +## Input Format + +The user will provide: +1. **Build ID** - Extracted by the main skill +2. **Bucket path** - Extracted by the main skill +3. **Target name** - Extracted by the main skill +4. **Working directory** - Already created by main skill + +## Metal-Specific Artifacts + +Metal jobs produce several diagnostic archives: + +### OFCIR Acquisition Logs +- **Location**: `{target}/ofcir-acquire/` +- **Purpose**: Shows the OFCIR host acquisition process +- **Contains**: + - `build-log.txt`: Log showing pool, provider, and host details + - `artifacts/junit_metal_setup.xml`: JUnit with test `[sig-metal] should get working host from infra provider` +- **Critical for**: Determining if the job failed to acquire a host before installation started +- **Key information**: + - Pool name (e.g., "cipool-ironic-cluster-el9", "cipool-ibmcloud") + - Provider (e.g., "ironic", "equinix", "aws", "ibmcloud") + - Host name and details + +### Dev-scripts Logs +- **Location**: `{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/` +- **Purpose**: Shows installation setup process and cluster installation +- **Contains**: Numbered log files showing each setup step (requirements, host config, Ironic setup, installer build, cluster creation). **Note**: dev-scripts invokes the installer, so installer logs (`.openshift_install*.log`) will also be present in the devscripts folders. +- **Critical for**: Early failures before cluster creation, Ironic/Metal3 setup issues, installation failures + +### libvirt-logs.tar +- **Location**: `{target}/baremetalds-devscripts-gather/artifacts/` +- **Purpose**: VM/node console logs showing boot sequence +- **Contains**: Console output from bootstrap and master VMs/nodes +- **Critical for**: Boot failures, Ignition errors, kernel panics, network configuration issues + +### log-bundle-*.tar (from gather or post-installation) +- **Location**: `{target}/baremetalds-devscripts-gather/artifacts/` +- **Purpose**: Cluster-level diagnostics including Ironic/Metal3 logs +- **Contains**: + - **Bootstrap Ironic logs**: Located at `bootstrap/journals/ironic.log` and `bootstrap/journals/metal3-baremetal-operator.log` + - Shows master node provisioning during bootstrap phase + - Contains Redfish/IPMI BMC communication for masters + - **Control-plane Ironic logs**: Located at `control-plane/{node-ip}/containers/metal3-ironic-*.log` and `control-plane/{node-ip}/containers/metal3-baremetal-operator-*.log` + - Shows worker node provisioning + - Bootstrap node journals (bootkube, kubelet, crio) + - Control plane container logs + - Cluster API resources +- **Critical for**: BareMetalHost registration failures, BMC connectivity issues (IPMI/Redfish), provisioning state problems, power management errors +- **Key Ironic errors to look for**: + - BMC (IPMI, Redfish) errors + - Node registration failures in Ironic + - Power state query failures + - Provisioning state transitions stuck +- **IMPORTANT**: + - Bootstrap Ironic logs only show master provisioning + - Control-plane Ironic logs show worker provisioning + - Always check control-plane logs when investigating worker issues + +### sosreport +- **Location**: `{target}/baremetalds-devscripts-gather/artifacts/` +- **Purpose**: Hypervisor system diagnostics +- **Contains**: Hypervisor logs, system configuration, diagnostic command output +- **Useful for**: Hypervisor-level issues, not typically needed for VM boot problems + +### squid-logs.tar +- **Location**: `{target}/baremetalds-devscripts-gather/artifacts/` +- **Purpose**: Squid proxy logs for inbound CI access to the cluster +- **Contains**: Logs showing CI system's inbound connections to the cluster under test. **Note**: The squid proxy runs on the hypervisor for INBOUND access (CI → cluster), NOT for outbound access (cluster → registry). +- **Critical for**: Debugging CI access issues to the cluster, particularly in IPv6/disconnected environments + +## Implementation Steps + +### Step 1: Check OFCIR Acquisition + +1. **Download OFCIR logs** + ```bash + gcloud storage cp gs://test-platform-results/{bucket-path}/artifacts/{target}/ofcir-acquire/build-log.txt .work/prow-job-analyze-install-failure/{build_id}/logs/ofcir-build-log.txt --no-user-output-enabled 2>&1 || echo "OFCIR build log not found" + gcloud storage cp gs://test-platform-results/{bucket-path}/artifacts/{target}/ofcir-acquire/artifacts/junit_metal_setup.xml .work/prow-job-analyze-install-failure/{build_id}/logs/junit_metal_setup.xml --no-user-output-enabled 2>&1 || echo "OFCIR JUnit not found" + ``` + +2. **Check junit_metal_setup.xml for acquisition failure** + - Read the JUnit file + - Look for test case: `[sig-metal] should get working host from infra provider` + - If the test failed, OFCIR failed to acquire a host + - This means installation never started - the failure is in host acquisition + +3. **Extract OFCIR details from build-log.txt** + - Parse the JSON in the build log to extract: + - `pool`: The OFCIR pool name + - `provider`: The infrastructure provider + - `name`: The host name allocated + - Save these for the final report + +4. **If OFCIR acquisition failed** + - Stop analysis - installation never started + - Report: "OFCIR host acquisition failed" + - Include pool and provider information + - Suggest: Check OFCIR pool availability and provider status + +### Step 2: Download Dev-Scripts Logs + +1. **Download dev-scripts logs directory** + ```bash + gcloud storage cp -r gs://test-platform-results/{bucket-path}/artifacts/{target}/baremetalds-devscripts-setup/artifacts/root/dev-scripts/logs/ .work/prow-job-analyze-install-failure/{build_id}/logs/devscripts/ --no-user-output-enabled + ``` + +2. **Handle missing dev-scripts logs** + - Some metal jobs may not have dev-scripts artifacts + - If missing, note this in the analysis and proceed with other artifacts + +### Step 2: Download libvirt Console Logs + +1. **Find and download libvirt-logs.tar** + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep "libvirt-logs\.tar$" + gcloud storage cp {full-gcs-path-to-libvirt-logs.tar} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + ``` + +2. **Extract libvirt logs** + ```bash + tar -xf .work/prow-job-analyze-install-failure/{build_id}/logs/libvirt-logs.tar -C .work/prow-job-analyze-install-failure/{build_id}/logs/ + ``` + +### Step 3: Download Optional Artifacts + +1. **Download sosreport (optional)** + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep "sosreport.*\.tar\.xz$" + gcloud storage cp {full-gcs-path-to-sosreport} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + tar -xf .work/prow-job-analyze-install-failure/{build_id}/logs/sosreport-{name}.tar.xz -C .work/prow-job-analyze-install-failure/{build_id}/logs/ + ``` + +2. **Download squid-logs (optional, for IPv6/disconnected jobs)** + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep "squid-logs.*\.tar$" + gcloud storage cp {full-gcs-path-to-squid-logs} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + tar -xf .work/prow-job-analyze-install-failure/{build_id}/logs/squid-logs-{name}.tar -C .work/prow-job-analyze-install-failure/{build_id}/logs/ + ``` + +### Step 4: Analyze Dev-Scripts Logs + +**Check dev-scripts logs FIRST** - they show what happened during setup and installation. + +1. **Read dev-scripts logs in order** + - Logs are numbered sequentially showing setup steps + - **Note**: dev-scripts invokes the installer, so you'll find `.openshift_install*.log` files in the devscripts directories + - Look for the first error or failure + +2. **Key errors to look for**: + - **Host configuration failures**: Networking, DNS, storage setup issues + - **Ironic/Metal3 setup issues**: BMC connectivity, provisioning network, node registration failures + - **Installer build failures**: Problems building the OpenShift installer binary + - **Install-config validation errors**: Invalid configuration before cluster creation + - **Installation failures**: Check installer logs (`.openshift_install*.log`) present in devscripts folders + +3. **Important distinction**: + - If failure is in dev-scripts setup logs (01-05), the problem is in the setup process + - If failure is in installer logs or 06_create_cluster, the problem is in the cluster installation (also analyzed by main skill) + +4. **Save dev-scripts analysis**: + - Save findings to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/devscripts-summary.txt` + +### Step 5: Analyze Ironic Logs (from log-bundle) + +**CRITICAL: Check the RIGHT Ironic logs based on what failed** + +The log bundle contains TWO sets of Ironic logs in different locations: +- **Bootstrap Ironic logs**: For master node provisioning +- **Control-plane Ironic logs**: For worker node provisioning + +**Which logs to check:** +- Masters failed to provision → Check `bootstrap/journals/ironic.log` +- Workers failed to provision → Check `control-plane/{ip}/containers/metal3-ironic-*.log` +- Unsure which failed → Check all + +1. **Download and extract log bundle** + ```bash + gcloud storage ls -r gs://test-platform-results/{bucket-path}/artifacts/ 2>&1 | grep "log-bundle.*\.tar$" + gcloud storage cp {full-gcs-path-to-log-bundle.tar} .work/prow-job-analyze-install-failure/{build_id}/logs/ --no-user-output-enabled + tar -xf .work/prow-job-analyze-install-failure/{build_id}/logs/log-bundle-*.tar -C .work/prow-job-analyze-install-failure/{build_id}/logs/ + ``` + +2. **Find ALL Ironic logs** + ```bash + # Bootstrap Ironic (master provisioning) + find .work/prow-job-analyze-install-failure/{build_id}/logs/ -path "*/bootstrap/journals/ironic.log" + find .work/prow-job-analyze-install-failure/{build_id}/logs/ -path "*/bootstrap/journals/metal3-baremetal-operator.log" + + # Control-plane Ironic (worker provisioning) - CRITICAL for worker failures + find .work/prow-job-analyze-install-failure/{build_id}/logs/ -path "*/control-plane/*/containers/metal3-ironic-*.log" + find .work/prow-job-analyze-install-failure/{build_id}/logs/ -path "*/control-plane/*/containers/metal3-baremetal-operator-*.log" + ``` + +3. **Analyze the Ironic logs**: + + **For Master Provisioning Issues** (check bootstrap logs): + - Location: `bootstrap/journals/ironic.log` and `bootstrap/journals/metal3-baremetal-operator.log` + - What to search: Master node UUIDs, master BareMetalHost names + + **For Worker Provisioning Issues** (check control-plane logs): + - Location: `control-plane/{node-ip}/containers/metal3-ironic-*.log` + - What to search: Worker node UUIDs, worker BareMetalHost names + +4. **Map node UUIDs to BareMetalHost names**: + - Ironic logs use node UUIDs (e.g., `b7fa5b83-91d0-46ee-acd2-e4b33e9ac983`) + - Find the corresponding BareMetalHost name from installer logs or must-gather + - This helps identify which specific worker or master failed + +5. **Save Ironic analysis**: + - Save findings to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/ironic-summary.txt` + - Include: + - Which Ironic logs were checked (bootstrap vs control-plane) + - Node UUIDs with errors + - Specific error messages (SSL, BMC connection, etc.) + - Whether masters or workers were affected + +### Step 6: Analyze libvirt Console Logs + +**Console logs are CRITICAL for metal failures during cluster creation.** + +1. **Find console logs** + ```bash + find .work/prow-job-analyze-install-failure/{build_id}/logs/ -name "*console*.log" + ``` + - Look for patterns like `{cluster-name}-bootstrap_console.log`, `{cluster-name}-master-{N}_console.log` + +2. **Analyze console logs for boot/provisioning issues**: + - **Kernel boot failures or panics**: Look for "panic", "kernel", "oops" + - **Ignition failures**: Look for "ignition", "config fetch failed", "Ignition failed" + - **Network configuration issues**: Look for "dhcp", "network unreachable", "DNS", "timeout" + - **Disk mounting failures**: Look for "mount", "disk", "filesystem" + - **Service startup failures**: Look for systemd errors, service failures + +3. **Console logs show the complete boot sequence**: + - As if you were watching a physical console + - Shows kernel messages, Ignition provisioning, CoreOS startup + - Critical for understanding what happened before the system was fully booted + +4. **Save console log analysis**: + - Save findings to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/console-summary.txt` + +### Step 7: Analyze sosreport + +**Only needed for hypervisor-level issues.** + +1. **Check sosreport for hypervisor diagnostics**: + - `var/log/messages` - Hypervisor system log + - `sos_commands/` - Output of diagnostic commands + - `etc/libvirt/` - Libvirt configuration + +2. **Look for hypervisor-level issues**: + - Libvirt errors + - Network configuration problems on hypervisor + - Resource constraints (CPU, memory, disk) + +### Step 8: Analyze squid-logs (If Downloaded) + +**Important for debugging CI access to the cluster.** + +1. **Check squid proxy logs**: + - Look for failed connections from CI to the cluster + - Look for HTTP errors or blocked requests + - Check patterns of CI test framework access issues + +2. **Common issues**: + - CI unable to connect to cluster API + - Proxy configuration errors blocking CI access + - Network routing issues between CI and cluster + - **Note**: These logs are for INBOUND access (CI → cluster), not for cluster's outbound access to registries + +### Step 9: Generate Metal-Specific Analysis Report + +1. **Create comprehensive metal analysis report**: + ``` + Metal Installation Failure Analysis + ==================================== + + Job: {job-name} + Build ID: {build_id} + Prow URL: {original-url} + + Installation Method: dev-scripts + Metal3 + Ironic + + OFCIR Host Acquisition + ---------------------- + Pool: {pool name from OFCIR build log} + Provider: {provider from OFCIR build log} + Host: {host name from OFCIR build log} + Status: {Success or Failure} + + {If OFCIR acquisition failed, note that installation never started} + + Dev-Scripts Analysis + -------------------- + {Summary of dev-scripts logs} + + Key Findings: + - {First error in dev-scripts setup} + - {Related errors} + + If dev-scripts failed: The problem is in the setup process (host config, Ironic, installer build) + If dev-scripts succeeded: The problem is in cluster installation (see main analysis) + + Console Logs Analysis + --------------------- + {Summary of VM/node console logs} + + Bootstrap Node: + - {Boot sequence status} + - {Ignition status} + - {Network configuration} + - {Key errors} + + Master Nodes: + - {Status for each master} + - {Key errors} + + Hypervisor Diagnostics (sosreport) + ----------------------------------- + {Summary of sosreport findings, if applicable} + + Proxy Logs (squid) + ------------------ + {Summary of proxy logs, if applicable} + Note: Squid logs show CI access to the cluster, not cluster's registry access + + Metal-Specific Recommended Steps + --------------------------------- + Based on the failure: + + For dev-scripts setup failures: + - Review host configuration (networking, DNS, storage) + - Check Ironic/Metal3 setup logs for BMC/provisioning issues + - Verify installer build completed successfully + - Check installer logs in devscripts folders + + For console boot failures: + - Check Ignition configuration and network connectivity + - Review kernel boot messages for hardware issues + - Verify network configuration (DHCP, DNS, routing) + + For CI access issues: + - Check squid proxy logs for failed CI connections to cluster + - Verify network routing between CI and cluster + - Check proxy configuration + + Artifacts Location + ------------------ + Dev-scripts logs: .work/prow-job-analyze-install-failure/{build_id}/logs/devscripts/ + Console logs: .work/prow-job-analyze-install-failure/{build_id}/logs/ + sosreport: .work/prow-job-analyze-install-failure/{build_id}/logs/sosreport-*/ + squid logs: .work/prow-job-analyze-install-failure/{build_id}/logs/squid-logs-*/ + ``` + +2. **Save report**: + - Save to: `.work/prow-job-analyze-install-failure/{build_id}/analysis/metal-analysis.txt` + +### Step 10: Return Metal Analysis to Main Skill + +1. **Provide summary to main skill**: + - Brief summary of metal-specific findings + - Indication of whether failure was in dev-scripts setup or cluster installation + - Key error messages and recommended actions + +## Common Metal Failure Patterns + +| Issue | Symptoms | Where to Look | +|-------|----------|---------------| +| **Dev-scripts host config** | Early failure before cluster creation | Dev-scripts logs (host configuration step) | +| **Ironic/Metal3 setup** | Provisioning failures, BMC errors | Dev-scripts logs (Ironic setup) | +| **BMC communication** | BareMetalHost stuck registering, power state failures | Ironic logs (in log-bundle), BareMetalHost status | +| **Node boot failure** | VMs/nodes won't boot | Console logs (kernel, boot sequence) | +| **Ignition failure** | Nodes boot but don't provision | Console logs (Ignition messages) | +| **Network config** | DHCP failures, DNS issues | Console logs (network messages), dev-scripts host config | +| **CI access issues** | Tests can't connect to cluster | squid logs (proxy logs for CI → cluster access) | +| **Hypervisor issues** | Resource constraints, libvirt errors | sosreport (system logs, libvirt config) | + +## Tips + +- **Check dev-scripts logs FIRST**: They show setup and installation (dev-scripts invokes the installer) +- **Installer logs in devscripts**: Look for `.openshift_install*.log` files in devscripts directories +- **Check Ironic logs for BMC issues**: BareMetalHost provisioning failures usually show detailed errors in Ironic logs +- **Console logs are critical**: They show the actual boot sequence like a physical console +- **Ironic/Metal3 setup errors** often appear in dev-scripts setup logs +- **BMC communication errors** appear in Ironic container logs in the log-bundle +- **Squid logs are for CI access**: They show inbound CI → cluster access, not outbound cluster → registry +- **Boot vs. provisioning**: Boot failures appear in console logs, provisioning failures in Ironic logs +- **Layer distinction**: Separate dev-scripts setup from Ironic provisioning from OpenShift installation diff --git a/plugins/ci/skills/prow-job-analyze-test-failure/README.md b/plugins/ci/skills/prow-job-analyze-test-failure/README.md new file mode 100644 index 000000000..db5b7aff7 --- /dev/null +++ b/plugins/ci/skills/prow-job-analyze-test-failure/README.md @@ -0,0 +1,13 @@ +## Using with Claude Code + +When you ask Claude to analyze a test failure in Prow job, it will automatically use this skill. The skill provides detailed instructions that guide Claude through: +- Validating prerequisites +- Parsing URLs +- Downloading artifacts +- Analyzing test failure +- Generating reports + +You can simply ask: +> "Analyze test failure XYZ in this Prow job: https://gcsweb-ci.../1978913325970362368/" + +Claude will execute the workflow and generate a text report diff --git a/plugins/ci/skills/prow-job-analyze-test-failure/SKILL.md b/plugins/ci/skills/prow-job-analyze-test-failure/SKILL.md new file mode 100644 index 000000000..a4f92bb1f --- /dev/null +++ b/plugins/ci/skills/prow-job-analyze-test-failure/SKILL.md @@ -0,0 +1,1289 @@ +--- +name: prow-job-analyze-test-failure +description: Analyze failed Prow CI tests by inspecting test code, downloading artifacts, and optionally integrating must-gather cluster diagnostics for root cause analysis +--- + +# Prow Job Analyze Test Failure + +This skill analyzes test failures by downloading Prow CI artifacts, checking test logs, inspecting resources and events, +analyzing test source code, and optionally integrating cluster diagnostics from must-gather data. + +## Prerequisites + +Identical with "prow-job-analyze-resource" skill. + +## Input Format + +The user will provide: + +1. **Prow job URL** - gcsweb URL containing `test-platform-results/` + + - Example: `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/pr-logs/pull/openshift_hypershift/6731/pull-ci-openshift-hypershift-main-e2e-aws/1962527613477982208` + - URL may or may not have trailing slash + +2. **Test name** (optional) - specific test name that failed + - When provided, focus the analysis on that test's stack trace and logs + - When omitted, analyze all failed CI steps by inspecting the JUnit XML, build logs, + and step artifacts to identify the root cause — this is the common case for + multi-step CI workflows (e.g. HyperShift, bare-metal OADP jobs) where the failure + is in a step rather than a named unit test + - Examples: + - `TestKarpenter/EnsureHostedCluster/ValidateMetricsAreExposed` + - `TestCreateClusterCustomConfig` + - `The openshift-console downloads pods [apigroup:console.openshift.io] should be scheduled on different nodes` + +3. Optional flags (optional): + - `--fast` - Skip must-gather extraction and analysis; continue with log/JUnit/step-artifact analysis only (scope is preserved: test-level when `test-name` is provided, step-level across all failed CI steps when omitted) + +## Implementation Steps + +### Step 1: Parse and Validate URL + +Use the "Parse and Validate URL" steps from "prow-job-analyze-resource" skill + +### Step 2: Create Working Directory + +1. **Check for existing artifacts first** + + - Check if `.work/prow-job-analyze-test-failure/{build_id}/logs/` directory exists and has content + - If it exists with content: + - Use AskUserQuestion tool to ask: + - Question: "Artifacts already exist for build {build_id}. Would you like to use the existing download or re-download?" + - Options: + - "Use existing" - Skip to step Analyze Test Failure + - "Re-download" - Continue to clean and re-download + - If user chooses "Re-download": + - Remove all existing content: `rm -rf .work/prow-job-analyze-test-failure/{build_id}/logs/` + - Also remove tmp directory: `rm -rf .work/prow-job-analyze-test-failure/{build_id}/tmp/` + - This ensures clean state before downloading new content + - If user chooses "Use existing": + - Skip directly to Step 4 (Analyze Test Failure) + - Still need to download prowjob.json if it doesn't exist + +2. **Create directory structure** + ```bash + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/logs + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/tmp + ``` + - Use `.work/prow-job-analyze-test-failure/` as the base directory (already in .gitignore) + - Use build_id as subdirectory name + - Create `logs/` subdirectory for all downloads + - Create `tmp/` subdirectory for temporary files (intermediate JSON, etc.) + - Working directory: `.work/prow-job-analyze-test-failure/{build_id}/` + +### Step 3: Download and Validate prowjob.json + +Use the `fetch-prowjob-json` skill to fetch the prowjob.json for this job. See `plugins/ci/skills/fetch-prowjob-json/SKILL.md` for complete implementation details. + +1. **Fetch prowjob.json** using the Prow job URL (convert to gcsweb URL per the `fetch-prowjob-json` skill) +2. **Save locally** to `.work/prow-job-analyze-test-failure/{build_id}/logs/prowjob.json` +3. **Parse and validate** + - Search for pattern: `--target=([a-zA-Z0-9-]+)` in the ci-operator args + - If not found: + - Display: "This is not a ci-operator job. The prowjob cannot be analyzed by this skill." + - Explain: ci-operator jobs have a --target argument specifying the test target + - Exit skill +4. **Extract target name and JOB_NAME** + - Capture the target value (e.g., `e2e-aws-ovn`) from the `--target=` arg + - Extract `JOB_NAME` from `.spec.job` in prowjob.json (the artifact directory key): + ```bash + JOB_NAME=$(jq -r '.spec.job' .work/prow-job-analyze-test-failure/{build_id}/logs/prowjob.json) + ``` + - Note: on PR jobs `{target}` and `{JOB_NAME}` often differ. All artifact-path lookups + (JUnit XML, step logs, step artifacts) must use `{JOB_NAME}`, not `{target}` + +### Step 3b: Confirm RHCOS Version + +Use `prow-job-artifact-search` to fetch `nodes.json` from the gather-extra artifacts: + +``` +artifacts/{target}/gather-extra/artifacts/nodes.json +``` + +Parse the JSON and extract `.status.nodeInfo.osImage` from each Node item. This confirms the actual RHCOS variant the cluster ran — the version number after "Red Hat Enterprise Linux CoreOS" indicates the RHEL base: +- Version starts with `9.` → RHCOS 9 (e.g., `Red Hat Enterprise Linux CoreOS 9.8.20260613-0 (Plow)`) +- Version starts with `10.` → RHCOS 10 (e.g., `Red Hat Enterprise Linux CoreOS 10.2.20260521-0 (Coughlan)`) + +If nodes show mixed osImage values, the cluster is heterogeneous (RHCOS 9 + 10). Report the confirmed RHCOS version(s) in the analysis output. If `nodes.json` is unavailable (e.g., gather-extra didn't run), skip this step — it's informational, not blocking. + +### Step 4: Analyze Test Failure + +### Step 4.0: Detect Aggregated Jobs + +Aggregated jobs run the same job in parallel (typically 10 times) and perform statistical +analysis of test results. Detect aggregation by checking for `aggregated-` prefix in the job +name or an `aggregator` container/step in prowjob.json. + +If the job is aggregated, the failure modes are different from normal jobs: + +1. **Statistically significant test failure** — The test itself fails frequently enough across + runs to be flagged as a regression. This is a real regression that needs investigation. + +2. **Insufficient completed runs** — Not enough runs completed successfully to perform the + statistical test (e.g., only 5 of 10 jobs produced results). This manifests as mass test + failures across many unrelated tests. The root cause is whatever prevented the other runs + from completing — this could be infrastructure issues, install failures, or an actual product + bug causing crashes. Investigate the underlying job runs that did not complete to determine + why. + +3. **Non-deterministic test presence** — A test only ran in a small subset of completed jobs, + even though the other jobs completed successfully. The failure message will say something like + "Passed 1 times, failed 0 times, skipped 0 times: we require at least 6 attempts to have a + chance at success". Every test must produce results in every job run; it is a bug if it does + not. This is a regression — someone introduced a test that doesn't produce results + deterministically. Investigate which test is non-deterministic and why it only runs in some + jobs. + +When analyzing an aggregated job failure, first determine which failure mode applies before +diving into individual test analysis. For mode 2, focus on why runs failed rather than +individual test results. For mode 3, investigate why the test only ran in some jobs. + +**Finding underlying job run URLs:** + +The aggregated junit XML contains links to every underlying job run. Download it from: +``` +gs://test-platform-results/{bucket-path}/artifacts/release-analysis-aggregator/openshift-release-analysis-aggregator/artifacts/release-analysis-aggregator/{job-name}/{payload-tag}/junit-aggregated.xml +``` + +Each `` element has a `` with YAML-formatted data including `passes:`, +`failures:`, and `skips:` lists. Each entry has: +- `jobrunid`: The build ID of the underlying job run +- `humanurl`: Prow URL for the job run (e.g., `https://prow.ci.openshift.org/view/gs/test-platform-results/logs/{job-name}/{jobrunid}`) +- `gcsartifacturl`: Direct link to GCS artifacts + +Use the `humanurl` links to investigate individual job run failures with the normal +(non-aggregated) analysis steps below. + +### Step 4.1: Download build-log.txt + +```bash +gcloud storage cp gs://test-platform-results/{bucket-path}/build-log.txt .work/prow-job-analyze-test-failure/{build_id}/logs/build-log.txt --no-user-output-enabled +``` + +### Step 4.2: Parse and validate + +- Read `.work/prow-job-analyze-test-failure/{build_id}/logs/build-log.txt` + +**Branch on whether {test_name} was provided:** + +#### Branch A: test_name provided +- Search build-log.txt for the exact test name string +- Gather the stack trace and surrounding context for that specific test +- Store the single failing test's error output for use in Steps 4.4 and 4.9 + +#### Branch B: test_name NOT provided (multi-step discovery) + +This is the common case for multi-step CI workflows (e.g., HyperShift, bare-metal, OADP jobs) +where failures occur in CI steps rather than named unit tests. + +1. **Discover failed CI steps from JUnit XML** + + Use `{JOB_NAME}` (from Step 3, extracted from `.spec.job`) for all GCS artifact paths — + not `{target}`. On PR jobs these values differ and `{target}` will miss the artifacts. + + Search for JUnit XML files under the artifacts directory: + ```bash + gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{JOB_NAME}/**/junit*.xml" 2>/dev/null + ``` + Download all found JUnit XML files: + ```bash + gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/{JOB_NAME}/**/junit*.xml" \ + .work/prow-job-analyze-test-failure/{build_id}/logs/ --no-user-output-enabled --recursive 2>/dev/null || true + ``` + Parse each downloaded XML file and collect every `` element where: + - A `` or `` child element is present, OR + - The `` has attribute `status="failed"` + + For each failed testcase record: + - `step_name`: value of `classname` or `name` attribute (whichever identifies the CI step) + - `failure_message`: text content of the `` or `` element + - `junit_file`: path of the XML file it came from + +2. **Classify each failed step by phase** + + The ci-operator JUnit XML (`junit_operator.xml`) includes phase-level testcases: + - `"Run multi-stage test pre phase"` — setup/installation steps + - `"Run multi-stage test test phase"` — functional test steps + - `"Run multi-stage test post phase"` — gather/cleanup steps + + Use the phase entries to classify each failed `step_name`: + + - Look at the failed **phase-level** testcases (those with `"pre phase"`, `"test phase"`, + or `"post phase"` in their name) — these tell you which phase failed + - Cross-reference the individual step testcases against their phase: + - Steps in the **`pre` phase** → installation/setup steps + - Steps in the **`test` phase** → functional test steps + - Steps in the **`post` phase** → gather/cleanup steps (rarely the root cause) + + If the phase-level testcases are absent from the JUnit XML (some jobs omit them), + fall back to the `ci-operator-step-graph.json` artifact, which lists all steps with + their dependencies and timing — use execution order and naming conventions as a + secondary signal. When still ambiguous, prefer classifying as a **test step** so + must-gather analysis is not skipped. + +3. **Route each failed step to the appropriate analysis path** + + - **`pre` phase steps (installation)** → invoke the `ci:analyze-prow-job-install-failure` + skill for each such step, passing the same Prow job URL. Must-gather is **not** + attempted for `pre` phase failures: must-gather requires a live apiserver, and when + the installation phase fails the cluster is not fully up, so collection will fail. + **Capture the delegated output** and store it + as a per-step entry in the shared results collection used by Step 5 and Step 5.5: + ``` + { + step_name: {step_name}, + type: "installation", + summary: {one-line summary from ci:analyze-prow-job-install-failure}, + evidence: {key error / stack trace from delegated analysis}, + recommendation: {recommended action from delegated analysis} + } + ``` + - **`test` phase steps (functional tests)** → proceed with the iteration in item 4 + below (download step log, artifacts, extract stack trace) and include must-gather + analysis (Steps 4.4–4.9). Store findings in the same per-step results collection: + ``` + { + step_name: {step_name}, + type: "test", + summary: {one-line failure summary}, + evidence: {stack trace / error output}, + recommendation: {recommended action} + } + ``` + - **`post` phase steps (gather/cleanup)** → treat as informational. Download the step + log and note the failure in the report, but do not perform must-gather analysis for + post-phase failures alone (they are usually a consequence of earlier failures). + +4. **Iterate over each `test` phase step** (`pre` and `post` phase steps already routed above) + + For each `step_name` in the **`test` phase**, perform the following sub-steps: + + a. **Download step-specific log** + ```bash + gcloud storage cp \ + "gs://test-platform-results/{bucket-path}/artifacts/{JOB_NAME}/{step_name}/build-log.txt" \ + .work/prow-job-analyze-test-failure/{build_id}/logs/{step_name}-build-log.txt \ + --no-user-output-enabled 2>/dev/null || true + ``` + If the step log is not found at that path, fall back to scanning build-log.txt for + lines mentioning `{step_name}` and collect surrounding context (±50 lines). + + b. **Download step-specific artifacts** + ```bash + gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/{JOB_NAME}/{step_name}/" \ + 2>/dev/null + ``` + Download any relevant artifacts (e.g., `*.json`, `*.yaml`, `events*.txt`) to + `.work/prow-job-analyze-test-failure/{build_id}/logs/{step_name}/`. + + c. **Extract stack trace and context** + - Scan the step log for panic traces, `FAIL`, `Error:`, `fatal`, or assertion failures + - Collect the full stack trace block (from the triggering line to the end of the trace) + - Note the failure message from the JUnit XML as additional context + + d. **Store per-step findings** + Record for each step: + - `step_name` + - `failure_message` (from JUnit XML) + - `stack_trace` (from step log or build-log.txt context) + - `artifacts` (list of downloaded artifact paths) + +3. **Produce a multi-step failure summary** + + After iterating all failed steps, produce an ordered list of findings: + ``` + Failed Steps Discovered: + 1. {step_name_1}: {one-line failure summary} + 2. {step_name_2}: {one-line failure summary} + ... + ``` + This list is used in Step 4.4 (evidence gathering) and Step 5 (report) to drive + per-step analysis rather than singular test analysis. + +### Step 4.3: Examine intervals files for cluster activity during E2E failures + +- Search recursively for E2E timeline artifacts (known as "interval files") within the bucket-path: + ```bash + gcloud storage ls 'gs://test-platform-results/{bucket-path}/**/e2e-timelines_spyglass_*json' + ``` +- The files can be nested at unpredictable levels below the bucket-path +- There could be as many as two matching files +- Download all matching interval files (use the full paths from the search results): + ```bash + gcloud storage cp gs://test-platform-results/{bucket-path}/**/e2e-timelines_spyglass_*.json .work/prow-job-analyze-test-failure/{build_id}/logs/ --no-user-output-enabled + ``` +- If the wildcard copy doesn't work, copy each file individually using the full paths from the search results +- **Scan interval files for test failure timing:** + - Look for intervals where `source = "E2ETest"` and `message.annotations.status = "Failed"` + - Note the `from` and `to` timestamps on this interval - this indicates when the test was running +- **Scan interval files for related cluster events:** + - Look for intervals that overlap the timeframe when the failed test was running + - Filter for intervals with: + - `level = "Error"` or `level = "Warning"` + - `source = "OperatorState"` + - These events may indicate cluster issues that caused or contributed to the test failure + +### Step 4.3b: Check for Known Symptom Labels + +The CI system may attach **symptom labels** to job runs — machine-detected patterns (e.g., "test failures during high CPU events") stored as JSON artifacts. These are **not root causes** but provide useful environmental context that may help explain failures when no other cause is found. + +1. **List the job_labels directory** + ```bash + gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/job_labels/" 2>/dev/null + ``` + - If the directory does not exist or returns an error, skip this step silently + +2. **Download any JSON symptom files** (exclude `label-summary.html`) + ```bash + gcloud storage cp "gs://test-platform-results/{bucket-path}/artifacts/job_labels/*.json" \ + .work/prow-job-analyze-test-failure/{build_id}/logs/job_labels/ --no-user-output-enabled 2>/dev/null || true + ``` + +3. **Parse symptom labels** — each JSON file describes a detected symptom with a summary and explanation. Collect all symptom summaries for inclusion in the report. + +4. **Use symptoms as investigative context** — symptoms are environmental observations, NOT definitive causes. They should inform your investigation (e.g., if 2 tests failed while high CPU was measured, CPU pressure could explain the failures if no other cause is found) but you must still perform thorough root cause analysis. Include them in the "Known Symptoms Seen" section of the report. + +### Step 4.4: Gather initial evidence + +- Analyze stack traces from build-log.txt +- Analyze related code in the code repository +- Store artifacts from Prow CI job (json/yaml files) related to the failure under `.work/prow-job-analyze-test-failure/{build_id}/tmp` +- Store logs under `.work/prow-job-analyze-test-failure/{build_id}/logs/` +- Collect evidence from logs and events and other json/yaml files +- **Check for CI step script errors**: If the error is a scripting issue in a CI step (e.g., unbound variable, syntax error, missing command, bad exit code from a shell script) rather than a product bug, check for recent commits to that step's script in the `openshift/release` repository. This is part of evidence gathering — identifying the responsible PR early informs the rest of the analysis. Include the responsible PR in your evidence. + +### Step 4.4b: Investigate crash-looping or failing containers + +**Be tenacious.** When the build log or test output mentions crash-looping pods, container restarts, or deployments not becoming ready, you MUST trace the failure to its root cause. Never stop at "containers are crash-looping" — find out *why*. + +**Pursue all available log sources:** + +- **Must-gather data** (Step 4.6–4.7): Contains pod YAMLs with `containerStatuses` (`exitCode`, `lastState.terminated.reason`, `restartCount`), container logs (current and previous), events, and operator conditions. This is often the richest source for diagnosing crash-looping pods. +- **Gather-extra / gather-audit logs**: Some jobs run additional gather steps that collect extra diagnostics. List the step artifacts directory for gather steps beyond `gather-must-gather` (e.g., `gather-extra`, `gather-audit-logs`) and download relevant logs. +- **Step-level build logs**: The build log for the failing test step (downloaded in Step 4.2) often contains error output, stack traces, and timeout messages that reference specific pods or containers. +- **Events from interval files** (Step 4.3): Operator state transitions and warning events correlated with the failure window. + +**Follow the dependency chain.** Always trace upstream to the originating error rather than stopping at the first symptom. + +### Step 4.5: Check for Must-Gather Availability + +1. **Parse optional flags** + - Parse user input for `--fast` flag + - If `--fast` flag present: + - Skip must-gather detection and analysis entirely + - Proceed directly to Step 5 — scope is preserved: + - If `test_name` was provided → produce singular test report (Branch A) + - If `test_name` was omitted → produce multi-step report (Branch B) + - Do NOT prompt user about must-gather + +2. **Extract actual test name from prowjob.json** + + The artifacts directory uses the test name from prowjob.json, NOT the full URL path. + + ```bash + # Extract test name from prowjob.json (e.g., "e2e-aws-operator-serial-ote") + JOB_NAME=$(jq -r '.spec.job' .work/prow-job-analyze-test-failure/{build_id}/logs/prowjob.json) + + # Note: For PR jobs, TARGET contains the full PR path like: + # pr-logs/pull/openshift_service-ca-operator/306/pull-ci-openshift-service-ca-operator-main-e2e-aws-operator-serial-ote + # But artifacts are stored under just the test name: + # e2e-aws-operator-serial-ote + ``` + +3. **Detect must-gather archive** (only if --fast not present) + + Use JOB_NAME (not TARGET) for artifact paths. + + HyperShift jobs may have **different must-gather patterns**: + + **Pattern 1: Unified Archive** (dump-management-cluster) + - Single archive with both management and hosted cluster data + - Used by: hypershift-aws-e2e-external workflow + + **Pattern 2: Dual Archives** (gather-must-gather + dump) + - Standard must-gather for management cluster + - Separate hypershift-dump for additional data (may or may not have hosted cluster) + - Used by: hypershift-kubevirt-e2e-aws workflow + + **Pattern 3: Standard Only** (gather-must-gather) + - Standard OpenShift must-gather only + - No HyperShift-specific dump + + **Detection logic**: + ```bash + # Check for Pattern 1: Unified archive (dump-management-cluster) + UNIFIED_DUMP=$(gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/$JOB_NAME/dump-management-cluster/artifacts/artifacts.tar*" 2>/dev/null | head -1 || true) + + # Check for Pattern 2/3: Standard must-gather + STANDARD_MG=$(gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/$JOB_NAME/gather-must-gather/artifacts/must-gather.tar" 2>/dev/null || true) + + # Check for Pattern 2: Additional hypershift-dump (multiple possible locations) + # Use wildcards to match all current and future HyperShift dump patterns: + # 1. **/artifacts/hypershift-dump.tar (covers dump/, hypershift-mce-dump/, etc.) + # 2. **/artifacts/**/hostedcluster.tar (covers all E2E test patterns) + HYPERSHIFT_DUMP="" + for pattern in \ + "**/artifacts/hypershift-dump.tar" \ + "**/artifacts/**/hostedcluster.tar"; do + FOUND=$(gcloud storage ls "gs://test-platform-results/{bucket-path}/artifacts/$JOB_NAME/$pattern" 2>/dev/null | head -1 || true) + if [ -n "$FOUND" ]; then + HYPERSHIFT_DUMP="$FOUND" + break + fi + done + + # Determine pattern and check for hosted cluster data + if [ -n "$UNIFIED_DUMP" ]; then + # Pattern 1: Unified archive + PATTERN="unified" + + # Download temporarily to check for hosted cluster data + TMP_CHECK="/tmp/check-unified-$$.tar" + gcloud storage cp "$UNIFIED_DUMP" "$TMP_CHECK" --no-user-output-enabled + + # Check if archive contains hostedcluster-* directory + HAS_HOSTED_CLUSTER=$(tar -tf "$TMP_CHECK" 2>/dev/null | grep -q "hostedcluster-" && echo "true" || echo "false") + rm -f "$TMP_CHECK" + + elif [ -n "$STANDARD_MG" ] && [ -n "$HYPERSHIFT_DUMP" ]; then + # Pattern 2: Dual archives + PATTERN="dual" + + # Download hypershift-dump temporarily to check for hosted cluster + TMP_CHECK="/tmp/check-dump-$$.tar" + gcloud storage cp "$HYPERSHIFT_DUMP" "$TMP_CHECK" --no-user-output-enabled + + # Check if hypershift-dump contains hostedcluster-* directory + HAS_HOSTED_CLUSTER=$(tar -tf "$TMP_CHECK" 2>/dev/null | grep -q "hostedcluster-" && echo "true" || echo "false") + rm -f "$TMP_CHECK" + + elif [ -n "$STANDARD_MG" ]; then + # Pattern 3: Standard must-gather only + PATTERN="standard" + HAS_HOSTED_CLUSTER=false + + else + # No must-gather found + PATTERN="none" + HAS_HOSTED_CLUSTER=false + fi + ``` + + Possible outcomes: + - **PATTERN="none"**: No must-gather found → Skip to Step 5 (silent, expected for some jobs) + - **PATTERN="standard"**: Standard OpenShift must-gather only → Extract single cluster + - **PATTERN="unified" + HAS_HOSTED_CLUSTER=false**: HyperShift unified archive with management only + - **PATTERN="unified" + HAS_HOSTED_CLUSTER=true**: HyperShift unified archive with both clusters + - **PATTERN="dual" + HAS_HOSTED_CLUSTER=false**: Two archives but hosted cluster not in hypershift-dump + - **PATTERN="dual" + HAS_HOSTED_CLUSTER=true**: Two archives with hosted cluster in hypershift-dump + + **Important Pattern Notes**: + - **Pattern 1 (unified)**: Management at `logs/artifacts/output/`, hosted at `logs/artifacts/output/hostedcluster-{name}/` + - **Pattern 2 (dual)**: Management in standard must-gather, hosted MAY be in hypershift-dump.tar or hostedcluster.tar (not guaranteed) + - Wildcard patterns match all current and future dump locations + - **Pattern 3 (standard)**: Management cluster only, no HyperShift-specific data + +4. **Ask user if they want must-gather analysis** + - Only if must-gather(s) were found and --fast not present + - Use AskUserQuestion tool: + - Question: "Must-gather data is available. Include cluster diagnostics in the analysis?" + - Header: "Must-gather" + - Options: + - Label: "Yes - Extract and analyze must-gather (Recommended)" + Description: "Provides cluster-level diagnostics that may reveal root causes (pods, operators, nodes, events). Takes additional time to download and analyze." + - Label: "No - Skip must-gather (faster)" + Description: "Only analyze test-level artifacts (build-log, intervals). Faster but may miss cluster-level issues." + - If user chooses "No", skip to Step 5 + +### Step 4.6: Extract Must-Gather (Conditional) + +Only if user chose "Yes" in Step 4.5: + +1. **Determine extraction strategy** + - If single must-gather detected → extract to `must-gather/logs/` + - If dual must-gather detected (HyperShift) → extract both: + - Management cluster → `must-gather-mgmt/logs/` + - Hosted cluster → `must-gather-hosted/logs/` + +2. **Check for existing extraction** + + For single must-gather: + - Check if `.work/prow-job-analyze-test-failure/{build_id}/must-gather/logs/` exists with content + + For dual must-gather (HyperShift): + - Check if `.work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs/` exists with content + - Check if `.work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs/` exists with content + + If either exists: + - Use AskUserQuestion tool: + - Question: "Must-gather already extracted for this build. Use existing data?" + - Header: "Reuse" + - Options: + - Label: "Use existing" + Description: "Reuse previously extracted must-gather data (faster)" + - Label: "Re-extract" + Description: "Download and extract fresh must-gather data" + - If "Re-extract": + - `rm -rf .work/prow-job-analyze-test-failure/{build_id}/must-gather*/` + - Continue to step 3 (fresh extraction) + - If "Use existing": + - Validate content directories exist and are not empty (see validation in step 4.6) + - If validation fails, fall back to re-extraction + - If validation succeeds, skip to Step 4.7 + +3. **Create must-gather directories** + + Based on PATTERN from Step 4.5.3: + + For Pattern 3 (standard only): + ```bash + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather/logs + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather/tmp + ``` + + For Pattern 1 (unified) or Pattern 2 (dual): + ```bash + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs + mkdir -p .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/tmp + ``` + +4. **Download must-gather archives** + + Use JOB_NAME (from Step 4.5.2) for artifact paths, not {target}: + + For Pattern 3 (standard only): + ```bash + gcloud storage cp "$STANDARD_MG" \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather/tmp/must-gather.tar \ + --no-user-output-enabled + ``` + + For Pattern 1 (unified): + ```bash + # Download unified archive (contains both management and hosted cluster data) + gcloud storage cp "$UNIFIED_DUMP" \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/unified-dump.tar \ + --no-user-output-enabled + ``` + + For Pattern 2 (dual): + ```bash + # Download management cluster must-gather + gcloud storage cp "$STANDARD_MG" \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/must-gather.tar \ + --no-user-output-enabled + + # Download hypershift-dump (may or may not contain hosted cluster) + gcloud storage cp "$HYPERSHIFT_DUMP" \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/tmp/hypershift-dump.tar \ + --no-user-output-enabled + ``` + +5. **Extract archives** + + For Pattern 3 (standard only): + ```bash + # Use existing extract_archives.py script for standard must-gather + python3 plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather/tmp/must-gather.tar \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather/logs + ``` + + For Pattern 1 (unified): + ```bash + # Extract unified archive to temporary location + TMP_EXTRACT=".work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/extracted" + mkdir -p "$TMP_EXTRACT" + + # Handle both .tar and .tar.gz + if [[ "$UNIFIED_DUMP" == *.tar.gz ]]; then + tar -xzf .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/unified-dump.tar -C "$TMP_EXTRACT" + else + tar -xf .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/unified-dump.tar -C "$TMP_EXTRACT" + fi + + # Find the output directory (may be at logs/artifacts/output or just output) + OUTPUT_DIR=$(find "$TMP_EXTRACT" -type d -name "output" | head -1) + + if [ -z "$OUTPUT_DIR" ]; then + echo "ERROR: Could not find output directory in unified dump" + rm -rf "$TMP_EXTRACT" + # Clear variables to prevent subsequent usage + HAS_HOSTED_CLUSTER="false" + unset HOSTED_DIR + unset OUTPUT_DIR + # Skip to Step 5 - no must-gather analysis possible + else + # Move management cluster data (root level in output/) + # Exclude hostedcluster-* directories + for item in "$OUTPUT_DIR"/*; do + if [ -e "$item" ] && [[ ! "$(basename "$item")" =~ ^hostedcluster- ]]; then + mv "$item" .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs/ + fi + done + + # Move hosted cluster data (hostedcluster-* subdirectory) + if [ "$HAS_HOSTED_CLUSTER" = "true" ]; then + HOSTED_DIR=$(find "$OUTPUT_DIR" -maxdepth 1 -type d -name "hostedcluster-*" | head -1) + if [ -n "$HOSTED_DIR" ]; then + mv "$HOSTED_DIR"/* .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs/ + echo "✓ Hosted cluster data extracted from unified archive" + else + echo "WARNING: Expected hosted cluster data but hostedcluster-* directory not found" + fi + fi + + # Cleanup temporary extraction directory + rm -rf "$TMP_EXTRACT" + fi + ``` + + For Pattern 2 (dual): + ```bash + # Extract management cluster must-gather (standard format) + python3 plugins/prow-job/skills/prow-job-extract-must-gather/extract_archives.py \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/tmp/must-gather.tar \ + .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs + + # Extract hypershift-dump (may contain hosted cluster) + TMP_EXTRACT=".work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/tmp/extracted" + mkdir -p "$TMP_EXTRACT" + tar -xf .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/tmp/hypershift-dump.tar -C "$TMP_EXTRACT" + + if [ "$HAS_HOSTED_CLUSTER" = "true" ]; then + # Look for hostedcluster-* directory in dump + HOSTED_DIR=$(find "$TMP_EXTRACT" -maxdepth 2 -type d -name "hostedcluster-*" | head -1) + if [ -n "$HOSTED_DIR" ]; then + mv "$HOSTED_DIR"/* .work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs/ + echo "✓ Hosted cluster data extracted from hypershift-dump" + else + echo "WARNING: HAS_HOSTED_CLUSTER=true but no hostedcluster-* directory found in dump" + echo "Hypershift-dump likely contains only management cluster data" + HAS_HOSTED_CLUSTER=false # Update flag since hosted cluster not actually present + fi + else + echo "INFO: Hypershift-dump does not contain hosted cluster data (management cluster only)" + fi + + # Cleanup temporary extraction directory + rm -rf "$TMP_EXTRACT" + ``` + +6. **Locate and validate content directories** + + For Pattern 3 (standard only): + ```bash + # Check for content/ directory first (renamed by extraction script) + if [ -d ".work/prow-job-analyze-test-failure/{build_id}/must-gather/logs/content" ]; then + MUST_GATHER_PATH=".work/prow-job-analyze-test-failure/{build_id}/must-gather/logs/content" + else + # Fall back to finding the directory containing -ci- (e.g., registry-build09-ci-...) + MUST_GATHER_PATH=$(find .work/prow-job-analyze-test-failure/{build_id}/must-gather/logs -maxdepth 1 -type d -name "*-ci-*" | head -1) + fi + + # Validate MUST_GATHER_PATH is set and directory exists + if [ -z "$MUST_GATHER_PATH" ] || [ ! -d "$MUST_GATHER_PATH" ]; then + echo "ERROR: Must-gather content directory not found after extraction" + # Skip to Step 5 (scope preserved: Branch A or Branch B per test_name presence) + elif [ -z "$(ls -A "$MUST_GATHER_PATH" 2>/dev/null)" ]; then + echo "ERROR: Must-gather content directory is empty" + # Skip to Step 5 (scope preserved: Branch A or Branch B per test_name presence) + else + echo "✓ Must-gather content located at: $MUST_GATHER_PATH" + # Continue to Step 4.7 with MUST_GATHER_PATH set + fi + ``` + + For Pattern 1 (unified) or Pattern 2 (dual): + ```bash + # Management cluster validation + if [ "$PATTERN" = "unified" ]; then + # Pattern 1: Data extracted directly to logs/ directory + MUST_GATHER_MGMT_PATH=".work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs" + else + # Pattern 2: Standard must-gather extraction (look for content/ or hash directory) + if [ -d ".work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs/content" ]; then + MUST_GATHER_MGMT_PATH=".work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs/content" + else + MUST_GATHER_MGMT_PATH=$(find .work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs -maxdepth 1 -type d -name "*-ci-*" | head -1) + fi + fi + + # Validate management cluster path + if [ -z "$MUST_GATHER_MGMT_PATH" ] || [ ! -d "$MUST_GATHER_MGMT_PATH" ]; then + echo "ERROR: Management cluster directory not found" + # Skip to Step 5 (scope preserved: Branch A or Branch B per test_name presence) + elif [ -z "$(ls -A "$MUST_GATHER_MGMT_PATH" 2>/dev/null)" ]; then + echo "ERROR: Management cluster directory is empty" + # Skip to Step 5 (scope preserved: Branch A or Branch B per test_name presence) + else + echo "✓ Management cluster data located at: $MUST_GATHER_MGMT_PATH" + fi + + # Hosted cluster validation - only if HAS_HOSTED_CLUSTER is true + if [ "$HAS_HOSTED_CLUSTER" = "true" ]; then + MUST_GATHER_HOSTED_PATH=".work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs" + + # Validate hosted cluster path + if [ ! -d "$MUST_GATHER_HOSTED_PATH" ]; then + echo "WARNING: Hosted cluster directory not found (expected based on archive detection)" + MUST_GATHER_HOSTED_PATH="" # Clear the path + elif [ -z "$(ls -A "$MUST_GATHER_HOSTED_PATH" 2>/dev/null)" ]; then + echo "WARNING: Hosted cluster directory is empty" + MUST_GATHER_HOSTED_PATH="" # Clear the path + else + echo "✓ Hosted cluster data located at: $MUST_GATHER_HOSTED_PATH" + fi + else + echo "✓ Management cluster must-gather located at: $MUST_GATHER_MGMT_PATH" + fi + + # Only validate hosted cluster if HAS_HOSTED_CLUSTER is true + if [ "$HAS_HOSTED_CLUSTER" = "true" ]; then + if [ -z "$MUST_GATHER_HOSTED_PATH" ] || [ ! -d "$MUST_GATHER_HOSTED_PATH" ]; then + echo "ERROR: Hosted cluster must-gather content directory not found" + elif [ -z "$(ls -A "$MUST_GATHER_HOSTED_PATH" 2>/dev/null)" ]; then + echo "ERROR: Hosted cluster must-gather content directory is empty" + else + echo "✓ Hosted cluster must-gather located at: $MUST_GATHER_HOSTED_PATH" + echo "✓ Hosted cluster namespace: $HOSTED_NAMESPACE" + fi + fi + ``` + +### Step 4.7: Analyze Must-Gather (Conditional) + +Only if Step 4.6 completed successfully: + +1. **Locate must-gather-analyzer scripts** + + The must-gather plugin provides analysis scripts. Locate the scripts directory: + + ```bash + # Try to find the must-gather-analyzer scripts in common locations + for SEARCH_PATH in \ + "plugins/must-gather/skills/must-gather-analyzer/scripts" \ + "~/.claude/plugins/cache/*/plugins/must-gather/skills/must-gather-analyzer/scripts" \ + "$(find ~ -type d -path "*/must-gather/skills/must-gather-analyzer/scripts" 2>/dev/null | head -1)"; do + SCRIPTS_DIR=$(eval echo "$SEARCH_PATH") + if [ -d "$SCRIPTS_DIR" ] && [ -f "$SCRIPTS_DIR/analyze_clusteroperators.py" ]; then + break + fi + SCRIPTS_DIR="" + done + + if [ -z "$SCRIPTS_DIR" ]; then + echo "WARNING: Must-gather analysis scripts not found." + echo "Install the must-gather plugin: /plugin install must-gather@ai-helpers" + # Continue to Step 5 without cluster analysis + fi + ``` + +2. **Run targeted cluster diagnostics** + + Focus on issues relevant to test failures (not full cluster analysis). + + **For single must-gather (standard OpenShift):** + + ```bash + # Core diagnostics (only if scripts and path are available) + if [ -n "$SCRIPTS_DIR" ] && [ -n "$MUST_GATHER_PATH" ]; then + python3 "$SCRIPTS_DIR/analyze_clusteroperators.py" "$MUST_GATHER_PATH" + python3 "$SCRIPTS_DIR/analyze_pods.py" "$MUST_GATHER_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_nodes.py" "$MUST_GATHER_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_events.py" "$MUST_GATHER_PATH" --type Warning --count 50 + else + echo "WARNING: Skipping must-gather analysis (scripts or path not available)" + fi + ``` + + **For dual must-gather (HyperShift):** + + ```bash + # Management cluster diagnostics (only if scripts and path are available) + if [ -n "$SCRIPTS_DIR" ] && [ -n "$MUST_GATHER_MGMT_PATH" ]; then + echo "=== Analyzing Management Cluster ===" + python3 "$SCRIPTS_DIR/analyze_clusteroperators.py" "$MUST_GATHER_MGMT_PATH" + python3 "$SCRIPTS_DIR/analyze_pods.py" "$MUST_GATHER_MGMT_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_nodes.py" "$MUST_GATHER_MGMT_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_events.py" "$MUST_GATHER_MGMT_PATH" --type Warning --count 50 + else + echo "WARNING: Skipping management cluster analysis (scripts or path not available)" + fi + + # Hosted cluster diagnostics (only if scripts and path are available) + if [ -n "$SCRIPTS_DIR" ] && [ -n "$MUST_GATHER_HOSTED_PATH" ]; then + echo "=== Analyzing Hosted Cluster (Namespace: $HOSTED_NAMESPACE) ===" + python3 "$SCRIPTS_DIR/analyze_clusteroperators.py" "$MUST_GATHER_HOSTED_PATH" + python3 "$SCRIPTS_DIR/analyze_pods.py" "$MUST_GATHER_HOSTED_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_nodes.py" "$MUST_GATHER_HOSTED_PATH" --problems-only + python3 "$SCRIPTS_DIR/analyze_events.py" "$MUST_GATHER_HOSTED_PATH" --type Warning --count 50 + else + echo "INFO: Skipping hosted cluster analysis (scripts or path not available)" + fi + ``` + +3. **Run conditional diagnostics based on test context** + + ```bash + # Network diagnostics (if test name suggests network issues) + if [[ "$JOB_NAME" =~ network|ovn|sdn|connectivity|route|ingress|egress ]]; then + if [ -n "$MUST_GATHER_PATH" ]; then + python3 "$SCRIPTS_DIR/analyze_network.py" "$MUST_GATHER_PATH" + fi + if [ -n "$MUST_GATHER_MGMT_PATH" ]; then + echo "=== Management Cluster Network ===" + python3 "$SCRIPTS_DIR/analyze_network.py" "$MUST_GATHER_MGMT_PATH" + fi + if [ -n "$MUST_GATHER_HOSTED_PATH" ]; then + echo "=== Hosted Cluster Network ===" + python3 "$SCRIPTS_DIR/analyze_network.py" "$MUST_GATHER_HOSTED_PATH" + fi + fi + + # etcd diagnostics (if test name suggests control-plane issues) + if [[ "$JOB_NAME" =~ etcd|apiserver|control-plane|kube-apiserver ]]; then + if [ -n "$MUST_GATHER_PATH" ]; then + python3 "$SCRIPTS_DIR/analyze_etcd.py" "$MUST_GATHER_PATH" + fi + if [ -n "$MUST_GATHER_MGMT_PATH" ]; then + echo "=== Management Cluster etcd ===" + python3 "$SCRIPTS_DIR/analyze_etcd.py" "$MUST_GATHER_MGMT_PATH" + fi + if [ -n "$MUST_GATHER_HOSTED_PATH" ]; then + echo "=== Hosted Cluster etcd ===" + python3 "$SCRIPTS_DIR/analyze_etcd.py" "$MUST_GATHER_HOSTED_PATH" + fi + fi + ``` + + See `plugins/must-gather/skills/must-gather-analyzer/SKILL.md` for all available analysis scripts. + +4. **Capture analysis output** + - Store script output for correlation in Step 4.8 + - Keep management and hosted cluster outputs separate + - Use in final report in Step 5 + +### Step 4.8: Correlate Cluster Issues with Test Failure + +Only if Step 4.7 completed: + +1. **Temporal correlation** + - From Step 4 (interval files), you identified when the test was running (from/to timestamps) + - Review cluster operator conditions, pod events, and warning events for timing alignment + - Identify cluster issues that occurred during or shortly before test failure (±5 minutes) + - Example: "Test failed at 10:23:45. Network operator became degraded at 10:23:12." + + **For HyperShift (dual must-gather):** + - Correlate issues from BOTH management and hosted clusters + - Note which cluster (management vs hosted) each issue occurred in + - Example: "Test failed at 10:23:45. Hosted cluster network operator became degraded at 10:23:12." + +2. **Component correlation** + - Map test failure to cluster components: + - **Namespace correlation**: Test runs in specific namespace → check for pod failures in that namespace + - For HyperShift: Tests typically run in hosted cluster namespace (e.g., `clusters-{namespace}`) + - **Test assertions correlation**: Test type suggests affected components + - Network tests → network operator status, CNI pods, network policies + - Storage tests → storage operator, CSI pods, PVs/PVCs + - API tests → kube-apiserver pods, API server operator + - **Stack trace correlation**: Error messages in stack trace → related Kubernetes resources + - "connection refused" → check pod restarts, network issues + - "timeout" → check node pressure, resource constraints + - "not found" → check resource deletion events + + **For HyperShift (dual must-gather):** + - **Management cluster issues** typically affect: + - HostedControlPlane pods (kube-apiserver, etcd, etc. in `clusters-{namespace}` namespace) + - HyperShift operator + - Management cluster nodes hosting control plane + - **Hosted cluster issues** typically affect: + - Worker node pods + - Cluster operators + - Application workloads + +3. **Generate correlated insights** + - Create specific, actionable correlations like: + - "Test failed at {time}. {Operator} became degraded at {time} with reason: {reason}" + - "Pod crash-looping in test namespace: {namespace}/{pod-name}" + - "Node {node-name} reported {condition} at {time}, test pod was scheduled on this node" + - "Warning event: {event-message} at {time} (during test execution)" + + **For HyperShift (dual must-gather):** + - Prefix correlations with cluster type: + - "[Management Cluster] HostedControlPlane pod restarting: clusters-{namespace}/kube-apiserver-*" + - "[Hosted Cluster] Network operator degraded with reason: {reason}" + - Cross-cluster correlations: + - "Management cluster node pressure → Hosted cluster control plane unavailable" + - "HyperShift operator error → HostedControlPlane rollout failed" + + - Store these insights for inclusion in Step 4.9 root cause determination + +### Step 4.9: Determine Root Cause + +Synthesize all gathered evidence to determine the most likely root cause for the test failure. + +**CRITICAL: Be tenacious — trace symptoms back to root cause.** Never stop at high-level symptoms like "nodes didn't join", "operator unavailable", or "containers are crash-looping". Trace backwards through the dependency chain (pod statuses → container logs → originating error) until you find the specific, actionable root cause. + +1. **Analyze all available evidence** + - Stack traces from Step 4.2 + - Test code analysis from Step 4.4 + - Interval file events from Step 4.3 + - Known symptom labels from Step 4.3b (if available — use as supporting context, not as root cause) + - Cluster diagnostics from Step 4.7 (if available) + - Correlations from Step 4.8 (if available) + +2. **Prioritize evidence based on temporal proximity** + - Issues occurring during test execution time (from interval files) are most relevant + - Cluster issues that occurred shortly before test failure (±5 minutes) are highly relevant + - Pre-existing cluster issues may be contributing factors but not root causes + +3. **Generate root cause hypothesis** + - Primary cause: The most direct, immediate cause of test failure + - Contributing factors: Cluster or environmental issues that enabled the failure + - Evidence summary: Key evidence supporting the hypothesis + + **For HyperShift jobs:** + - Distinguish between management cluster vs hosted cluster root causes + - Identify cross-cluster dependencies (e.g., management node pressure → hosted control plane unavailable) + +4. **Formulate actionable recommendations** + - What needs to be fixed (code, configuration, cluster state) + - Where to look for more information + - Suggested next steps for debugging or resolution + +### Step 5: Present Results to User + +1. **Display structured summary with enhanced formatting** + + **Branch on whether test_name was provided (mirrors Step 4.2 branching):** + + - **Branch A (test_name provided):** Use the singular `{test_name}` report shape below. + - **Branch B (test_name NOT provided):** Use the multi-step report shape further below, + iterating over each failed step discovered in Step 4.2 Branch B and aggregating + their findings into a combined report. + + **For single must-gather or no must-gather (Branch A — test_name provided):** + + ```text + # Test Failure Analysis Complete + + ## Job Information + - **Prow Job**: {prowjob-name} + - **Build ID**: {build_id} + - **Target**: {target} + - **Test**: {test_name} + + ## Known Symptoms Seen + *(Only if symptom labels were found in Step 4.3b — omit section entirely if none)* + - {symptom summary}: {symptom explanation} + > **Note**: Symptoms are machine-detected environmental observations, not definitive causes. They add context to help explain failures when correlated with other evidence. + + ## Test Failure Analysis + + ### Error + {error message from stack trace} + + ### Summary + {failure analysis from stack trace and code} + + ### Evidence + {evidence from build-log.txt and interval files} + + ### Additional Evidence + {additional evidence from logs/events} + + --- + + ## Cluster Diagnostics + *(Only if must-gather was analyzed)* + + ### Cluster Operators + {output from analyze_clusteroperators.py} + + ### Problematic Pods + {output from analyze_pods.py --problems-only} + + ### Node Issues + {output from analyze_nodes.py --problems-only} + + ### Recent Warning Events + {output from analyze_events.py} + + ### Network Analysis + *(Only if network-related test)* + {output from analyze_network.py} + + ### etcd Analysis + *(Only if etcd-related test)* + {output from analyze_etcd.py} + + --- + + ## Correlation + *(Only if must-gather was analyzed)* + + ### Timeline + - **Test started**: {from timestamp} + - **Test failed**: {to timestamp} + - **Cluster events during test**: + - {cluster-event} at {timestamp} + - {cluster-event} at {timestamp} + + ### Affected Components + - {affected operators/pods/nodes} + + ### Root Cause Hypothesis + {correlated analysis combining test-level and cluster-level evidence} + + --- + + ## Artifacts + - **Test artifacts**: `.work/prow-job-analyze-test-failure/{build_id}/logs/` + - **Must-gather**: `.work/prow-job-analyze-test-failure/{build_id}/must-gather/logs/` *(if extracted)* + ``` + + **For dual must-gather (HyperShift):** + + ```text + # Test Failure Analysis Complete (HyperShift) + + ## Job Information + - **Prow Job**: {prowjob-name} + - **Build ID**: {build_id} + - **Target**: {target} + - **Test**: {test_name} + - **Hosted Cluster Namespace**: {HOSTED_NAMESPACE} + + ## Known Symptoms Seen + *(Only if symptom labels were found in Step 4.3b — omit section entirely if none)* + - {symptom summary}: {symptom explanation} + > **Note**: Symptoms are machine-detected environmental observations, not definitive causes. They add context to help explain failures when correlated with other evidence. + + ## Test Failure Analysis + + ### Error + {error message from stack trace} + + ### Summary + {failure analysis from stack trace and code} + + ### Evidence + {evidence from build-log.txt and interval files} + + ### Additional Evidence + {additional evidence from logs/events} + + --- + + ## Management Cluster Diagnostics + + ### Cluster Operators + {output from analyze_clusteroperators.py for management cluster} + + ### Problematic Pods + {output from analyze_pods.py --problems-only for management cluster} + + ### Node Issues + {output from analyze_nodes.py --problems-only for management cluster} + + ### Recent Warning Events + {output from analyze_events.py for management cluster} + + ### Network Analysis + *(Only if network-related test)* + {output from analyze_network.py for management cluster} + + ### etcd Analysis + *(Only if etcd-related test)* + {output from analyze_etcd.py for management cluster} + + --- + + ## Hosted Cluster Diagnostics + + **Namespace**: `{HOSTED_NAMESPACE}` + + ### Cluster Operators + {output from analyze_clusteroperators.py for hosted cluster} + + ### Problematic Pods + {output from analyze_pods.py --problems-only for hosted cluster} + + ### Node Issues + {output from analyze_nodes.py --problems-only for hosted cluster} + + ### Recent Warning Events + {output from analyze_events.py for hosted cluster} + + ### Network Analysis + *(Only if network-related test)* + {output from analyze_network.py for hosted cluster} + + ### etcd Analysis + *(Only if etcd-related test)* + {output from analyze_etcd.py for hosted cluster} + + --- + + ## Correlation + + ### Timeline + - **Test started**: {from timestamp} + - **Test failed**: {to timestamp} + - **Management cluster events during test**: + - {cluster-event} at {timestamp} + - **Hosted cluster events during test**: + - {cluster-event} at {timestamp} + + ### Affected Components + + **Management Cluster**: + - {affected operators/pods/nodes} + + **Hosted Cluster**: + - {affected operators/pods/nodes} + + ### Root Cause Hypothesis + {correlated analysis combining: + - Test-level evidence + - Management cluster diagnostics + - Hosted cluster diagnostics + - Cross-cluster interactions} + + --- + + ## Artifacts + - **Test artifacts**: `.work/prow-job-analyze-test-failure/{build_id}/logs/` + - **Management cluster must-gather**: `.work/prow-job-analyze-test-failure/{build_id}/must-gather-mgmt/logs/` + - **Hosted cluster must-gather**: `.work/prow-job-analyze-test-failure/{build_id}/must-gather-hosted/logs/` + ``` + + **For Branch B (test_name NOT provided — multi-step failure report):** + + Iterate over the ordered list of failed steps discovered in Step 4.2 Branch B and produce + one section per step, then aggregate into a combined root cause section. + + ```text + # Test Failure Analysis Complete (Multi-Step) + + ## Job Information + - **Prow Job**: {prowjob-name} + - **Build ID**: {build_id} + - **Target**: {target} + - **Failed Steps**: {count of discovered failed steps} + + ## Known Symptoms Seen + *(Only if symptom labels were found in Step 4.3b — omit section entirely if none)* + - {symptom summary}: {symptom explanation} + > **Note**: Symptoms are machine-detected environmental observations, not definitive causes. + + --- + + ## Failed Step Analyses + + *(Repeat the following block for each failed step discovered in Step 4.2 Branch B)* + + ### Step: {step_name} + + #### Error + {failure_message from JUnit XML for this step} + + #### Summary + {analysis of this step's stack trace and log context} + + #### Evidence + {stack trace extracted from {step_name}-build-log.txt or build-log.txt context} + + #### Artifacts Examined + - `.work/prow-job-analyze-test-failure/{build_id}/logs/{step_name}-build-log.txt` + - `.work/prow-job-analyze-test-failure/{build_id}/logs/{step_name}/` *(step artifacts if downloaded)* + + --- + + ## Cluster Diagnostics + *(Only if must-gather was analyzed — same structure as Branch A; include per-cluster + sections as appropriate for standard or HyperShift patterns)* + + --- + + ## Aggregated Root Cause + + ### Failed Steps Summary + | Step | One-line Failure | + |------|-----------------| + | {step_name_1} | {summary} | + | {step_name_2} | {summary} | + | ... | ... | + + ### Timeline + *(Emit one entry per failed step. If interval data cannot be mapped to a specific + step, omit the Timeline section entirely rather than showing a misleading single window.)* + + - **{step_name_1}**: started {from timestamp} — failed {to timestamp} + - Cluster events during this window: {cluster-event} at {timestamp} + - **{step_name_2}**: started {from timestamp} — failed {to timestamp} + - Cluster events during this window: {cluster-event} at {timestamp} + - *(repeat for each failed step with mappable interval data)* + + ### Root Cause Hypothesis + {synthesized root cause drawn from all per-step findings (install + test steps), + per-step interval events, symptom labels, and cluster diagnostics; clearly + identify whether one step is the root cause or whether multiple independent + failures occurred} + + ### Recommendations + - {actionable next step 1} + - {actionable next step 2} + + --- + + ## Artifacts + - **Test artifacts**: `.work/prow-job-analyze-test-failure/{build_id}/logs/` + - **Must-gather**: `.work/prow-job-analyze-test-failure/{build_id}/must-gather*/logs/` *(if extracted)* + ``` + +2. **Display completion message** + + ```text + ✅ Analysis complete! + 📄 Report: .work/prow-job-analyze-test-failure/{build_id}/analysis.md + + 💡 Tip: The Markdown report can be copied directly into JIRA Description field + ``` + +## Error Handling + +Handle errors in the same way as "Error handling" in "prow-job-analyze-resource" skill, with these additional must-gather-specific cases: + +1. **Must-gather not available** + - If `gcloud storage ls` returns 404 for must-gather.tar, this is expected (not all jobs have must-gather) + - Silently skip must-gather analysis - do NOT warn the user + - Continue to Step 5 preserving scope: Branch A (singular test) or Branch B (multi-step) as determined by whether `test_name` was provided + +2. **Must-gather extraction fails** + - If download or extraction fails, warn the user but continue with test analysis + - Display: "WARNING: Must-gather extraction failed: {error}. Continuing without cluster diagnostics." + - Continue to Step 5 preserving scope: Branch A (singular test) or Branch B (multi-step) as determined by whether `test_name` was provided + +3. **Analysis scripts not found** + - If `find` command returns empty (no scripts found), warn the user + - Display: "WARNING: Must-gather analysis scripts not installed. Install the must-gather plugin from openshift-eng/ai-helpers for cluster diagnostics." + - Continue to Step 5 preserving scope: Branch A (singular test) or Branch B (multi-step) as determined by whether `test_name` was provided + +4. **Partial analysis script failures** + - If one script fails (non-zero exit code), continue with other scripts + - Capture and report which analyses succeeded/failed + - Display failed analyses as: "WARNING: {script-name} analysis failed: {error}" + - Include successful analyses in final report + +5. **Empty analysis results** + - If a script runs successfully but produces no output or no issues found + - Display: "{Analysis-type}: No issues detected" + - This is informational, not an error + +## Performance Considerations + +Follow the instructions in "Performance Considerations" in "prow-job-analyze-resource" skill diff --git a/plugins/ci/skills/prow-job-artifact-search/SKILL.md b/plugins/ci/skills/prow-job-artifact-search/SKILL.md new file mode 100644 index 000000000..f79fb6ae3 --- /dev/null +++ b/plugins/ci/skills/prow-job-artifact-search/SKILL.md @@ -0,0 +1,259 @@ +--- +name: prow-job-artifact-search +description: Search, list, and fetch artifacts from Prow CI job runs stored in GCS using the gcloud CLI +--- + +# Prow Job Artifact Search + +This skill provides a reusable Python script for browsing, searching, and fetching artifacts from Prow CI job runs stored in the `test-platform-results` GCS bucket. It wraps `gcloud storage` commands so callers do not need to construct GCS URIs or manage the CLI directly. + +## When to Use This Skill + +Use this skill when you need to: +- Browse the artifact directory tree of a Prow job run +- Search for specific files within a job's artifacts (e.g., interval files, node logs, journal logs) +- Fetch the contents of a specific artifact file for analysis +- Discover what artifacts are available before running a more targeted analysis skill + +This skill is a building block used by other analysis workflows. Use it when you need ad-hoc artifact access that is not covered by a dedicated analysis skill. + +## Prerequisites + +1. **gcloud CLI Installation** + - Check if installed: `which gcloud` + - If not installed, provide instructions for the user's platform + - Installation guide: https://cloud.google.com/sdk/docs/install + +2. **gcloud Authentication (Optional)** + - The `test-platform-results` bucket is publicly accessible + - No authentication is required for read access + +3. **Python 3** (3.6 or later) + - Check: `which python3` + +## Script Location + +``` +plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py +``` + +## Operations + +### list — List directory contents + +List files and subdirectories at a given path within the job's artifact tree. + +```bash +python3 plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py \ + list [subpath] +``` + +**Arguments:** +- `prow-url` (required): The Prow job URL +- `subpath` (optional): Subdirectory path relative to the job root. If omitted, lists the job root. + +**Examples:** + +```bash +# List the job root +python3 .../prow_job_artifact_search.py "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-.../1234567890" list + +# List e2e test artifacts +python3 .../prow_job_artifact_search.py list artifacts/e2e-test/openshift-e2e-test + +# List gather-extra oc_cmds +python3 .../prow_job_artifact_search.py list artifacts/e2e-test/gather-extra/artifacts/oc_cmds +``` + +**Output:** + +```json +{ + "success": true, + "path": "gs://test-platform-results/logs///", + "count": 5, + "entries": [ + { + "name": "artifacts/", + "path": "artifacts", + "type": "directory", + "gcs_uri": "gs://test-platform-results/logs///artifacts/" + }, + { + "name": "build-log.txt", + "path": "build-log.txt", + "type": "file", + "gcs_uri": "gs://test-platform-results/logs///build-log.txt" + } + ] +} +``` + +### search — Search for files matching a glob pattern + +Recursively search for files matching a glob pattern under a given path. + +```bash +python3 plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py \ + search [subpath] +``` + +**Arguments:** +- `prow-url` (required): The Prow job URL +- `pattern` (required): Glob pattern to match. Supports `**` for recursive matching and `*` for wildcards. +- `subpath` (optional): Subdirectory to search within. If omitted, searches from the job root. + +**Examples:** + +```bash +# Find all interval JSON files +python3 .../prow_job_artifact_search.py search "**/*intervals*.json" + +# Find all e2e timeline files +python3 .../prow_job_artifact_search.py search "**/e2e-timelines_spyglass_*.json" + +# Find node-related oc_cmds output +python3 .../prow_job_artifact_search.py search "**/nodes" artifacts + +# Find journal logs for a specific node +python3 .../prow_job_artifact_search.py search "**/*worker-c-7t6ng*" artifacts + +# Find all junit files +python3 .../prow_job_artifact_search.py search "**/junit*.xml" + +# Find must-gather archives +python3 .../prow_job_artifact_search.py search "**/must-gather*" +``` + +**Output:** + +```json +{ + "success": true, + "pattern": "gs://test-platform-results/logs///**/*intervals*.json", + "count": 3, + "matches": [ + { + "name": "e2e-timelines_spyglass_20260209-043512.json", + "path": "artifacts/e2e-test/openshift-e2e-test/e2e-timelines_spyglass_20260209-043512.json", + "type": "file", + "gcs_uri": "gs://test-platform-results/logs///artifacts/..." + } + ] +} +``` + +### fetch — Fetch a specific file's contents + +Download and return the contents of a specific file from the job's artifacts. + +```bash +python3 plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py \ + fetch [--max-bytes N] +``` + +**Arguments:** +- `prow-url` (required): The Prow job URL +- `filepath` (required): Path to the file relative to the job root +- `--max-bytes` (optional): Maximum bytes to read (default: 524288 = 512KB). Files larger than this are truncated. + +**Examples:** + +```bash +# Fetch the build log +python3 .../prow_job_artifact_search.py fetch build-log.txt + +# Fetch a specific oc_cmds output +python3 .../prow_job_artifact_search.py fetch artifacts/e2e-test/gather-extra/artifacts/oc_cmds/nodes + +# Fetch a large file with higher limit +python3 .../prow_job_artifact_search.py fetch artifacts/e2e-test/openshift-e2e-test/build-log.txt --max-bytes 2097152 +``` + +**Output:** + +```json +{ + "success": true, + "path": "gs://test-platform-results/logs///build-log.txt", + "size_bytes": 45230, + "truncated": false, + "max_bytes": 524288, + "content": "... file contents ..." +} +``` + +## Common Artifact Paths + +These are common artifact paths within a Prow job. The `{target}` is the CI workflow step name (e.g., `e2e-gcp-ovn-rt-rhcos10-techpreview`). + +| Path | Description | +|------|-------------| +| `build-log.txt` | Top-level build log | +| `artifacts/{target}/` | All artifacts for the test step | +| `artifacts/{target}/openshift-e2e-test/` | E2E test output (build-log, junit, timelines) | +| `artifacts/{target}/openshift-e2e-test/build-log.txt` | E2E test console log | +| `artifacts/{target}/openshift-e2e-test/artifacts/e2e-timelines_spyglass_*.json` | Interval/timeline data | +| `artifacts/{target}/gather-extra/artifacts/oc_cmds/` | Cluster state snapshots (nodes, pods, events, etc.) | +| `artifacts/{target}/gather-extra/artifacts/pods/` | Pod logs from all namespaces | +| `artifacts/{target}/gather-extra/artifacts/audit_logs/` | API server audit logs | +| `artifacts/{target}/gather-extra/artifacts/journal_logs/` | Node journal logs (systemd) | +| `artifacts/{target}/gather-must-gather/artifacts/` | Must-gather archives | +| `prowjob.json` | Job metadata (payload tag, timing, etc.) | + +## URL Formats + +The script accepts both Prow UI URLs and gcsweb URLs: + +- `https://prow.ci.openshift.org/view/gs/test-platform-results/logs//` +- `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs//` + +## Error Handling + +### No matches found + +When a search returns no results, the script returns success with an empty matches array: + +```json +{ + "success": true, + "pattern": "...", + "count": 0, + "matches": [] +} +``` + +### gcloud not installed + +```json +{ + "success": false, + "error": "gcloud CLI is not installed. Install from: https://cloud.google.com/sdk/docs/install" +} +``` + +### Invalid path or 404 + +```json +{ + "success": false, + "error": "gcloud storage ls failed: ...", + "path": "gs://..." +} +``` + +## Notes + +- The `test-platform-results` bucket is publicly accessible. No authentication is required. +- The `--no-user-output-enabled` flag is used on `gcloud storage cp` to suppress progress bars. +- For large files, use `--max-bytes` to limit download size. The default is 512KB. +- The `search` command uses `gcloud storage ls` with glob patterns, which supports `**` for recursive matching. +- All output is JSON on stdout. Errors also produce a JSON object with `"success": false`. + +## See Also + +- Related Skill: `prow-job-analyze-test-failure` — Analyzes test failures using build logs and timelines +- Related Skill: `prow-job-analyze-install-failure` — Analyzes install failures using installer logs +- Related Skill: `prow-job-analyze-resource` — Analyzes resource lifecycles using audit logs +- Related Skill: `prow-job-extract-must-gather` — Extracts must-gather archives +- Related Skill: `fetch-prowjob-json` — Fetches prowjob.json metadata diff --git a/plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py b/plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py new file mode 100755 index 000000000..fdd7b50f9 --- /dev/null +++ b/plugins/ci/skills/prow-job-artifact-search/prow_job_artifact_search.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +""" +Search and retrieve artifacts from Prow CI job runs stored in GCS. + +Provides list, search, and fetch operations against the test-platform-results +GCS bucket using the gcloud CLI. + +Usage: + prow_job_artifact_search.py list [subpath] + prow_job_artifact_search.py search [subpath] + prow_job_artifact_search.py fetch [--max-bytes N] + +Examples: + # List top-level artifacts + prow_job_artifact_search.py list + + # List a specific subdirectory + prow_job_artifact_search.py list artifacts/e2e-test/openshift-e2e-test + + # Search for files matching a glob pattern (recursive) + prow_job_artifact_search.py search "**/*intervals*.json" + + # Search within a subdirectory + prow_job_artifact_search.py search "**/nodes" artifacts/e2e-test/gather-extra + + # Fetch a specific file + prow_job_artifact_search.py fetch artifacts/e2e-test/build-log.txt + + # Fetch with size limit (default 512KB) + prow_job_artifact_search.py fetch artifacts/e2e-test/build-log.txt --max-bytes 1048576 +""" + +import argparse +import json +import os +import re +import subprocess +import sys + + +BUCKET = "test-platform-results" +DEFAULT_MAX_BYTES = 512 * 1024 # 512KB + + +def check_gcloud(): + """Verify gcloud CLI is available.""" + try: + subprocess.run( + ["gcloud", "version"], + capture_output=True, + timeout=10, + ) + except FileNotFoundError: + print( + json.dumps({ + "success": False, + "error": "gcloud CLI is not installed. Install from: https://cloud.google.com/sdk/docs/install", + }), + ) + sys.exit(1) + except subprocess.TimeoutExpired: + print( + json.dumps({ + "success": False, + "error": "gcloud CLI timed out during version check", + }), + ) + sys.exit(1) + + +def parse_prow_url(url): + """Extract the GCS path prefix from a Prow job URL. + + Accepts either: + - https://prow.ci.openshift.org/view/gs/test-platform-results/logs// + - https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs// + + Returns the path after the bucket name, e.g.: + logs// + """ + # Normalise to just the portion after the bucket name + patterns = [ + r"prow\.ci\.openshift\.org/view/gs/test-platform-results/(.+?)/?$", + r"gcsweb-ci\.apps\.ci\.l2s4\.p1\.openshiftapps\.com/gcs/test-platform-results/(.+?)/?$", + ] + for pat in patterns: + m = re.search(pat, url) + if m: + return m.group(1).rstrip("/") + + # Fallback: look for the bucket name anywhere in the URL + if "test-platform-results/" in url: + return url.split("test-platform-results/", 1)[1].rstrip("/") + + raise ValueError( + f"Cannot parse Prow URL: {url}\n" + "Expected format: https://prow.ci.openshift.org/view/gs/test-platform-results/logs//" + ) + + +def gcs_path(prefix, subpath=None): + """Build a gs:// URI.""" + base = f"gs://{BUCKET}/{prefix}" + if subpath: + subpath = subpath.strip("/") + base = f"{base}/{subpath}" + return base + + +def run_gcloud(args, timeout=60): + """Run a gcloud command and return (stdout, stderr, returncode).""" + cmd = ["gcloud"] + args + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + return result.stdout, result.stderr, result.returncode + except subprocess.TimeoutExpired: + return "", f"Command timed out after {timeout}s", 1 + + +def cmd_list(prefix, subpath=None): + """List contents of a GCS directory.""" + target = gcs_path(prefix, subpath) + if not target.endswith("/"): + target += "/" + + stdout, stderr, rc = run_gcloud( + ["storage", "ls", target], + timeout=30, + ) + + if rc != 0: + return { + "success": False, + "error": f"gcloud storage ls failed: {stderr.strip()}", + "path": target, + } + + entries = [] + for line in stdout.strip().splitlines(): + line = line.strip() + if not line: + continue + # Strip the gs://bucket/ prefix for readability + full_prefix = f"gs://{BUCKET}/{prefix}/" + relative = line.replace(full_prefix, "") if line.startswith(full_prefix) else line + is_dir = line.endswith("/") + entries.append({ + "name": relative.rstrip("/").split("/")[-1] + ("/" if is_dir else ""), + "path": relative.rstrip("/"), + "type": "directory" if is_dir else "file", + "gcs_uri": line.rstrip("/") + ("/" if is_dir else ""), + }) + + return { + "success": True, + "path": target, + "count": len(entries), + "entries": entries, + } + + +def cmd_search(prefix, pattern, subpath=None): + """Search for files matching a glob pattern under a GCS path.""" + target = gcs_path(prefix, subpath) + if not target.endswith("/"): + target += "/" + + search_pattern = f"{target}{pattern}" + + stdout, stderr, rc = run_gcloud( + ["storage", "ls", search_pattern], + timeout=120, + ) + + if rc != 0: + # gcloud returns non-zero when no matches found + if "CommandException" in stderr or "One or more URLs matched no objects" in stderr or "matched no objects" in stderr.lower(): + return { + "success": True, + "pattern": search_pattern, + "count": 0, + "matches": [], + } + return { + "success": False, + "error": f"gcloud storage ls failed: {stderr.strip()}", + "pattern": search_pattern, + } + + matches = [] + full_prefix = f"gs://{BUCKET}/{prefix}/" + for line in stdout.strip().splitlines(): + line = line.strip() + if not line: + continue + relative = line.replace(full_prefix, "") if line.startswith(full_prefix) else line + is_dir = line.endswith("/") + matches.append({ + "name": relative.rstrip("/").split("/")[-1] + ("/" if is_dir else ""), + "path": relative.rstrip("/"), + "type": "directory" if is_dir else "file", + "gcs_uri": line, + }) + + return { + "success": True, + "pattern": search_pattern, + "count": len(matches), + "matches": matches, + } + + +def cmd_fetch(prefix, filepath, max_bytes=DEFAULT_MAX_BYTES): + """Fetch contents of a specific file from GCS.""" + target = gcs_path(prefix, filepath) + + # Download to a temp file, then read + import tempfile + + with tempfile.NamedTemporaryFile(delete=False, suffix=".artifact") as tmp: + tmp_path = tmp.name + + try: + _stdout, stderr, rc = run_gcloud( + ["storage", "cp", target, tmp_path, "--no-user-output-enabled"], + timeout=60, + ) + + if rc != 0: + return { + "success": False, + "error": f"gcloud storage cp failed: {stderr.strip()}", + "path": target, + } + + file_size = os.path.getsize(tmp_path) + truncated = file_size > max_bytes + + with open(tmp_path, "r", errors="replace") as f: + content = f.read(max_bytes) + + return { + "success": True, + "path": target, + "size_bytes": file_size, + "truncated": truncated, + "max_bytes": max_bytes, + "content": content, + } + finally: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + + +def main(): + parser = argparse.ArgumentParser( + description="Search and retrieve artifacts from Prow CI job runs in GCS.", + ) + parser.add_argument( + "prow_url", + help="Prow job URL (https://prow.ci.openshift.org/view/gs/...)", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + # list + list_parser = subparsers.add_parser("list", help="List directory contents") + list_parser.add_argument( + "subpath", + nargs="?", + default=None, + help="Subdirectory path relative to the job root (optional)", + ) + + # search + search_parser = subparsers.add_parser("search", help="Search for files matching a glob pattern") + search_parser.add_argument( + "pattern", + help='Glob pattern to match (e.g., "**/*intervals*.json", "**/nodes")', + ) + search_parser.add_argument( + "subpath", + nargs="?", + default=None, + help="Subdirectory to search within (optional, searches from job root by default)", + ) + + # fetch + fetch_parser = subparsers.add_parser("fetch", help="Fetch a specific file's contents") + fetch_parser.add_argument( + "filepath", + help="Path to the file relative to the job root", + ) + fetch_parser.add_argument( + "--max-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help=f"Maximum bytes to read (default: {DEFAULT_MAX_BYTES})", + ) + + args = parser.parse_args() + + check_gcloud() + + try: + prefix = parse_prow_url(args.prow_url) + except ValueError as e: + print(json.dumps({"success": False, "error": str(e)})) + sys.exit(1) + + if args.command == "list": + result = cmd_list(prefix, args.subpath) + elif args.command == "search": + result = cmd_search(prefix, args.pattern, args.subpath) + elif args.command == "fetch": + result = cmd_fetch(prefix, args.filepath, args.max_bytes) + else: + print(json.dumps({"success": False, "error": f"Unknown command: {args.command}"})) + sys.exit(1) + + print(json.dumps(result, indent=2)) + if not result.get("success"): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/plugins/ci/skills/prow-job-extract-must-gather/CHANGELOG.md b/plugins/ci/skills/prow-job-extract-must-gather/CHANGELOG.md new file mode 100644 index 000000000..fc0c0e2c8 --- /dev/null +++ b/plugins/ci/skills/prow-job-extract-must-gather/CHANGELOG.md @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to the Prow Job Extract Must-Gather skill will be documented in this file. + +## [1.0.0] - 2025-01-17 + +### Added +- Initial release of Prow Job Extract Must-Gather skill +- Command file: `plugins/ci/commands/extract-prow-job-must-gather.md` +- Comprehensive SKILL.md with detailed implementation instructions +- `extract_archives.py` script for recursive archive extraction + - Extracts must-gather.tar to specified directory + - Renames long "-ci-" containing subdirectory to "content/" + - Recursively processes nested .tar.gz, .tgz, and .gz archives + - Removes original compressed files after extraction + - Handles up to 10 levels of nesting + - Reports extraction statistics +- `generate_html_report.py` script for HTML file browser generation + - Scans directory tree and collects file metadata + - Classifies files by type (log, yaml, json, xml, cert, archive, script, config, other) + - Generates interactive HTML with dark theme matching analyze-resource skill + - Multi-select file type filters + - Regex pattern filter for powerful file searches + - Text search for file names and paths + - Direct links to files with relative paths + - Statistics dashboard showing file counts and sizes + - Scroll to top button +- Comprehensive README.md documentation +- Working directory structure: `.work/prow-job-extract-must-gather/{build_id}/` +- Subdirectory organization: `logs/` for extracted content, `tmp/` for temporary files +- Same URL parsing logic as analyze-resource skill +- Support for caching extracted content (ask user before re-extracting) +- Error handling for corrupted archives, missing files, and invalid URLs +- Progress indicators for all long-running operations +- Platform-aware browser opening (xdg-open, open, start) + +### Features +- **Automatic Archive Extraction**: Handles all nested archive formats automatically +- **Directory Renaming**: Shortens long subdirectory names for better usability +- **Interactive File Browser**: Modern HTML interface with powerful filtering +- **Regex Pattern Matching**: Search files using full regex syntax +- **File Type Classification**: Automatic detection and categorization of file types +- **Relative File Links**: Click to open files directly from HTML browser +- **Statistics Dashboard**: Visual overview of extracted content +- **Extraction Caching**: Avoid re-extracting by reusing cached content +- **Error Recovery**: Continue processing despite individual archive failures + +### Technical Details +- Python 3 scripts using standard library (tarfile, gzip, os, pathlib) +- No external dependencies required +- Memory-efficient incremental processing +- Follows same patterns as analyze-resource skill +- Integrated with Claude Code permissions system +- Uses `.work/` directory (already in .gitignore) diff --git a/plugins/ci/skills/prow-job-extract-must-gather/README.md b/plugins/ci/skills/prow-job-extract-must-gather/README.md new file mode 100644 index 000000000..7cb730f60 --- /dev/null +++ b/plugins/ci/skills/prow-job-extract-must-gather/README.md @@ -0,0 +1,350 @@ +# Prow Job Extract Must-Gather Skill + +This skill extracts and decompresses must-gather archives from Prow CI job artifacts, automatically handling nested tar and gzip archives, and generating an interactive HTML file browser. + +## Overview + +The skill provides both a Claude Code skill interface and standalone scripts for extracting must-gather data from Prow CI jobs. It eliminates the manual steps of downloading and recursively extracting nested archives. + +## Components + +### 1. SKILL.md +Claude Code skill definition that provides detailed implementation instructions for the AI assistant. + +### 2. Python Scripts + +#### extract_archives.py +Extracts and recursively processes must-gather archives. + +**Features:** +- Extracts must-gather.tar to specified directory +- Renames long subdirectory (containing "-ci-") to "content/" for readability +- Recursively processes nested archives: + - `.tar.gz` and `.tgz`: Extract in place, remove original + - `.gz` (plain gzip): Decompress in place, remove original +- Handles up to 10 levels of nested archives +- Reports extraction statistics + +**Usage:** +```bash +python3 extract_archives.py +``` + +**Example:** +```bash +python3 plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py \ + .work/prow-job-extract-must-gather/1965715986610917376/tmp/must-gather.tar \ + .work/prow-job-extract-must-gather/1965715986610917376/logs +``` + +**Output:** +``` +================================================================================ +Must-Gather Archive Extraction +================================================================================ + +Step 1: Extracting must-gather.tar + From: .work/.../tmp/must-gather.tar + To: .work/.../logs + Extracting: .work/.../tmp/must-gather.tar + +Step 2: Renaming long directory to 'content/' + From: registry-build09-ci-openshift-org-ci-op-... + To: content/ + +Step 3: Processing nested archives + Extracting: .../content/namespaces/openshift-etcd/pods/etcd-0.tar.gz + Decompressing: .../content/cluster-scoped-resources/nodes/ip-10-0-1-234.log.gz + ... (continues for all archives) + +================================================================================ +Extraction Complete +================================================================================ + +Statistics: + Total files: 3,421 + Total size: 234.5 MB + Archives processed: 247 + +Extracted to: .work/prow-job-extract-must-gather/1965715986610917376/logs +``` + +#### generate_html_report.py +Generates an interactive HTML file browser with filters and search. + +**Features:** +- Scans directory tree and collects file metadata +- Classifies files by type (log, yaml, json, xml, cert, archive, script, config, other) +- Generates statistics (total files, total size, counts by type) +- Creates interactive HTML with: + - Multi-select file type filters + - Regex pattern filter for powerful searches + - Text search for file names/paths + - Direct links to files (relative paths) + - Same dark theme as analyze-resource skill + +**Usage:** +```bash +python3 generate_html_report.py +``` + +**Example:** +```bash +python3 plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py \ + .work/prow-job-extract-must-gather/1965715986610917376/logs \ + "periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview" \ + "1965715986610917376" \ + "e2e-aws-ovn-techpreview" \ + "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376" +``` + +**Output:** +- Creates `.work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + +## Prerequisites + +1. **Python 3** - For running extraction and report generator scripts +2. **gcloud CLI** - For downloading artifacts from GCS + - Install: https://cloud.google.com/sdk/docs/install + - Authentication NOT required (bucket is publicly accessible) + +## Workflow + +1. **URL Parsing** + - Validate URL contains `test-platform-results/` + - Extract build_id (10+ digits) + - Extract prowjob name + - Construct GCS paths + +2. **Working Directory** + - Create `.work/prow-job-extract-must-gather/{build_id}/` directory + - Create `logs/` subdirectory for extraction + - Create `tmp/` subdirectory for temporary files + - Check for existing extraction (offers to skip re-extraction) + +3. **prowjob.json Validation** + - Download prowjob.json + - Search for `--target=` pattern + - Exit if not a ci-operator job + +4. **Must-Gather Download** + - Download from: `artifacts/{target}/gather-must-gather/artifacts/must-gather.tar` + - Save to: `{build_id}/tmp/must-gather.tar` + +5. **Extraction and Processing** + - Extract must-gather.tar to `{build_id}/logs/` + - Rename long subdirectory to "content/" + - Recursively extract nested archives (.tar.gz, .tgz, .gz) + - Remove original compressed files after extraction + +6. **HTML Report Generation** + - Scan directory tree + - Classify files by type + - Calculate statistics + - Generate interactive HTML browser + - Output to `{build_id}/must-gather-browser.html` + +## Output + +### Console Output +``` +Must-Gather Extraction Complete + +Prow Job: periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview +Build ID: 1965715986610917376 +Target: e2e-aws-ovn-techpreview + +Extraction Statistics: +- Total files: 3,421 +- Total size: 234.5 MB +- Archives extracted: 247 +- Log files: 1,234 +- YAML files: 856 +- JSON files: 423 + +Extracted to: .work/prow-job-extract-must-gather/1965715986610917376/logs/ + +File browser generated: .work/prow-job-extract-must-gather/1965715986610917376/must-gather-browser.html + +Open in browser to browse and search extracted files. +``` + +### HTML File Browser + +The generated HTML report includes: + +1. **Header Section** + - Prow job name + - Build ID + - Target name + - GCS URL (link to gcsweb) + - Local extraction path + +2. **Statistics Dashboard** + - Total files count + - Total size (human-readable) + - Counts by file type (log, yaml, json, xml, cert, archive, script, config, other) + +3. **Filter Controls** + - **File Type Filter**: Multi-select buttons to filter by type + - **Regex Pattern Filter**: Input field for regex patterns (e.g., `.*etcd.*`, `.*\.log$`, `^content/namespaces/.*`) + - **Name Search**: Text search for file names and paths + +4. **File List** + - Icon for each file type + - File name (clickable link to open file) + - Directory path + - File size + - File type badge (color-coded) + - Sorted alphabetically by path + +5. **Interactive Features** + - All filters work together (AND logic) + - Real-time filtering (300ms debounce) + - Regex pattern validation + - Scroll to top button + - No results message when filters match nothing + +### Directory Structure +``` +.work/prow-job-extract-must-gather/{build_id}/ +├── tmp/ +│ ├── prowjob.json +│ └── must-gather.tar (downloaded, not deleted) +├── logs/ +│ └── content/ # Renamed from long directory +│ ├── cluster-scoped-resources/ +│ │ ├── nodes/ +│ │ ├── clusterroles/ +│ │ └── ... +│ ├── namespaces/ +│ │ ├── openshift-etcd/ +│ │ │ ├── pods/ +│ │ │ ├── services/ +│ │ │ └── ... +│ │ └── ... +│ └── ... (all extracted and decompressed) +└── must-gather-browser.html +``` + +## Performance Features + +1. **Caching** + - Extracted files are cached in `{build_id}/logs/` + - Offers to skip re-extraction if content already exists + +2. **Incremental Processing** + - Archives processed iteratively (up to 10 passes) + - Handles deeply nested archive structures + +3. **Progress Indicators** + - Colored output for different stages + - Status messages for long-running operations + - Final statistics summary + +4. **Error Handling** + - Graceful handling of corrupted archives + - Continues processing after errors + - Reports all errors in final summary + +## Examples + +### Basic Usage +```bash +# Via Claude Code +User: "Extract must-gather from https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376" + +# Standalone script +python3 plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py \ + .work/prow-job-extract-must-gather/1965715986610917376/tmp/must-gather.tar \ + .work/prow-job-extract-must-gather/1965715986610917376/logs + +python3 plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py \ + .work/prow-job-extract-must-gather/1965715986610917376/logs \ + "periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview" \ + "1965715986610917376" \ + "e2e-aws-ovn-techpreview" \ + "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376" +``` + +### Using Regex Filters in HTML Browser + +**Find all etcd-related files:** +```regex +.*etcd.* +``` + +**Find all log files:** +```regex +.*\.log$ +``` + +**Find files in specific namespace:** +```regex +^content/namespaces/openshift-etcd/.* +``` + +**Find YAML manifests for pods:** +```regex +.*pods/.*\.yaml$ +``` + +## Using with Claude Code + +When you ask Claude to extract a must-gather, it will automatically use this skill. The skill provides detailed instructions that guide Claude through: +- Validating prerequisites +- Parsing URLs +- Downloading archives +- Extracting and decompressing +- Generating HTML browser + +You can simply ask: +> "Extract must-gather from this Prow job: https://gcsweb-ci.../1965715986610917376/" + +Claude will execute the workflow and generate the interactive HTML file browser. + +## Troubleshooting + +### gcloud not installed +```bash +# Check installation +which gcloud + +# Install (follow platform-specific instructions) +# https://cloud.google.com/sdk/docs/install +``` + +### must-gather.tar not found +- Verify job completed successfully +- Check target name is correct +- Confirm gather-must-gather ran in the job +- Manually check GCS path in gcsweb + +### Corrupted archives +- Check error messages in extraction output +- Extraction continues despite individual failures +- Final summary lists all errors + +### No "-ci-" directory found +- Extraction continues with original directory names +- Check logs for warning message +- Files will still be accessible + +### HTML browser not opening files +- Verify files were extracted to `logs/` directory +- Check that relative paths are correct +- Files must be opened from the same directory as HTML file + +## File Type Classifications + +| Extension | Type | Badge Color | +|-----------|------|-------------| +| .log, .txt | log | Blue | +| .yaml, .yml | yaml | Purple | +| .json | json | Green | +| .xml | xml | Yellow | +| .crt, .pem, .key | cert | Red | +| .tar, .gz, .tgz, .zip | archive | Gray | +| .sh, .py | script | Blue | +| .conf, .cfg, .ini | config | Yellow | +| others | other | Gray | diff --git a/plugins/ci/skills/prow-job-extract-must-gather/SKILL.md b/plugins/ci/skills/prow-job-extract-must-gather/SKILL.md new file mode 100644 index 000000000..db0e6026e --- /dev/null +++ b/plugins/ci/skills/prow-job-extract-must-gather/SKILL.md @@ -0,0 +1,490 @@ +--- +name: prow-job-extract-must-gather +description: Extract and decompress must-gather archives from Prow CI job artifacts, generating an interactive HTML file browser with filters +--- + +# Prow Job Extract Must-Gather + +This skill extracts and decompresses must-gather archives from Prow CI job artifacts, automatically handling nested tar and gzip archives, and generating an interactive HTML file browser. + +## When to Use This Skill + +Use this skill when the user wants to: +- Extract must-gather archives from Prow CI job artifacts +- Avoid manually downloading and extracting nested archives +- Browse must-gather contents with an interactive HTML interface +- Search for specific files or file types in must-gather data +- Analyze OpenShift cluster state from CI test runs + +## Prerequisites + +Before starting, verify these prerequisites: + +1. **gcloud CLI Installation** + - Check if installed: `which gcloud` + - If not installed, provide instructions for the user's platform + - Installation guide: https://cloud.google.com/sdk/docs/install + +2. **gcloud Authentication (Optional)** + - The `test-platform-results` bucket is publicly accessible + - No authentication is required for read access + - Skip authentication checks + +## Input Format + +The user will provide: +1. **Prow job URL** - gcsweb URL containing `test-platform-results/` + - Example: `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376/` + - URL may or may not have trailing slash + +## Implementation Steps + +### Step 1: Parse and Validate URL + +1. **Extract bucket path** + - Find `test-platform-results/` in URL + - Extract everything after it as the GCS bucket relative path + - If not found, error: "URL must contain 'test-platform-results/'" + +2. **Extract build_id** + - Search for pattern `/(\\d{10,})/` in the bucket path + - build_id must be at least 10 consecutive decimal digits + - Handle URLs with or without trailing slash + - If not found, error: "Could not find build ID (10+ digits) in URL" + +3. **Extract prowjob name** + - Find the path segment immediately preceding build_id + - Example: In `.../periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376/` + - Prowjob name: `periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview` + +4. **Construct GCS paths** + - Bucket: `test-platform-results` + - Base GCS path: `gs://test-platform-results/{bucket-path}/` + - Ensure path ends with `/` + +### Step 2: Create Working Directory + +1. **Check for existing extraction first** + - Check if `.work/prow-job-extract-must-gather/{build_id}/logs/` directory exists and has content + - If it exists with content: + - Use AskUserQuestion tool to ask: + - Question: "Must-gather already extracted for build {build_id}. Would you like to use the existing extraction or re-extract?" + - Options: + - "Use existing" - Skip to HTML report generation (Step 6) + - "Re-extract" - Continue to clean and re-download + - If user chooses "Re-extract": + - Remove all existing content: `rm -rf .work/prow-job-extract-must-gather/{build_id}/logs/` + - Also remove tmp directory: `rm -rf .work/prow-job-extract-must-gather/{build_id}/tmp/` + - This ensures clean state before downloading new content + - If user chooses "Use existing": + - Skip directly to Step 6 (Generate HTML Report) + +2. **Create directory structure** + ```bash + mkdir -p .work/prow-job-extract-must-gather/{build_id}/logs + mkdir -p .work/prow-job-extract-must-gather/{build_id}/tmp + ``` + - Use `.work/prow-job-extract-must-gather/` as the base directory (already in .gitignore) + - Use build_id as subdirectory name + - Create `logs/` subdirectory for extraction + - Create `tmp/` subdirectory for temporary files + - Working directory: `.work/prow-job-extract-must-gather/{build_id}/` + +### Step 3: Download and Validate prowjob.json + +Use the `fetch-prowjob-json` skill to fetch the prowjob.json for this job. See `plugins/ci/skills/fetch-prowjob-json/SKILL.md` for complete implementation details. + +1. **Fetch prowjob.json** using the Prow job URL (convert to gcsweb URL per the `fetch-prowjob-json` skill) +2. **Save locally** to `.work/prow-job-extract-must-gather/{build_id}/tmp/prowjob.json` +3. **Parse and validate** + - Search for pattern: `--target=([a-zA-Z0-9-]+)` in the ci-operator args + - If not found: + - Display: "This is not a ci-operator job. The prowjob cannot be analyzed by this skill." + - Explain: ci-operator jobs have a --target argument specifying the test target + - Exit skill +4. **Extract target name** + - Capture the target value (e.g., `e2e-aws-ovn-techpreview`) + - Store for constructing must-gather path + +### Step 4: Download Must-Gather Archive + +1. **Construct must-gather path** + - GCS path: `gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-must-gather/artifacts/must-gather.tar` + - Local path: `.work/prow-job-extract-must-gather/{build_id}/tmp/must-gather.tar` + +2. **Download must-gather.tar** + ```bash + gcloud storage cp gs://test-platform-results/{bucket-path}/artifacts/{target}/gather-must-gather/artifacts/must-gather.tar .work/prow-job-extract-must-gather/{build_id}/tmp/must-gather.tar --no-user-output-enabled + ``` + - Use `--no-user-output-enabled` to suppress progress output + - If file not found, error: "No must-gather archive found. Job may not have completed or gather-must-gather may not have run." + +### Step 5: Extract and Process Archives + +**IMPORTANT: Use the provided Python script `extract_archives.py` from the skill directory.** + +**Usage:** +```bash +python3 plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py \ + .work/prow-job-extract-must-gather/{build_id}/tmp/must-gather.tar \ + .work/prow-job-extract-must-gather/{build_id}/logs +``` + +**What the script does:** + +1. **Extract must-gather.tar** + - Extract to `{build_id}/logs/` directory + - Uses Python's tarfile module for reliable extraction + +2. **Rename long subdirectory to "content/"** + - Find subdirectory containing "-ci-" in the name + - Example: `registry-build09-ci-openshift-org-ci-op-m8t77165-stable-sha256-d1ae126eed86a47fdbc8db0ad176bf078a5edebdbb0df180d73f02e5f03779e0/` + - Rename to: `content/` + - Preserves all files and subdirectories + +3. **Recursively process nested archives** + - Walk entire directory tree + - Find and process archives: + + **For .tar.gz and .tgz files:** + ```python + # Extract in place + with tarfile.open(archive_path, 'r:gz') as tar: + tar.extractall(path=parent_dir) + # Remove original archive + os.remove(archive_path) + ``` + + **For .gz files (no tar):** + ```python + # Gunzip in place + with gzip.open(gz_path, 'rb') as f_in: + with open(output_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + # Remove original archive + os.remove(gz_path) + ``` + +4. **Progress reporting** + - Print status for each extracted archive + - Count total files and archives processed + - Report final statistics + +5. **Error handling** + - Skip corrupted archives with warning + - Continue processing other files + - Report all errors at the end + +### Step 6: Generate HTML File Browser + +**IMPORTANT: Use the provided Python script `generate_html_report.py` from the skill directory.** + +**Usage:** +```bash +python3 plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py \ + .work/prow-job-extract-must-gather/{build_id}/logs \ + "{prowjob_name}" \ + "{build_id}" \ + "{target}" \ + "{gcsweb_url}" +``` + +**Output:** The script generates `.work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + +**What the script does:** + +1. **Scan directory tree** + - Recursively walk `{build_id}/logs/` directory + - Collect all files with metadata: + - Relative path from logs/ + - File size (human-readable: KB, MB, GB) + - File extension + - Directory depth + - Last modified time + +2. **Classify files** + - Detect file types based on extension: + - Logs: `.log`, `.txt` + - YAML: `.yaml`, `.yml` + - JSON: `.json` + - XML: `.xml` + - Certificates: `.crt`, `.pem`, `.key` + - Binaries: `.tar`, `.gz`, `.tgz`, `.tar.gz` + - Other + - Count files by type for statistics + +3. **Generate HTML structure** + + **Header Section:** + ```html +
+

Must-Gather File Browser

+
+

Prow Job: {prowjob-name}

+

Build ID: {build_id}

+

gcsweb URL: {original-url}

+

Target: {target}

+

Total Files: {count}

+

Total Size: {human-readable-size}

+
+
+ ``` + + **Filter Controls:** + ```html +
+
+ +
+ + + + +
+
+
+ + +
+
+ + +
+
+ ``` + + **File List:** + ```html +
+
+
{icon}
+
+
+ {filename} +
+
+ {directory-path} + {size} + {type} +
+
+
+
+ ``` + + **CSS Styling:** + - Use same dark theme as analyze-resource skill + - Modern, clean design with good contrast + - Responsive layout + - File type color coding + - Monospace fonts for paths + - Hover effects on file items + + **JavaScript Interactivity:** + ```javascript + // Multi-select file type filters + document.querySelectorAll('.filter-btn').forEach(btn => { + btn.addEventListener('click', function() { + // Toggle active state + // Apply filters + }); + }); + + // Regex pattern filter + document.getElementById('pattern').addEventListener('input', function() { + const pattern = this.value; + if (pattern) { + const regex = new RegExp(pattern); + // Filter files matching regex + } + }); + + // Name search filter + document.getElementById('search').addEventListener('input', function() { + const query = this.value.toLowerCase(); + // Filter files by name substring + }); + + // Combine all active filters + function applyFilters() { + // Show/hide files based on all active filters + } + ``` + +4. **Statistics Section:** + ```html +
+
+
{total-files}
+
Total Files
+
+
+
{total-size}
+
Total Size
+
+
+
{log-count}
+
Log Files
+
+
+
{yaml-count}
+
YAML Files
+
+ +
+ ``` + +5. **Write HTML to file** + - Script automatically writes to `.work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + - Includes proper HTML5 structure + - All CSS and JavaScript are inline for portability + +### Step 7: Present Results to User + +1. **Display summary** + ``` + Must-Gather Extraction Complete + + Prow Job: {prowjob-name} + Build ID: {build_id} + Target: {target} + + Extraction Statistics: + - Total files: {file-count} + - Total size: {human-readable-size} + - Archives extracted: {archive-count} + - Log files: {log-count} + - YAML files: {yaml-count} + - JSON files: {json-count} + + Extracted to: .work/prow-job-extract-must-gather/{build_id}/logs/ + + File browser generated: .work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html + + Open in browser to browse and search extracted files. + ``` + +2. **Open report in browser** + - Detect platform and automatically open the HTML report in the default browser + - Linux: `xdg-open .work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + - macOS: `open .work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + - Windows: `start .work/prow-job-extract-must-gather/{build_id}/must-gather-browser.html` + - On Linux (most common for this environment), use `xdg-open` + +3. **Offer next steps** + - Ask if user wants to search for specific files + - Explain that extracted files are available in `.work/prow-job-extract-must-gather/{build_id}/logs/` + - Mention that extraction is cached for faster subsequent browsing + +## Error Handling + +Handle these error scenarios with specific user-facing messages, continuing to process remaining files on non-fatal errors: + +1. **Invalid URL format** + - Error: "URL must contain 'test-platform-results/' substring" + - Provide example of valid URL + +2. **Build ID not found** + - Error: "Could not find build ID (10+ decimal digits) in URL path" + - Explain requirement and show URL parsing + +3. **gcloud not installed** + - Detect with: `which gcloud` + - Provide installation instructions for user's platform + - Link: https://cloud.google.com/sdk/docs/install + +4. **prowjob.json not found** + - Suggest verifying URL and checking if job completed + - Provide gcsweb URL for manual verification + +5. **Not a ci-operator job** + - Error: "This is not a ci-operator job. No --target found in prowjob.json." + - Explain: Only ci-operator jobs can be analyzed by this skill + +6. **must-gather.tar not found** + - Warn: "Must-gather archive not found at expected path" + - Suggest: Job may not have completed or gather-must-gather may not have run + - Provide full GCS path that was checked + +7. **Corrupted archive** + - Warn: "Could not extract {archive-path}: {error}" + - Continue processing other archives + - Report all errors in final summary + +8. **No "-ci-" subdirectory found** + - Warn: "Could not find expected subdirectory to rename to 'content/'" + - Continue with extraction anyway + - Files will be in original directory structure + +## Performance Considerations + +1. **Avoid re-extracting** + - Check if `.work/prow-job-extract-must-gather/{build_id}/logs/` already has content + - Ask user before re-extracting + +2. **Efficient downloads** + - Use `gcloud storage cp` with `--no-user-output-enabled` to suppress verbose output + +3. **Memory efficiency** + - Process archives incrementally + - Don't load entire files into memory + - Use streaming extraction + +4. **Progress indicators** + - Show "Downloading must-gather archive..." before gcloud command + - Show "Extracting must-gather.tar..." before extraction + - Show "Processing nested archives..." during recursive extraction + - Show "Generating HTML file browser..." before report generation + +## Examples + +### Example 1: Extract must-gather from periodic job +``` +User: "Extract must-gather from this Prow job: https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn-techpreview/1965715986610917376" + +Output: +- Downloads must-gather.tar to: .work/prow-job-extract-must-gather/1965715986610917376/tmp/ +- Extracts to: .work/prow-job-extract-must-gather/1965715986610917376/logs/ +- Renames long subdirectory to: content/ +- Processes 247 nested archives (.tar.gz, .tgz, .gz) +- Creates: .work/prow-job-extract-must-gather/1965715986610917376/must-gather-browser.html +- Opens browser with interactive file list (3,421 files, 234 MB) +``` + +## Tips + +- Always verify gcloud prerequisites before starting (gcloud CLI must be installed) +- Authentication is NOT required - the bucket is publicly accessible +- Use `.work/prow-job-extract-must-gather/{build_id}/` directory structure for organization +- All work files are in `.work/` which is already in .gitignore +- The Python scripts handle all extraction and HTML generation - use them! +- Cache extracted files in `.work/prow-job-extract-must-gather/{build_id}/` to avoid re-extraction +- The HTML file browser supports regex patterns for powerful file filtering +- Extracted files can be opened directly from the HTML browser (links are relative) + +## Important Notes + +1. **Archive Processing:** + - The script automatically handles nested archives + - Original compressed files are removed after successful extraction + - Corrupted archives are skipped with warnings + +2. **Directory Renaming:** + - The long subdirectory name (containing "-ci-") is renamed to "content/" for brevity + - Files within "content/" are NOT altered + - This makes paths more readable in the HTML browser + +3. **File Type Detection:** + - File types are detected based on extension + - Common types are color-coded in the HTML browser + - All file types can be filtered + +4. **Regex Pattern Filtering:** + - Users can enter regex patterns in the filter input + - Patterns match against full file paths + - Invalid regex patterns are caught and silently skipped without filtering + +5. **Working with Scripts:** + - All scripts are in `plugins/ci/skills/prow-job-extract-must-gather/` + - `extract_archives.py` - Extracts and processes archives + - `generate_html_report.py` - Generates interactive HTML file browser diff --git a/plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py b/plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py new file mode 100755 index 000000000..4688694b3 --- /dev/null +++ b/plugins/ci/skills/prow-job-extract-must-gather/extract_archives.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Extract and recursively decompress must-gather archives.""" + +import os +import sys +import tarfile +import gzip +import shutil +from pathlib import Path + + +def human_readable_size(size_bytes): + """Convert bytes to human-readable format.""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size_bytes < 1024.0: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024.0 + return f"{size_bytes:.1f} PB" + + +def extract_tar_archive(tar_path, extract_to): + """Extract a tar archive (including .tar.gz and .tgz).""" + try: + print(f" Extracting: {tar_path}") + with tarfile.open(tar_path, 'r:*') as tar: + tar.extractall(path=extract_to) + return True + except Exception as e: + print(f" ERROR: Failed to extract {tar_path}: {e}", file=sys.stderr) + return False + + +def gunzip_file(gz_path): + """Gunzip a .gz file (not a tar.gz).""" + try: + # Output file is the same name without .gz extension + output_path = gz_path[:-3] if gz_path.endswith('.gz') else gz_path + '.decompressed' + + print(f" Decompressing: {gz_path}") + with gzip.open(gz_path, 'rb') as f_in: + with open(output_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + return True, output_path + except Exception as e: + print(f" ERROR: Failed to decompress {gz_path}: {e}", file=sys.stderr) + return False, None + + +def find_and_rename_ci_directory(base_path): + """Find directory containing '-ci-' and rename it to 'content'.""" + try: + for item in os.listdir(base_path): + item_path = os.path.join(base_path, item) + if os.path.isdir(item_path) and '-ci-' in item: + content_path = os.path.join(base_path, 'content') + print(f"\nRenaming directory:") + print(f" From: {item}") + print(f" To: content/") + os.rename(item_path, content_path) + return True + print("\nWARNING: No directory containing '-ci-' found to rename", file=sys.stderr) + return False + except Exception as e: + print(f"ERROR: Failed to rename directory: {e}", file=sys.stderr) + return False + + +def process_nested_archives(base_path): + """Recursively find and extract nested archives.""" + archives_processed = 0 + errors = [] + + print("\nProcessing nested archives...") + + # Keep processing until no more archives are found + # (since extracting one archive might create new archives) + max_iterations = 10 + iteration = 0 + + while iteration < max_iterations: + iteration += 1 + found_archives = False + + # Walk directory tree + for root, dirs, files in os.walk(base_path): + for filename in files: + file_path = os.path.join(root, filename) + processed = False + + # Handle .tar.gz and .tgz files + if filename.endswith('.tar.gz') or filename.endswith('.tgz'): + parent_dir = os.path.dirname(file_path) + if extract_tar_archive(file_path, parent_dir): + os.remove(file_path) + archives_processed += 1 + processed = True + found_archives = True + else: + errors.append(f"Failed to extract: {file_path}") + + # Handle plain .gz files (not .tar.gz) + elif filename.endswith('.gz') and not filename.endswith('.tar.gz'): + success, output_path = gunzip_file(file_path) + if success: + os.remove(file_path) + archives_processed += 1 + processed = True + found_archives = True + else: + errors.append(f"Failed to decompress: {file_path}") + + # If no archives were found in this iteration, we're done + if not found_archives: + break + + if iteration >= max_iterations: + print(f"\nWARNING: Stopped after {max_iterations} iterations. Some nested archives may remain.", file=sys.stderr) + + return archives_processed, errors + + +def count_files_and_size(base_path): + """Count total files and calculate total size.""" + total_files = 0 + total_size = 0 + + for root, dirs, files in os.walk(base_path): + for filename in files: + file_path = os.path.join(root, filename) + try: + total_files += 1 + total_size += os.path.getsize(file_path) + except: + pass + + return total_files, total_size + + +def main(): + if len(sys.argv) != 3: + print("Usage: extract_archives.py ") + print(" : Path to the must-gather.tar file") + print(" : Directory to extract to") + sys.exit(1) + + tar_file = sys.argv[1] + output_dir = sys.argv[2] + + # Validate inputs + if not os.path.exists(tar_file): + print(f"ERROR: Input file not found: {tar_file}", file=sys.stderr) + sys.exit(1) + + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + print("=" * 80) + print("Must-Gather Archive Extraction") + print("=" * 80) + + # Step 1: Extract main tar file + print(f"\nStep 1: Extracting must-gather.tar") + print(f" From: {tar_file}") + print(f" To: {output_dir}") + + if not extract_tar_archive(tar_file, output_dir): + print("ERROR: Failed to extract must-gather.tar", file=sys.stderr) + sys.exit(1) + + # Step 2: Rename directory containing '-ci-' to 'content' + print(f"\nStep 2: Renaming long directory to 'content/'") + find_and_rename_ci_directory(output_dir) + + # Step 3: Process nested archives + print(f"\nStep 3: Processing nested archives") + archives_processed, errors = process_nested_archives(output_dir) + + # Final statistics + print("\n" + "=" * 80) + print("Extraction Complete") + print("=" * 80) + + total_files, total_size = count_files_and_size(output_dir) + + print(f"\nStatistics:") + print(f" Total files: {total_files:,}") + print(f" Total size: {human_readable_size(total_size)}") + print(f" Archives processed: {archives_processed}") + + if errors: + print(f"\nErrors encountered: {len(errors)}") + for error in errors[:10]: # Show first 10 errors + print(f" - {error}") + if len(errors) > 10: + print(f" ... and {len(errors) - 10} more errors") + + print(f"\nExtracted to: {output_dir}") + print("") + + +if __name__ == '__main__': + main() diff --git a/plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py b/plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py new file mode 100755 index 000000000..49f0a4078 --- /dev/null +++ b/plugins/ci/skills/prow-job-extract-must-gather/generate_html_report.py @@ -0,0 +1,1289 @@ +#!/usr/bin/env python3 +"""Generate interactive HTML file browser for must-gather extraction.""" + +import os +import sys +import json +import hashlib +import html as html_module +from datetime import datetime +from pathlib import Path + + +def human_readable_size(size_bytes): + """Convert bytes to human-readable format.""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size_bytes < 1024.0: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024.0 + return f"{size_bytes:.1f} PB" + + +def get_file_type(filename): + """Determine file type based on extension.""" + ext = filename.lower().split('.')[-1] if '.' in filename else '' + + type_map = { + 'log': 'log', + 'txt': 'log', + 'yaml': 'yaml', + 'yml': 'yaml', + 'json': 'json', + 'xml': 'xml', + 'crt': 'cert', + 'pem': 'cert', + 'key': 'cert', + 'tar': 'archive', + 'gz': 'archive', + 'tgz': 'archive', + 'zip': 'archive', + 'sh': 'script', + 'py': 'script', + 'conf': 'config', + 'cfg': 'config', + 'ini': 'config', + } + + return type_map.get(ext, 'other') + + +def get_file_icon(file_type): + """Get icon character for file type.""" + icons = { + 'log': '📄', + 'yaml': '📋', + 'json': '{ }', + 'xml': '', + 'cert': '🔐', + 'archive': '📦', + 'script': '⚙️', + 'config': '⚙️', + 'other': '📄', + } + return icons.get(file_type, '📄') + + +def scan_directory(base_path): + """Scan directory and collect file information.""" + files = [] + type_counts = {} + dir_counts = {} + total_size = 0 + + for root, dirs, filenames in os.walk(base_path): + # Skip the _links directory + if '_links' in dirs: + dirs.remove('_links') + + for filename in filenames: + file_path = os.path.join(root, filename) + try: + # Get relative path from base_path + rel_path = os.path.relpath(file_path, base_path) + + # Get file info + stat_info = os.stat(file_path) + size = stat_info.st_size + total_size += size + + # Determine file type + file_type = get_file_type(filename) + type_counts[file_type] = type_counts.get(file_type, 0) + 1 + + # Get directory path (everything except filename) + dir_path = os.path.dirname(rel_path) + + # Get top-level directory (first segment after content/) + top_level_dir = '' + if dir_path.startswith('content/'): + path_parts = dir_path.split('/', 2) + if len(path_parts) >= 2: + top_level_dir = path_parts[1] + dir_counts[top_level_dir] = dir_counts.get(top_level_dir, 0) + 1 + elif '/' in dir_path: + # If not under content/, use first directory + top_level_dir = dir_path.split('/', 1)[0] + dir_counts[top_level_dir] = dir_counts.get(top_level_dir, 0) + 1 + + files.append({ + 'name': filename, + 'path': rel_path, + 'dir': dir_path, + 'top_level_dir': top_level_dir, + 'size': size, + 'size_human': human_readable_size(size), + 'type': file_type, + 'modified': datetime.fromtimestamp(stat_info.st_mtime).isoformat(), + }) + except Exception as e: + print(f"WARNING: Could not process {file_path}: {e}", file=sys.stderr) + + # Sort files by path + files.sort(key=lambda f: f['path']) + + return files, type_counts, dir_counts, total_size + + +def generate_html_report(files, type_counts, dir_counts, total_size, prowjob_name, build_id, target, gcsweb_url): + """Generate an interactive HTML file browser.""" + + total_files = len(files) + total_size_human = human_readable_size(total_size) + + html = f''' + + + + + Must-Gather Browser: {build_id} + + + + + +
+
+
File Viewer
+
+ Height: + + +
+
+ +
+ +
+

Must-Gather File Browser

+
+
+

Prow Job: {prowjob_name}

+

Build ID: {build_id}

+

Target: {target}

+
+
+

GCS URL: View in gcsweb

+

Local Path: .work/prow-job-extract-must-gather/{build_id}/logs/

+
+
+
+
+
{total_files:,}
+
Total Files
+
+
+
{total_size_human}
+
Total Size
+
''' + + # Add stats for each file type + for file_type in sorted(type_counts.keys()): + count = type_counts[file_type] + html += f''' +
+
{count:,}
+
{file_type}
+
''' + + html += ''' +
+
+ +
+
+ +
''' + + # Add filter buttons for each type + for file_type in sorted(type_counts.keys()): + count = type_counts[file_type] + html += f''' + ''' + + html += f''' +
+
+
+ +
''' + + # Add filter buttons for each top-level directory + for directory in sorted(dir_counts.keys()): + count = dir_counts[directory] + # Display name with proper formatting + display_name = directory if directory else '(root)' + html += f''' + ''' + + html += f''' +
+
+
+ + +
Invalid regex pattern
+
+
+ + +
+
+ +
+
''' + + # Add file items + for file in files: + icon = get_file_icon(file['type']) + # Use symlink path for iframe if available, otherwise use original + iframe_path = file.get('symlink_path', f"logs/{file['path']}") + original_path = f"logs/{file['path']}" + + html += f''' +
+
{icon}
+
+
+ {file['name']} + + + + + +
+
+ {file['dir']} + {file['size_human']} + {file['type']} +
+
+
''' + + html += f''' +
+ +
+ + + +''' + + return html + + +def create_txt_symlinks(logs_dir, files): + """Create .html files with escaped content for files under 1MB to prevent download dialogs.""" + MAX_INLINE_SIZE = 1 * 1024 * 1024 # 1MB + links_dir = os.path.join(logs_dir, 'content', '_links') + + # Create _links directory if it doesn't exist + os.makedirs(links_dir, exist_ok=True) + + html_count = 0 + + for file in files: + if file['size'] < MAX_INLINE_SIZE: + # Create HTML file with escaped content + original_path = os.path.join(logs_dir, file['path']) + + # Generate unique HTML name by hashing the full path + path_hash = hashlib.md5(file['path'].encode()).hexdigest()[:8] + html_name = f"{file['name']}.{path_hash}.html" + html_path = os.path.join(links_dir, html_name) + + try: + # Read original file content + with open(original_path, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + + # Split into lines and add line numbers + lines = content.split('\n') + line_count = len(lines) + line_number_width = len(str(line_count)) + + # Build content with line numbers + numbered_lines = [] + for i, line in enumerate(lines, 1): + escaped_line = html_module.escape(line) + line_num = str(i).rjust(line_number_width) + numbered_lines.append(f'{line_num} {escaped_line}') + + numbered_content = '\n'.join(numbered_lines) + + # Wrap in HTML + html_content = f''' + + + + {html_module.escape(file['name'])} + + + +
+
+ + +
+
Invalid regex pattern
+
+
+ 
{numbered_content}
+
+ + +''' + + # Write HTML file + with open(html_path, 'w', encoding='utf-8') as f: + f.write(html_content) + + # Store HTML path in file metadata + file['symlink_path'] = f"logs/content/_links/{html_name}" + html_count += 1 + except Exception as e: + print(f"WARNING: Could not create HTML for {file['path']}: {e}", file=sys.stderr) + file['symlink_path'] = None + else: + file['symlink_path'] = None + + print(f"Created {html_count:,} .html files for inline viewing") + return files + + +def main(): + if len(sys.argv) < 6: + print("Usage: generate_html_report.py ") + sys.exit(1) + + logs_dir = sys.argv[1] + prowjob_name = sys.argv[2] + build_id = sys.argv[3] + target = sys.argv[4] + gcsweb_url = sys.argv[5] + + # Validate logs directory + if not os.path.exists(logs_dir): + print(f"ERROR: Logs directory not found: {logs_dir}", file=sys.stderr) + sys.exit(1) + + print("Scanning directory tree...") + files, type_counts, dir_counts, total_size = scan_directory(logs_dir) + + print(f"Found {len(files):,} files ({human_readable_size(total_size)})") + + print("Creating .html files for inline viewing...") + files = create_txt_symlinks(logs_dir, files) + + print("Generating HTML report...") + html = generate_html_report(files, type_counts, dir_counts, total_size, prowjob_name, build_id, target, gcsweb_url) + + # Determine output path + output_dir = os.path.dirname(logs_dir) + output_file = os.path.join(output_dir, 'must-gather-browser.html') + + # Write to file + with open(output_file, 'w') as f: + f.write(html) + + print(f"Report generated: {output_file}") + + +if __name__ == '__main__': + main() diff --git a/plugins/ci/skills/reevaluate-job-runs/SKILL.md b/plugins/ci/skills/reevaluate-job-runs/SKILL.md deleted file mode 100644 index 4da720b55..000000000 --- a/plugins/ci/skills/reevaluate-job-runs/SKILL.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -name: reevaluate-job-runs -description: Retroactively re-run Sippy Symptom detection on completed Prow CI job runs to apply or preview failure Labels ---- - -# Reevaluate Job Runs - -Sippy Symptoms are known-failure signatures for OpenShift CI. A symptom is a rule made of a file pattern (a glob over a CI job run's artifact files, e.g. `**/build-log.txt`) and a matcher (`string` = substring, `regex` = regular expression, `none` = file merely exists, `cel` = a compound CEL expression over other label names). When a symptom matches a job run's artifacts, Sippy applies one or more **Labels** — human-readable tags like `InfraFailure` — to that run. Labels appear in the Sippy UI and Spyglass and help everyone quickly recognize known failure modes without re-debugging them. You do not need any prior Sippy knowledge to use this skill. - -Symptom detection normally runs automatically as job artifacts arrive. Reevaluation is for runs that completed **before** a symptom was created or changed — it asks Sippy to re-scan those runs server-side and apply the current symptom set. - -## When to Use This Skill - -Use this skill when you need to: - -- Apply a newly created or updated symptom to job runs that finished before the change (see `manage-symptoms`) -- Preview (`--dry-run`) which symptoms would match a run without writing anything -- Re-scan every job run behind a triage or regression so known-failure labels appear on them - -## Prerequisites - -1. **OpenShift CLI Authentication**: Required for authenticating to the sippy-auth API - - Must be logged into the DPCR cluster via `oc login` - - Cluster API: `https://api.cr.j7t7.p1.openshiftapps.com:6443` - - Use the `oc-auth` skill to obtain the Bearer token - -2. **Python 3**: Python 3.6 or later - - Check: `python3 --version` - - Uses only standard library (no external dependencies) - -## Implementation Steps - -### Step 1: Obtain Authentication Token - -Use the `oc-auth` skill to obtain a Bearer token from the DPCR cluster: - -```bash -# Get token from the DPCR cluster context -# The oc-auth skill's curl_with_token.sh uses this cluster for sippy-auth -DPCR_CLUSTER="https://api.cr.j7t7.p1.openshiftapps.com:6443" - -# Find the oc context for the DPCR cluster and get the token -CONTEXT=$(oc config get-contexts -o name 2>/dev/null | while read -r ctx; do - server=$(oc config view -o jsonpath="{.clusters[?(@.name=='$(oc config view -o jsonpath="{.contexts[?(@.name=='$ctx')].context.cluster}" 2>/dev/null)')].cluster.server}" 2>/dev/null || echo "") - server_clean=$(echo "$server" | sed -E 's|^https?://||') - if [ "$server_clean" = "api.cr.j7t7.p1.openshiftapps.com:6443" ]; then - echo "$ctx" - break - fi -done) - -if [ -z "$CONTEXT" ]; then - echo "Error: Not logged into DPCR cluster. Please run: oc login $DPCR_CLUSTER" - exit 1 -fi - -export SIPPY_TOKEN=$(oc whoami -t --context="$CONTEXT" 2>/dev/null) -if [ -z "$SIPPY_TOKEN" ]; then - echo "Error: Failed to get token. Please re-authenticate to DPCR cluster." - exit 1 -fi -``` - -Prefer exporting `SIPPY_TOKEN` as above rather than passing `--token` on the command line — command-line arguments are visible in process listings. `--token` still works and takes precedence over the environment variable. - -### Step 2: Dry-run First - -Always suggest a `--dry-run` first — it reports what would match without writing anything. Pass numeric build IDs or full Prow job URLs (any count — the script deduplicates and batches automatically): - -```bash -python3 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py \ - https://prow.ci.openshift.org/view/gs/test-platform-results/logs// --dry-run --format summary -``` - -### Step 3: Apply - -Rerun without `--dry-run` to actually write labels: - -```bash -python3 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py \ - 1856789012345678848 1856789012345678849 --format summary -``` - -### Bulk workflow: reevaluate all runs behind a triage - -Sippy has no triage-level reevaluate endpoint. To "reevaluate symptoms on a triage", collect the `prowjob_run_id` of every job run from each regression in the triage using the `fetch-regression-details` skill, then pass them all to this script: - -```bash -# For each regression ID in the triage, collect its job run IDs -RUN_IDS="" -for REG_ID in 12345 12346 12347; do - IDS=$(python3 plugins/ci/skills/fetch-regression-details/fetch_regression_details.py "$REG_ID" \ - | jq -r '.job_runs[].prowjob_run_id') - RUN_IDS="$RUN_IDS $IDS" -done - -# The script deduplicates and batches (default 10 per request) automatically -python3 plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py \ - $RUN_IDS --format summary -``` - -**Arguments**: -- `runs`: One or more Prow build IDs or Prow job URLs (positional, required; batched automatically) - -**Options**: -- `--token `: Bearer token from the oc-auth skill (optional if the `SIPPY_TOKEN` environment variable is set, which is preferred — argv is visible in process listings; `--token` takes precedence) -- `--dry-run`: Report matches without writing anything -- `--batch-size `: Runs per API request (default 10; max 50, but large batches risk 504 gateway timeouts) -- `--format json|summary`: Output format (default: json) - -## API Details - -**Endpoint**: `POST https://sippy-auth.dptools.openshift.org/api/jobs/runs/reevaluate` - -**Request**: - -```json -{"prow_job_build_ids": ["1856789012345678848"], "dry_run": false} -``` - -The API accepts a maximum of 50 build IDs per request. - -**Response**: `results[]` with per-run fields: - -| Field | Description | -|-------|-------------| -| `status` | `success`, `missing_error` (run artifacts not found), `eval_error`, or `rewrite_error` | -| `symptoms_evaluated` | Number of symptoms checked against the run | -| `symptoms_matched` | Number of symptoms that matched | -| `labels_applied` | Label IDs applied to the run | -| `bq_entries_written` | BigQuery rows written | -| `gcs_artifacts_written` | GCS label artifacts written | -| `postgres_updated` | Whether the Postgres record was updated | - -**Authentication**: `Authorization: Bearer ` from the DPCR cluster. - -Reevaluation is delete-then-insert and **idempotent** — running it twice on the same run is safe. Manually-applied labels (those with an empty `symptom_id`) are preserved. - -## Batching & timeouts (field-tested 2026-07) - -- The server evaluates roughly **3-4 seconds per run**, and the fronting gateway times out around **60-90 seconds**, returning an HTML `504 Gateway Time-out` **page** (not JSON). This means 50-run batches reliably fail even though the API nominally accepts them. -- The script therefore defaults to **batches of 10**, with **3 attempts per batch (2 retries)** and a 5-second backoff. Transient gateway errors (HTTP 502/503/504, HTML error pages, and non-JSON response bodies) are all retried. Retries are safe because reevaluation is idempotent — even a batch that partially completed server-side can be resent. -- If 504s persist, lower `--batch-size` (e.g. `--batch-size 5`). -- **Warning:** an HTML **login page** response means the token expired — the SSO proxy redirects to login instead of returning 401. The script detects this and tells you to refresh the token via the `oc-auth` skill. - -## Error Handling - -- **Invalid/non-numeric IDs**: Caught client-side before any request (exit 1) — pass a numeric build ID or a Prow URL ending in one (query strings and `#fragments` are stripped automatically). -- **Invalid `--batch-size`**: Must be between 1 and 50 (exit 1). -- **Transient gateway errors (502/503/504, HTML error pages, non-JSON bodies)**: Retried automatically (3 attempts, i.e. 2 retries, 5s backoff); persistent failures are reported in `failed_batches` and the script exits 1 — rerun with just those IDs (idempotent, safe). -- **Authentication failure (HTML login page or 401/403)**: Token missing/expired — the script **stops immediately** and marks all remaining batches as `not attempted` in `failed_batches` instead of hammering the API with a bad token. Refresh the token via the `oc-auth` skill and rerun. -- **`missing_error` status**: The run's artifacts were not found — check the build ID. -- **501**: You hit the read-only Sippy instance; make sure the sippy-auth base URL is used (the script already does). - -**Exit Codes**: -- `0`: All batches succeeded -- `1`: Validation error, or one or more batches failed (see `failed_batches` in JSON output) - -## See Also - -- Related Skill: `oc-auth` (provides authentication tokens for sippy-auth) -- Related Skill: `manage-symptoms` (create/update the symptoms you then apply retroactively) -- Related Skill: `diagnose-job-run-symptoms` (explain which symptoms/labels apply to a run) -- Related Skill: `fetch-regression-details` (source of `.job_runs[].prowjob_run_id` values for triage-wide reevaluation) -- Related Skill: `fetch-prow-job-runs` (discover run IDs by job name, variant, result, or time window) diff --git a/plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py b/plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py deleted file mode 100644 index 90ec51d0a..000000000 --- a/plugins/ci/skills/reevaluate-job-runs/reevaluate_job_runs.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Re-run Sippy symptom detection on completed Prow job runs (requires auth token). - -Field lesson (2026-07): the API accepts up to 50 build IDs per request, but the -server evaluates roughly 3-4 seconds per run and the fronting gateway times out -around 60-90s, returning an HTML "504 Gateway Time-out" page. Batches of ~10 -are reliable. Reevaluation is delete-then-insert and idempotent, so retrying a -batch (even one that may have partially completed server-side) is safe. -""" -import argparse -import json -import os -import sys -import time -import urllib.error -import urllib.request - -URL = "https://sippy-auth.dptools.openshift.org/api/jobs/runs/reevaluate" -API_MAX_IDS = 50 -DEFAULT_BATCH_SIZE = 10 -RETRIES_PER_BATCH = 3 -RETRY_DELAY_SECONDS = 5 - - -def resolve_token(arg_token, env=None): - """Return the Bearer token from --token or the SIPPY_TOKEN env var. - - --token takes precedence over the environment variable. Prefer the env - var: command-line arguments are visible in process listings. - """ - env = os.environ if env is None else env - return arg_token or env.get("SIPPY_TOKEN") or None - - -def extract_build_id(value): - value = value.strip().split("#", 1)[0].split("?", 1)[0].rstrip("/") - candidate = value.rsplit("/", 1)[-1] - if candidate.isdigit(): - return candidate - raise ValueError("cannot extract a numeric build ID from %r " - "(pass a numeric prow build ID or a Prow job URL ending in one)" % value) - - -def chunk(items, size): - return [items[i:i + size] for i in range(0, len(items), size)] - - -def send_batch(ids, token, dry_run): - """POST one batch. Returns (results, error, auth_failed). - - Retries on transient gateway errors (502/503/504, HTML error pages, non-JSON - bodies). auth_failed=True signals the caller to stop sending further batches. - """ - payload = {"prow_job_build_ids": ids, "dry_run": dry_run} - last_err = None - for attempt in range(1, RETRIES_PER_BATCH + 1): - req = urllib.request.Request( - URL, data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json", "Authorization": "Bearer %s" % token}, - method="POST") - try: - with urllib.request.urlopen(req, timeout=300) as resp: - body = resp.read().decode("utf-8") - if body.lstrip().startswith("<"): - # HTML instead of JSON: SSO login page (bad token) or gateway error page - if "log in" in body.lower(): - return None, ("got an SSO login page instead of JSON — token is " - "missing/expired; use the oc-auth skill to refresh it"), True - last_err = "gateway returned an HTML error page (likely 504 timeout)" - else: - try: - return json.loads(body).get("results", []), None, False - except ValueError: - last_err = "server returned a non-JSON response body" - except urllib.error.HTTPError as e: - detail = "" - try: - detail = e.read().decode("utf-8") - except Exception: - pass - if e.code == 501: - return None, "HTTP 501 (write endpoints disabled; use sippy-auth)", False - if e.code in (401, 403): - return None, "HTTP %d (token missing/expired; use the oc-auth skill)" % e.code, True - if e.code not in (502, 503, 504): - return None, "HTTP %d: %s\n%s" % (e.code, e.reason, detail), False - last_err = "HTTP %d gateway error" % e.code - except urllib.error.URLError as e: - last_err = "connection error: %s" % e.reason - if attempt < RETRIES_PER_BATCH: - print("Batch attempt %d/%d failed (%s); retrying in %ds (reevaluation is " - "idempotent, retries are safe)..." % (attempt, RETRIES_PER_BATCH, - last_err, RETRY_DELAY_SECONDS), - file=sys.stderr) - time.sleep(RETRY_DELAY_SECONDS) - return None, "%s after %d attempts (try a smaller --batch-size)" % (last_err, RETRIES_PER_BATCH), False - - -def main(): - p = argparse.ArgumentParser(description="Reevaluate symptoms on Prow job runs") - p.add_argument("runs", nargs="+", help="Prow build IDs or Prow job URLs (any count; batched automatically)") - p.add_argument("--token", help="Bearer token (or set SIPPY_TOKEN env var, preferred; use oc-auth skill)") - p.add_argument("--dry-run", action="store_true", help="Report matches without writing anything") - p.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, - help="Runs per API request (default %d; max %d, but large batches " - "risk 504 gateway timeouts)" % (DEFAULT_BATCH_SIZE, API_MAX_IDS)) - p.add_argument("--format", choices=["json", "summary"], default="json") - args = p.parse_args() - - token = resolve_token(args.token) - if not token: - print("Error: no token provided — pass --token or set the SIPPY_TOKEN " - "environment variable (preferred; use the oc-auth skill to obtain " - "one)", file=sys.stderr) - return 1 - - if not 1 <= args.batch_size <= API_MAX_IDS: - print("Error: --batch-size must be between 1 and %d" % API_MAX_IDS, file=sys.stderr) - return 1 - try: - ids = [extract_build_id(r) for r in args.runs] - except ValueError as e: - print("Error: %s" % e, file=sys.stderr) - return 1 - ids = sorted(set(ids)) - - all_results = [] - failed_batches = [] - batches = chunk(ids, args.batch_size) - for i, batch in enumerate(batches, 1): - if len(batches) > 1: - print("Batch %d/%d (%d runs)..." % (i, len(batches), len(batch)), file=sys.stderr) - results, err, auth_failed = send_batch(batch, token, args.dry_run) - if err: - print("Error: batch %d failed: %s" % (i, err), file=sys.stderr) - failed_batches.append({"batch": i, "ids": batch, "error": err}) - if auth_failed: - auth_err = "not attempted: %s" % err - for j, remaining in enumerate(batches[i:], i + 1): - failed_batches.append({"batch": j, "ids": remaining, "error": auth_err}) - print("Error: authentication failed; skipping remaining batches. " - "Refresh the token via the oc-auth skill and rerun.", file=sys.stderr) - break - else: - all_results.extend(results) - - if args.format == "json": - print(json.dumps({"results": all_results, "failed_batches": failed_batches}, indent=2)) - else: - mode = "DRY RUN" if args.dry_run else "APPLIED" - print("Reevaluation (%s) — %d runs processed, %d batches failed" % - (mode, len(all_results), len(failed_batches))) - print("=" * 60) - for r in all_results: - print("Run %s: %s" % (r.get("prow_job_build_id", "?"), r.get("status"))) - print(" Symptoms evaluated: %s, matched: %s" % - (r.get("symptoms_evaluated"), r.get("symptoms_matched"))) - labels = r.get("labels_applied") or [] - print(" Labels applied: %s" % (", ".join(map(str, labels)) if labels else "none")) - for fb in failed_batches: - print("FAILED batch %d (%d runs): %s" % (fb["batch"], len(fb["ids"]), fb["error"])) - return 1 if failed_batches else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/ci/skills/reevaluate-job-runs/test_reevaluate_job_runs.py b/plugins/ci/skills/reevaluate-job-runs/test_reevaluate_job_runs.py deleted file mode 100644 index dac8cf84d..000000000 --- a/plugins/ci/skills/reevaluate-job-runs/test_reevaluate_job_runs.py +++ /dev/null @@ -1,112 +0,0 @@ -import json - -import pytest -import reevaluate_job_runs -from reevaluate_job_runs import extract_build_id, chunk, send_batch - -def test_plain_numeric_id(): - assert extract_build_id("1856789012345678848") == "1856789012345678848" - -def test_prow_url(): - url = ("https://prow.ci.openshift.org/view/gs/test-platform-results/logs/" - "periodic-ci-openshift-release-master-ci-4.20-e2e-aws-ovn/1856789012345678848") - assert extract_build_id(url) == "1856789012345678848" - -def test_prow_url_trailing_slash(): - assert extract_build_id("https://prow.ci.openshift.org/view/gs/b/logs/job/123456/") == "123456" - -def test_invalid_input_raises(): - with pytest.raises(ValueError): - extract_build_id("not-a-build-id") - -def test_chunk_splits_evenly(): - assert chunk([1, 2, 3, 4], 2) == [[1, 2], [3, 4]] - -def test_chunk_last_partial(): - assert chunk([1, 2, 3], 2) == [[1, 2], [3]] - -def test_chunk_smaller_than_size(): - assert chunk([1], 10) == [[1]] - -def test_url_with_query_string(): - assert extract_build_id( - "https://prow.ci.openshift.org/view/gs/b/logs/job/123456?tab=x") == "123456" - -def test_url_with_fragment(): - assert extract_build_id( - "https://prow.ci.openshift.org/view/gs/b/logs/job/123456#fragment") == "123456" - - -class FakeResponse: - def __init__(self, body): - self._body = body.encode("utf-8") - - def read(self): - return self._body - - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - -def _patch_urlopen(monkeypatch, bodies): - """urlopen returns successive bodies from the list.""" - calls = [] - - def fake_urlopen(req, timeout=None): - calls.append(req) - return FakeResponse(bodies[len(calls) - 1]) - - monkeypatch.setattr(reevaluate_job_runs.urllib.request, "urlopen", fake_urlopen) - monkeypatch.setattr(reevaluate_job_runs.time, "sleep", lambda s: None) - return calls - - -def test_send_batch_json_success(monkeypatch): - body = json.dumps({"results": [{"prow_job_build_id": "1", "status": "success"}]}) - calls = _patch_urlopen(monkeypatch, [body]) - results, err, auth_failed = send_batch(["1"], "tok", True) - assert err is None - assert auth_failed is False - assert results == [{"prow_job_build_id": "1", "status": "success"}] - assert len(calls) == 1 - - -def test_send_batch_sso_login_page_no_retries(monkeypatch): - calls = _patch_urlopen(monkeypatch, ["Log in to your account"] * 3) - results, err, auth_failed = send_batch(["1"], "tok", True) - assert results is None - assert auth_failed is True - assert "token" in err - assert len(calls) == 1 # no retries on auth failure - - -def test_send_batch_504_html_then_success(monkeypatch): - ok = json.dumps({"results": [{"prow_job_build_id": "1", "status": "success"}]}) - calls = _patch_urlopen(monkeypatch, ["504 Gateway Time-out", ok]) - results, err, auth_failed = send_batch(["1"], "tok", True) - assert err is None - assert auth_failed is False - assert len(results) == 1 - assert len(calls) == 2 - - -def test_send_batch_plaintext_body_is_retryable(monkeypatch): - calls = _patch_urlopen(monkeypatch, ["not json at all"] * 3) - results, err, auth_failed = send_batch(["1"], "tok", True) - assert results is None - assert auth_failed is False - assert "non-JSON" in err - assert len(calls) == 3 # retried up to the limit - - -def test_resolve_token_arg_wins_over_env(): - assert reevaluate_job_runs.resolve_token("argtok", {"SIPPY_TOKEN": "envtok"}) == "argtok" - -def test_resolve_token_falls_back_to_env(): - assert reevaluate_job_runs.resolve_token(None, {"SIPPY_TOKEN": "envtok"}) == "envtok" - -def test_resolve_token_none_when_unset(): - assert reevaluate_job_runs.resolve_token(None, {}) is None