diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9d797a46..84b66561 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -58,7 +58,7 @@ "name": "lvms-ci", "source": "./plugins/lvms-ci", "description": "LVMS CI Automation", - "version": "1.2.1" + "version": "1.3.0" }, { "name": "mcp-atlassian", @@ -70,7 +70,7 @@ "name": "microshift-ci", "source": "./plugins/microshift-ci", "description": "MicroShift CI Automation", - "version": "1.5.2" + "version": "1.5.3" }, { "name": "microshift-dev", diff --git a/plugins/lvms-ci/.claude-plugin/plugin.json b/plugins/lvms-ci/.claude-plugin/plugin.json index 9b85f6fa..ec0dc045 100644 --- a/plugins/lvms-ci/.claude-plugin/plugin.json +++ b/plugins/lvms-ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "lvms-ci", "description": "LVMS CI Automation", - "version": "1.2.1", + "version": "1.3.0", "author": { "name": "knarra" }, diff --git a/plugins/lvms-ci/.claude/settings.json b/plugins/lvms-ci/.claude/settings.json new file mode 100644 index 00000000..a3f78109 --- /dev/null +++ b/plugins/lvms-ci/.claude/settings.json @@ -0,0 +1,17 @@ +{ + "hooks": { + "SubagentStop": [ + { + "matcher": "lvms-ci:prow-job-analyzer", + "hooks": [ + { + "type": "command", + "command": "python3 \"$CLAUDE_PLUGIN_ROOT/scripts/validate-rca-output.py\"", + "timeout": 30, + "statusMessage": "Validating RCA output schema..." + } + ] + } + ] + } +} diff --git a/plugins/lvms-ci/agents/prow-job-analyzer.md b/plugins/lvms-ci/agents/prow-job-analyzer.md new file mode 100644 index 00000000..981e5029 --- /dev/null +++ b/plugins/lvms-ci/agents/prow-job-analyzer.md @@ -0,0 +1,193 @@ +--- +name: prow-job-analyzer +description: Analyzes a prow CI job's artifacts to produce a structured root cause analysis as JSON. Use for LVMS CI failure analysis. +tools: Bash, Read, Glob, Grep +model: inherit +effort: inherit +--- + +# Prow Job Root Cause Analyzer + +You analyze CI test job artifacts and produce a structured root cause analysis as a JSON array. + +## Input + +Your prompt contains: + +- `artifacts_dir` (required): local path to downloaded prow job artifacts (contains `build-log.txt` and `finished.json`) +- `job_url` (required): the full prow job URL — use directly when provided instead of reconstructing +- `job_name` (required): the full prow job name — use directly when provided instead of extracting + +## Output + +Respond with a valid JSON array only — no prose, no markdown fences. One object per independent failure (max 10). + +## Glossary + +- **ci-config**: Top level configuration file specifying build inputs, versions, and test workflows to execute. Periodic tests are suffixed with `__periodic.yaml`. +- **test**: The set of configurations and commands that specify how to execute the test. Can be defined in-line in ci-config, or as individual "steps" (see below). +- **step-registry**: Root directory where all openshift-ci test step configs and commands are stored. +- **step**: Smallest component of the test infrastructure. A step yaml specifies the command or script to execute, environmental variables and default values, and step metadata. Also called "ref" or "step ref". +- **chain**: A yaml configuration specifying 1 or more steps or chains in an array. Steps and chains are exploded and executed serially by index. May override step environment variable values. +- **workflow**: A yaml configuration specifying 1 or more steps, chains, or workflows in an array. Steps, chains, and workflows are exploded and executed serially. May override chain or step environmental variable values. Typically referenced by a test in a ci-config. +- **LVMS**: Logical Volume Manager Storage — an operator that manages local storage on OpenShift clusters using LVM thin provisioning via TopoLVM. +- **CatalogSource**: An OLM resource that defines an index of operator bundles. LVMS CI jobs create a CatalogSource to install the operator under test. +- **TopoLVM**: The CSI driver component of LVMS that manages logical volumes on nodes. +- **LVMCluster**: The custom resource that defines the LVMS storage configuration (device classes, thin pool settings). +- **vg-manager**: The LVMS component responsible for managing volume groups on each node. + +## Important Files + +- `/build-log.txt`: Prow job output — AWS infra and hypervisor errors surface here. The step diagram URL at the end links to the step execution graph. +- `/build-log.txt`: Per-step log — each CI step has its own `build-log.txt`. +- `/artifacts//lvms-catalogsource/build-log.txt`: CatalogSource creation step log. +- `/artifacts//operatorhub-subscribe-lvm-operator/build-log.txt`: LVMS operator subscription step log. +- `/artifacts//storage-create-lvm-cluster/build-log.txt`: LVMCluster creation step log. +- `/artifacts//lvms-sno-integration-test/build-log.txt`: Integration test step log (SNO variant; MNO variant uses `lvms-mno-integration-test`). This file is a JSON array of test result objects (not plain text). Each entry has `name` (full Ginkgo test name), `result` (`passed`/`failed`), `output` (test stdout), and `error` (failure message). The array may be followed by a trailing summary line like `Error: 2 tests failed` — strip it before parsing. Use the `name` field of failed entries to populate `scenarios`. +- `/artifacts//gather-extra/artifacts/pods/`: Pod logs collected at the end of the test run. Filenames follow the pattern `__.log` (and `_previous.log` for previous container instances). LVMS operator and component logs are under `openshift-lvm-storage_*`. Check these when the failure involves operator components (vg-manager, lvms-operator, topolvm-controller, topolvm-node). +- `/artifacts//gather-extra/artifacts/events.json`: Cluster events collected at test end — contains Kubernetes events including LVMS-specific events like `InconsistentLVs`, `VGsDegraded`, and `ResourceReconciliationIncomplete`. +- `/artifacts//gather-extra/artifacts/oc_cmds/`: Outputs of diagnostic `oc` commands (e.g., `oc get nodes`, `oc get pods`). + +## Important Links + +**Step Diagram URL** (found at the end of the main build-log): + +```text +https://steps.ci.openshift.org/job?org=openshift&repo=lvm-operator&branch=main&test=e2e-aws-sno-qe-integration-tests +``` + +Check the step diagram URL at the end of `build-log.txt` when identifying which step failed — not all fatal errors cause the current step to fail but may cause the next one to fail. + +## Investigation Principles + +Check the operator setup chain early: `lvms-catalogsource` → `operatorhub-subscribe-lvm-operator` → `storage-create-lvm-cluster`. If any of these failed, the operator was never fully deployed and all downstream test failures are secondary. + +The first error found is the anchor for deduplication, not the conclusion of the investigation. Drill from symptom → mechanism → actionable cause, or record the evidence gap in `analysis_gaps`. A timeout is not a root cause — explain what was slow or absent. A crash is not a root cause — explain what triggered it. + +The purpose of this analysis is to surface product defects. When a product component was unavailable, crashed, or flapped (readiness flips, liveness probe refused, container exits and restarts), reconstruct its timeline from the journal and pod logs before attributing fault. If the component became ready and later failed, that is a product defect even if a test-side wait would mask the symptom. A test defect is when the component was still starting up normally and the test ran too early. + +When the failure involves LVMS operator components (vg-manager, lvms-operator, topolvm-controller, topolvm-node), always check the operator and controller logs in `gather-extra/artifacts/pods/` and the cluster events in `gather-extra/artifacts/events.json`. Do not record an analysis gap for missing logs without first checking these directories. + +Use timeline ordering — not error-text similarity — to decide whether multiple failures are cascading (one root cause) or independent. + +## Tips + +1. There are many setup and teardown stages so fatal errors may be buried by log output from the teardown phase. It is not common to find the fatal error at the end of the log. +2. You can quickly determine the failed step from the build-log.txt by reading the last `Running step ...` line before the container logs appear. +3. Check the CatalogSource and operator setup steps (`lvms-catalogsource`, `operatorhub-subscribe-lvm-operator`, `storage-create-lvm-cluster`) early — if any failed, the operator was never fully deployed and all downstream test failures are secondary. +4. For test failures, always read the integration test step's `build-log.txt` (`lvms-sno-integration-test` or `lvms-mno-integration-test`). Parse it as JSON (strip any trailing non-JSON line), iterate the entries, and collect the `name` field from every entry with `"result": "failed"`. These are the scenario names for the `scenarios` field. Group failures that share the same root cause into a single output entry with all their scenario names. + +## JSON Schema + +Each entry in the output array has exactly these fields: + +```json +{ + "severity": 3, + "stack_layer": "test", + "step_name": "lvms-sno-integration-test", + "error_signature": "LVMCluster not ready within timeout", + "root_cause": "TopoLVM node agent failed to initialize volume group", + "raw_error": "LVMCluster not ready after 600s", + "infrastructure_failure": false, + "job_url": "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests/123456", + "job_name": "periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests", + "release": "main", + "remediation": "investigate TopoLVM node agent logs for volume group initialization errors", + "finished": "2026-06-01", + "causal_chain": [ + {"cause": "LVMCluster CR not ready after 600s — the storage-create-lvm-cluster step timed out waiting for the LVMCluster to reach Ready state, but the vg-manager pod was in CrashLoopBackOff due to a missing block device", + "evidence": "/tmp/lvm-operator-ci-claude-workdir.260601/artifacts/123456/artifacts/e2e-aws-sno-qe-integration-tests/storage-create-lvm-cluster/build-log.txt:234", + "quote": "LVMCluster not ready after 600s"} + ], + "confidence": "medium", + "analysis_gaps": [], + "scenarios": ["[sig-storage] STORAGE Author:mmakwana-High-66241-[OTP][LVMS] Check workload management annotations are present in LVMS resources [Disruptive]"] +} +``` + +### Field descriptions + +- `severity`: 1-5 per the severity rubric below +- `stack_layer`: one of `AWS Infra`, `External Infrastructure`, `build phase`, `deploy phase`, `test setup phase`, `Test Configuration`, `test`, `teardown` +- `step_name`: the CI step where the error occurred +- `error_signature`: concise one-line failure signature — used as bug titles for deduplication +- `root_cause`: one-line (~80 chars) WHY it failed (the mechanism, not the symptom) — used for cross-release dedup, so use stable terms without version numbers or timestamps +- `raw_error`: primary error message copied verbatim from the log (timestamps stripped, ~150 chars max) — used for deterministic grouping +- `infrastructure_failure`: `true` when the failure is AWS/CI infrastructure rather than product code +- `job_url`, `job_name`: use from the prompt when provided +- `release`: extract from job_name (e.g. `4.22` from `release-4.22`), default `main` +- `remediation`: suggested fix (~120 chars). Do not propose making the test more tolerant unless the causal chain shows the product behaved correctly +- `finished`: job finish date (`YYYY-MM-DD`) from `finished.json` timestamp +- `causal_chain`: array of `{"cause", "evidence", "quote"}` — each link toward root cause. `evidence` is an absolute path with line number (`/path/file:line`; `:1` for images). `quote` is a short verbatim excerpt (empty for images). Re-read every cited `file:line` before finalizing. Aim for 2-4 links. +- `confidence`: `high` (every link directly evidenced), `medium` (inferred but consistent), `low` (symptom-level, evidence exhausted — populate `analysis_gaps`) +- `analysis_gaps`: array of strings naming missing evidence. Empty when nothing was skipped. +- `scenarios`: array of Ginkgo test names (`name` field from the integration test step's JSON build-log) affected by this failure. For `stack_layer: "test"` entries, parse the integration test step's `build-log.txt` as JSON and collect the `name` from each entry with `"result": "failed"` that matches this root cause. Empty array only for non-test failures (build, infra, deploy). + +### Severity rubric + +| Severity | Meaning | +|---|---| +| 5 | LVMS operator or setup issue — operator subscription failure, LVMCluster not ready, storage class misconfiguration | +| 4 | Genuine test failure in LVMS code — integration test assertion failure, regression in operator logic | +| 3 | Infrastructure or CI config issue — CatalogSource image unavailable, base image build failure, cluster provisioning failure | +| 2 | Intermittent failure / likely flake | +| 1 | Infrastructure noise or self-healing condition | + +### RAW_ERROR rules + +The `raw_error` field is used by downstream scripts for deterministic grouping. Two runs analyzing the same job MUST produce the same `raw_error`. Keep it simple — fewer rules mean less room for variation. + +1. **Copy-paste the exact error text** from the log — do NOT paraphrase, summarize, or reword +2. **Pick only ONE error** — the primary error that caused the step to fail. If multiple errors exist, pick the first fatal one. +3. **Only strip timestamps** — remove leading timestamps like `2026-04-01T06:21:48Z`. Keep everything else verbatim. +4. **Never concatenate multiple errors** — pick ONE error, not a semicolon-separated list +5. **Truncate to ~150 characters** if the raw message is very long — keep the distinctive part + +Examples of good `raw_error` values (copied verbatim from logs): + +- `An error occurred (InvalidClientTokenId) when calling the CreateStack operation: The security token included in the request is invalid.` +- `panic: runtime error: index out of range [6] with length 6` +- `Process did not finish before 4h0m0s timeout` +- `error: the server doesn't have a resource type "clusterversion"` + +### ROOT_CAUSE rules + +The `root_cause` field captures the underlying mechanism — used alongside `raw_error` for cross-release deduplication. + +**How it differs from the other fields:** + +- `error_signature` = WHAT failed (human-readable, used for bug titles) +- `root_cause` = WHY it failed (mechanism-focused, used for dedup) +- `raw_error` = verbatim log text (deterministic anchor) + +**Rules:** + +1. **One line, ~80 characters max** — short enough for token-based matching +2. **Focus on the mechanism**, not the symptom — ask "why did this happen?" not "what error appeared?" +3. **Be consistent across releases** — the same underlying problem in 4.20 and 4.22 MUST produce the same `root_cause` even if the error messages differ +4. **Use stable terms** — avoid version numbers, timestamps, job names, or other run-specific details + +**Examples:** + +| ERROR_SIGNATURE | ROOT_CAUSE | +|---|---| +| CatalogSource not ready — operator bundle image pull failure | index image unavailable or registry authentication failure | +| LVMCluster not ready within timeout | TopoLVM node agent failed to initialize volume group | +| e2e test PVC provisioning timeout on SNO | LVM thin pool exhausted or volume group misconfigured | +| InvalidClientTokenId when calling CreateStack | expired or invalid AWS credentials in CI environment | + +### CONFIDENCE rules + +Downstream automation uses confidence to decide whether to act — do not inflate it. + +- `high`: every causal-chain link is directly evidenced by a quoted artifact line or graph +- `medium`: the mechanism is inferred but consistent with all available evidence +- `low`: symptom-level only — populate `analysis_gaps` + +### Multiple independent failures + +- One entry per independent failure — same root cause = one entry with all affected scenarios +- At most 10 entries per job, report the most severe +- Cascading failures are not independent — report only the root failure +- Single failures are still wrapped in an array diff --git a/plugins/lvms-ci/scripts/validate-rca-output.py b/plugins/lvms-ci/scripts/validate-rca-output.py new file mode 120000 index 00000000..8367a04f --- /dev/null +++ b/plugins/lvms-ci/scripts/validate-rca-output.py @@ -0,0 +1 @@ +../../shared/scripts/validate-rca-output.py \ No newline at end of file diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index 733f905e..d4548ce5 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -67,35 +67,24 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa - If a release has no failed jobs, its jobs JSON will be an empty array — skip analysis for that release - If a release has an `"error"` field in the JSON summary, data collection failed for that release — report the error to the user but continue with other releases -### Step 2: Analyze Each Job Using /lvms-ci:prow-job +### Step 2: Analyze Each Job Using prow-job-analyzer Agent **Goal**: Get detailed root cause analysis for each failed job using pre-downloaded artifacts. **Actions**: -1. Use the JSON summary output from Step 1 to build agent prompts. Do NOT read the job JSON files into the main conversation — the prepare script already printed all job details (artifacts_dir, build_id, job name) and agents receive artifacts_dir directly in their prompt. -2. For **every** failed job across all releases and PRs, launch a separate **Agent** (using the `Agent` tool, NOT the `Skill` tool). For PR jobs, only launch agents for jobs with FAILURE status. +1. Use the JSON summary output from Step 1 to build agent prompts. Do NOT read the job JSON files into the main conversation — the prepare script already printed all job details (artifacts_dir, build_id, job name, url) and agents receive these directly in their prompt. +2. For **every** failed job across all releases and PRs, launch a separate **Agent** (using the `Agent` tool, NOT the `Skill` tool) with `subagent_type=lvms-ci:prow-job-analyzer`. For PR jobs, only launch agents for jobs with FAILURE status. - **For release jobs:** + The agent returns a JSON array directly — no extraction needed. Build the prompt with the job's `artifacts_dir`, `url` (as `job_url`), and `job` (as `job_name`) from the prepare output. - ```text - Agent: subagent_type=general_purpose, prompt="Analyze this Prow job and save the report: - 1. Run /lvms-ci:prow-job - 2. After the analysis completes, extract only the JSON array from the output - and save it to: - /jobs/release--job--.json - Use the Write tool. The file must contain ONLY the valid JSON array — no prose, no markers." - ``` - - **For PR jobs:** + **Example prompt:** ```text - Agent: subagent_type=general_purpose, prompt="Analyze this Prow job and save the report: - 1. Run /lvms-ci:prow-job - 2. After the analysis completes, extract only the JSON array from the output - and save it to: - /jobs/prs-job--pr-.json - Use the Write tool. The file must contain ONLY the valid JSON array — no prose, no markers." + Analyze this prow job: + artifacts_dir: + job_url: + job_name: ``` After each agent completes, save its JSON response to the corresponding file using the Write tool: @@ -171,7 +160,8 @@ Z-stream test results are collected automatically when `--pull-requests` is pass ## Related Skills -- **lvms-ci:prow-job**: Single job analysis (used by Step 2 agents) +- **lvms-ci:prow-job**: Single job analysis (thin wrapper around the `lvms-ci:prow-job-analyzer` agent) +- **lvms-ci:prow-job-analyzer**: Dedicated agent for root cause analysis of a single prow job (used directly by Step 2) ## Notes diff --git a/plugins/lvms-ci/skills/prow-job/SKILL.md b/plugins/lvms-ci/skills/prow-job/SKILL.md index e6345158..ea0bfc91 100644 --- a/plugins/lvms-ci/skills/prow-job/SKILL.md +++ b/plugins/lvms-ci/skills/prow-job/SKILL.md @@ -3,7 +3,7 @@ name: lvms-ci:prow-job argument-hint: description: Download Prow job artifacts, identify root cause of failure, and produce a structured error report user-invocable: true -allowed-tools: Skill, Bash, Read, Write, Glob, Grep, Agent +allowed-tools: Bash, Read, Write, Agent --- # lvms-ci:prow-job @@ -17,7 +17,7 @@ allowed-tools: Skill, Bash, Read, Write, Glob, Grep, Agent ## Description -Analyzes a single Prow CI test job by scanning artifacts for errors and producing a structured failure report. Accepts either a Prow job URL (downloads artifacts) or a local directory path (uses pre-downloaded artifacts). +Analyzes a single Prow CI test job by downloading artifacts and running a root cause analysis agent. Accepts either a Prow job URL (downloads artifacts) or a local directory path (uses pre-downloaded artifacts). The analysis produces a structured JSON report that is formatted as human-readable prose for the user. ## Arguments @@ -26,25 +26,6 @@ Analyzes a single Prow CI test job by scanning artifacts for errors and producin - **GCS web URL**: `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests/1984108354347208704` - **Local artifacts directory**: `/tmp/lvm-operator-ci-claude-workdir.260404/artifacts/1984108354347208704` (must contain `build-log.txt` and `finished.json`) -## Goal - -Reduce noise for developers by processing large logs from a CI test pipeline and correctly classifying fatal errors with a false-positive rate of 0.01% and false-negative rate of 0.5%. - -## Audience - -Software Engineer - -## Glossary - -- **ci-config**: Top level configuration file specifying build inputs, versions, and test workflows to execute. Periodic tests are suffixed with `__periodic.yaml`. -- **test**: The set of configurations and commands that specify how to execute the test. Can be defined in-line in ci-config, or as individual "steps" (see below). -- **step-registry**: Root directory where all openshift-ci test step configs and commands are stored. -- **step**: Smallest component of the test infrastructure. A step yaml specifies the command or script to execute, environmental variables and default values, and step metadata. Also called "ref" or "step ref". -- **chain**: A yaml configuration specifying 1 or more steps or chains in an array. Steps and chains are exploded and executed serially by index. May override step environment variable values. -- **workflow**: A yaml configuration specifying 1 or more steps, chains, or workflows in an array. Steps, chains, and workflows are exploded and executed serially. May override chain or step environmental variable values. Typically referenced by a test in a ci-config. -- **LVMS**: Logical Volume Manager Storage — an operator that manages local storage on OpenShift clusters using LVM thin provisioning via TopoLVM. -- **CatalogSource**: An OLM resource that defines an index of operator bundles. LVMS CI jobs create a CatalogSource to install the operator under test. - ## Job Name and Job ID The Job Name and Job ID are encoded in the URL. There are two URL formats depending on the job type: @@ -72,27 +53,6 @@ To determine the GCS path from any job URL, strip the web prefix and replace wit - Prow URL: strip `https://prow.ci.openshift.org/view/gs/` - GCS web URL: strip `https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/` -## Important Files - -> These files are available after artifacts are downloaded (via the download script or workflow step 0). - -- `/build-log.txt`: Log containing prow job output and most likely place to identify AWS infra related errors. -- `/build-log.txt`: Each step in the CI job is individually logged in a build-log.txt file. -- `/artifacts//lvms-catalogsource/build-log.txt`: CatalogSource creation step log. -- `/artifacts//operatorhub-subscribe-lvm-operator/build-log.txt`: LVMS operator subscription step log. -- `/artifacts//storage-create-lvm-cluster/build-log.txt`: LVMCluster creation step log. -- `/artifacts//lvms-sno-integration-test/build-log.txt`: Integration test execution step log (SNO variant; MNO variant uses `lvms-mno-integration-test`). - -## Important Links - -**Step Diagram URL** (found at the end of the main build-log): - -```text -https://steps.ci.openshift.org/job?org=openshift&repo=lvm-operator&branch=main&test=e2e-aws-sno-qe-integration-tests -``` - -This link provides a diagram of the steps that make up the test. Think about reading this diagram when identifying step failures because not all fatal errors cause the current step to fail but may cause the next step to fail. - ## Work Directory Compute once at the start by running `date +%y%m%d` and substituting into the path below. In all commands, replace `` with the computed path — do not store the work directory in a shell variable. @@ -101,181 +61,63 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa /tmp/lvm-operator-ci-claude-workdir. ``` -## Common Commands - -Scan the build log for arbitrary text: - -```bash -grep '${SOME_TEXT}' ${GREP_OPTS} ${TMP}/build-log.txt -``` - -Download all prow job artifacts (only needed when given a URL, not a local path): +## Prerequisites -```bash -GCS_PATH=$(echo "${PROW_URL}" | sed -e 's|https://prow.ci.openshift.org/view/gs/|gs://|' -e 's|https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/|gs://|') -gsutil -q -m cp -r "${GCS_PATH}/" ${TMP}/ -``` +- `gsutil` CLI must be installed for GCS access (uses anonymous access on public buckets; only needed for URL input — pre-downloaded artifacts skip it) +- Internet access to fetch job data from Prow/GCS +- Bash shell ## Workflow The user argument is: `` 0. **Determine input type and set up artifacts directory**: - - If `` is a **local directory path** (starts with `/` and contains `build-log.txt`): set `TMP` to that directory. Skip step 1. - - If `` is a **URL** (starts with `http`): create a temporary working directory with `mktemp -d /openshift-ci-analysis-XXXX`, set `TMP` to that directory, and proceed to step 1. + - If `` is a **local directory path** (starts with `/` and contains `build-log.txt`): set `TMP` to that directory. Skip step 1. Derive `JOB_URL` from the `build-log.txt` "Link to job on registry info site" line, and extract `JOB_NAME` from the URL path (the path segment before the numeric job ID). + - If `` is a **URL** (starts with `http`): set `JOB_URL` to ``. Extract `JOB_NAME` from the URL path (the path segment before the numeric job ID — e.g. `periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests` from the URL). Create a temporary working directory with `mktemp -d /openshift-ci-analysis-XXXX`, set `TMP` to that directory, and proceed to step 1. 1. **Download all artifacts** (skip if using pre-downloaded artifacts from step 0): Download all prow job artifacts using `gsutil -q -m cp -r` into the temporary working directory. Derive the GCS path by stripping the web prefix from the job URL (handles both Prow and GCS web URL formats): ```bash - GCS_PATH=$(echo "${PROW_URL}" | sed -e 's|https://prow.ci.openshift.org/view/gs/|gs://|' -e 's|https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/|gs://|') + GCS_PATH=$(echo "${JOB_URL}" | sed -e 's|https://prow.ci.openshift.org/view/gs/|gs://|' -e 's|https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/|gs://|') gsutil -q -m cp -r "${GCS_PATH}/" ${TMP}/ ``` This works for both periodic (`logs/...`) and presubmit PR (`pr-logs/pull/...`) job URLs, and for both Prow and GCS web URL formats. This makes all build logs, step logs, and SOS reports available locally for analysis. -2. **Scan for errors**: Start by scanning the top level `build-log.txt` file for errors and determine the step where the error occurred. Record each error with the filepath and line number for later reference. - -3. **Read context**: Iterate over each recorded error, locate the log file and line number, then read 50 lines before and 50 lines after the error. Use this information to characterize the error. Think about whether this error is transient and think about where in the stack the error occurs. Does it occur in the cloud infra, the openshift or prow ci-config, the hypervisor, or is it a legitimate test failure? If it is a legitimate test failure, determine what stage of the test failed: setup, testing, teardown. Record the filepath and line number of each piece of evidence — these become `causal_chain` entries in the output. - -4. **Analyze the error**: Based on the context of the error, think hard about whether this error caused the test to fail, is a transient error, or is a red herring. +2. **Run root cause analysis**: + Spawn a single Agent with `subagent_type=lvms-ci:prow-job-analyzer` to analyze the artifacts. Build the prompt with: - 4.1 If it is a legitimate test error, analyze the test logs to determine the source of the error. - 4.2 If the source of the error appears to be related to the LVMS operator or its components (TopoLVM, LVMCluster), check the operator and controller logs in the step artifacts. - 4.3 Assess `confidence` based on the strength of evidence: `high` when every causal link is directly evidenced, `medium` when inferred but consistent, `low` when evidence ran out. If `low`, populate `analysis_gaps` with what was missing (e.g. `"operator logs not available"`). Record any failing test scenario names for the `scenarios` field. + - `artifacts_dir`: the `TMP` path from step 0/1 + - `job_url`: the `JOB_URL` from step 0 + - `job_name`: the `JOB_NAME` from step 0 -5. **Produce the output**: Populate the JSON fields for each independent failure. Each entry must specify: - - `stack_layer` and `step_name` for where in the pipeline the error occurred - - `infrastructure_failure` for whether the failure is due to CI infrastructure rather than LVMS code - - `causal_chain` linking the observed symptom to the underlying cause, with evidence file paths and quotes + Example prompt: -## Prerequisites - -- `gsutil` CLI must be installed for GCS access (uses anonymous access on public buckets) -- Internet access to fetch job data from Prow/GCS -- Bash shell - -## Tips - -1. There are many setup and teardown stages so fatal errors may be buried by log output from the teardown phase. It is not common to find the fatal error at the end of the log. -2. You can quickly determine the failed step from the build-log.txt by reading the last `Running step ...` line before the container logs appear. -3. Check the CatalogSource and operator setup steps (`lvms-catalogsource`, `operatorhub-subscribe-lvm-operator`, `storage-create-lvm-cluster`) early — if any failed, the operator was never fully deployed and all downstream test failures are secondary. - -## Output Template - -Your entire response must be a valid JSON array. No prose, no markdown fences, no explanation before or after. One object per independent failure (max 5). Single failures are still wrapped in a JSON array. - -### Severity Guide - -| Severity | Meaning | Examples | -|----------|---------|----------| -| 1 | Cosmetic or informational, no action needed | Flaky teardown warning, non-fatal log noise | -| 2 | Transient infrastructure flake, retrigger likely fixes | AWS quota, image pull timeout, CI registry blip | -| 3 | Infrastructure or CI config issue, not LVMS code | CatalogSource image unavailable, base image build failure (`PullBuilderImageFailed`), cluster provisioning failure | -| 4 | Genuine test failure in LVMS code | Integration test assertion failure, regression in operator logic | -| 5 | LVMS operator or setup issue | LVMCluster not ready, operator subscription failure, storage class misconfiguration | - -### JSON Schema - -```json -[ - { - "severity": 3, - "stack_layer": "test", - "step_name": "lvms-sno-integration-test", - "error_signature": "LVMCluster not ready within timeout", - "root_cause": "TopoLVM node agent failed to initialize volume group", - "raw_error": "LVMCluster not ready after 600s", - "infrastructure_failure": false, - "job_url": "https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests/123456", - "job_name": "periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests", - "release": "main", - "remediation": "investigate TopoLVM node agent logs for volume group initialization errors", - "finished": "2026-06-01", - "causal_chain": [ - {"cause": "LVMCluster CR not ready after 600s", "evidence": "/tmp/lvms-ci-claude-workdir.260601/artifacts/123456/build-log.txt:1234", "quote": "LVMCluster not ready after 600s"} - ], - "confidence": "medium", - "analysis_gaps": [], - "scenarios": [] - } -] -``` - -**Field descriptions:** - -- `severity`: 1-5, same as Error Severity above -- `stack_layer`: one of: AWS Infra, External Infrastructure, build phase, deploy phase, test setup phase, Test Configuration, test, teardown -- `step_name`: the CI step where the error occurred -- `error_signature`: a concise, unique one-line description of the root cause — not the full error, just enough to identify and deduplicate this failure -- `root_cause`: one-line description of WHY the failure happened — the underlying mechanism, not the surface symptom (~80 chars max, see rules below) -- `raw_error`: the primary error message copied VERBATIM from the log file (see rules below) -- `infrastructure_failure`: true if stack_layer is AWS Infra or the failure is due to CI infrastructure rather than product code, false otherwise -- `job_url`: the full prow job URL — when given a URL as input, use it directly; when given a local artifacts dir, reconstruct from the build-log.txt "Link to job on registry info site" line or from the directory path structure -- `job_name`: the full job name — extract from the job_url path, or from the build-log.txt "Running step" lines, or from the artifacts directory structure -- `release`: the release branch — extract from job_name (e.g. 4.22 from release-4.22), or from finished.json metadata repos field, or default to "main" -- `remediation`: suggested fix or next step — what should be done to address this failure (~120 chars max). For infrastructure failures, state the infra action (e.g. "retry the job", "rotate AWS credentials"). For product bugs, state the code-level fix direction -- `finished`: the job finish date in YYYY-MM-DD format, extracted from finished.json timestamp field or build log timestamps -- `causal_chain`: array of links from observed symptom toward root cause. Each link: `{"cause": ..., "evidence": ..., "quote": ...}` where `evidence` is the **absolute** file path with a mandatory line number (`/absolute/path:lineNum`; use `:1` for binary files) and `quote` is a short verbatim excerpt from the cited line (empty for binary files). Before finalizing, re-read every cited `file:line` and confirm the quote is actually there -- `confidence`: one of `high`, `medium`, `low` — `high` when every chain link is directly evidenced, `medium` when inferred but consistent, `low` when evidence ran out -- `analysis_gaps`: array of strings naming evidence that was missing (e.g. `"no sosreport in artifacts"`). Empty array when nothing was skipped -- `scenarios`: array of scenario names where this failure occurred. Empty array for non-scenario jobs and build/infra failures - -### RAW_ERROR rules - -The `RAW_ERROR` field is used by downstream scripts for deterministic grouping. Two runs analyzing the same job MUST produce the same RAW_ERROR. Keep it simple — fewer rules mean less room for variation. - -1. **Copy-paste the exact error text** from the log — do NOT paraphrase, summarize, or reword -2. **Pick only ONE error** — the primary error that caused the step to fail. If multiple errors exist, pick the first fatal one. -3. **Only strip timestamps** — remove leading timestamps like `2026-04-01T06:21:48Z`. Keep everything else verbatim, including prefixes like `An error occurred...` or `error:`. -4. **Never concatenate multiple errors** — pick ONE error, not a semicolon-separated list -5. **Truncate to ~150 characters** if the raw message is very long — keep the distinctive part - -Examples of good RAW_ERROR values (copied verbatim from logs): - -- `An error occurred (InvalidClientTokenId) when calling the CreateStack operation: The security token included in the request is invalid.` -- `panic: runtime error: index out of range [6] with length 6` -- `Process did not finish before 4h0m0s timeout` -- `error: the server doesn't have a resource type "clusterversion"` -- `package github.com/opencontainers/runc/libcontainer/cgroups: module github.com/opencontainers/runc@latest found, but does not contain package` - -The ERROR_SIGNATURE field remains as a human-readable description for reports and Jira bug titles. - -### ROOT_CAUSE rules - -The `ROOT_CAUSE` field captures the underlying mechanism behind the failure — used by downstream scripts alongside `RAW_ERROR` for cross-release deduplication. Two jobs that fail with different surface errors but the same root cause should produce the same `ROOT_CAUSE`. - -**How it differs from the other fields:** - -- `ERROR_SIGNATURE` = WHAT failed (human-readable, used for bug titles) -- `ROOT_CAUSE` = WHY it failed (mechanism-focused, used for dedup) -- `RAW_ERROR` = verbatim log text (deterministic anchor) - -**Rules:** - -1. **One line, ~80 characters max** — short enough for token-based matching -2. **Focus on the mechanism**, not the symptom — ask "why did this happen?" not "what error appeared?" -3. **Be consistent across releases** — the same underlying problem in 4.20 and 4.22 MUST produce the same ROOT_CAUSE even if the error messages differ -4. **Use stable terms** — avoid version numbers, timestamps, job names, or other run-specific details - -**Examples:** - -| ERROR_SIGNATURE | ROOT_CAUSE | -|---|---| -| CatalogSource not ready — operator bundle image pull failure | index image unavailable or registry authentication failure | -| LVMCluster not ready within timeout | TopoLVM node agent failed to initialize volume group | -| e2e test PVC provisioning timeout on SNO | LVM thin pool exhausted or volume group misconfigured | -| InvalidClientTokenId when calling CreateStack | expired or invalid AWS credentials in CI environment | - -### Multiple independent failures - -When a job has multiple independent test failures across different scenarios, produce **one entry per failure** in the JSON array. Each entry must be self-contained with all fields populated. + ```text + Analyze this prow job: + artifacts_dir: /tmp/lvm-operator-ci-claude-workdir.260710/artifacts/2075422415638237184 + job_url: https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests/2075422415638237184 + job_name: periodic-ci-openshift-lvm-operator-main-e2e-aws-sno-qe-integration-tests + ``` -**Rules:** +3. **Display results**: + Parse the JSON array returned by the agent. For each entry, format the output as: + + ```text + Error Severity: {severity}/5 + Stack Layer: {stack_layer} + Step Name: {step_name} + Error: {raw_error} + Causal Chain: + (for each link in causal_chain, numbered starting at 1) + N. {link.cause} + Evidence: {link.evidence} — "{link.quote}" + Confidence: {confidence} + Suggested Remediation: {remediation} + ``` -1. **One entry per independent failure** — failures are independent when they occur in different test scenarios with different root causes (e.g., CatalogSource pull failure in one test and PVC timeout in another) -2. **Same root cause = one entry** — when multiple scenarios fail with the same root cause, produce ONE entry. Do NOT split them into separate entries. -3. **At most 5 entries per job** — if more than 5 independent failures exist, report the 5 most severe -4. **Cascading failures are NOT independent** — when one failure causes others (e.g., a setup failure causing all subsequent tests to fail), report only the root failure -5. **Single failures are still an array** — even when there is only one failure, wrap it in a JSON array + If `TMP` is inside a `` structure, also save the raw JSON to: + `/jobs/release--job-.json` + (derive RELEASE and JOB_ID from the artifacts path and the JSON content). diff --git a/plugins/microshift-ci/.claude-plugin/plugin.json b/plugins/microshift-ci/.claude-plugin/plugin.json index d6ed298d..f4b4acda 100644 --- a/plugins/microshift-ci/.claude-plugin/plugin.json +++ b/plugins/microshift-ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "microshift-ci", "description": "MicroShift CI Automation", - "version": "1.5.2", + "version": "1.5.3", "author": { "name": "ggiguash" }, diff --git a/plugins/microshift-ci/.claude/settings.json b/plugins/microshift-ci/.claude/settings.json index fdd5552b..120a4366 100644 --- a/plugins/microshift-ci/.claude/settings.json +++ b/plugins/microshift-ci/.claude/settings.json @@ -2,7 +2,7 @@ "hooks": { "SubagentStop": [ { - "matcher": "prow-job-analyzer", + "matcher": "microshift-ci:prow-job-analyzer", "hooks": [ { "type": "command", diff --git a/plugins/microshift-ci/scripts/validate-rca-output.py b/plugins/microshift-ci/scripts/validate-rca-output.py deleted file mode 100755 index 36f5d023..00000000 --- a/plugins/microshift-ci/scripts/validate-rca-output.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""SubagentStop hook validator for prow-job-analyzer agent output. - -Reads the hook payload from stdin, extracts last_assistant_message, -and validates it against the expected JSON schema. Returns a block -decision with specific corrections when validation fails. -""" - -import json -import os -import re -import sys - -REQUIRED_FIELDS = { - "severity", "stack_layer", "step_name", "error_signature", - "root_cause", "raw_error", "infrastructure_failure", - "job_url", "job_name", "release", "remediation", "finished", - "causal_chain", "confidence", "analysis_gaps", "scenarios", -} - -NON_EMPTY_STRING_FIELDS = { - "error_signature", "raw_error", "job_url", "job_name", "finished", - "step_name", "root_cause", "remediation", -} - -# Keep in sync with prow-job-analyzer.md (field descriptions) and -# lvms-ci/skills/prow-job/SKILL.md (severity guide / JSON schema). -VALID_CONFIDENCE = {"high", "medium", "low"} -VALID_STACK_LAYERS = { - "AWS Infra", "External Infrastructure", "build phase", "deploy phase", - "test setup phase", "Test Configuration", "test", "teardown", -} - - -BINARY_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".tar.xz", ".gz", ".bz2", ".xz", ".zip") - - -def _read_lines(path, cache): - """Read file lines with caching to avoid re-reading large build logs.""" - if path in cache: - return cache[path] - try: - with open(path, errors="replace") as f: - lines = f.readlines() - except OSError: - lines = None - cache[path] = lines - return lines - - -def validate_evidence(evidence, quote, prefix, file_cache): - """Validate that a causal_chain evidence citation is real. - - Checks: format (absolute_path:line), file exists, line in range, - quote appears on cited line. Returns a list of error strings. - """ - m = re.fullmatch(r"(.+):(\d+)", evidence) - if not m: - return [f"{prefix}: evidence must be absolute_path:line_number, got: {evidence}"] - - path, line_no = m.group(1), int(m.group(2)) - - if not os.path.isabs(path): - return [f"{prefix}: evidence path must be absolute, got: {path}"] - - if not os.path.isfile(path): - return [f"{prefix}: evidence file not found: {path}"] - - if any(path.endswith(ext) for ext in BINARY_EXTENSIONS): - return [] - - lines = _read_lines(path, file_cache) - if lines is None: - return [f"{prefix}: evidence file could not be read: {path}"] - - if line_no < 1 or line_no > len(lines): - return [f"{prefix}: evidence cites line {line_no} but file has only {len(lines)} lines"] - - if not isinstance(quote, str) or len(quote) < 10: - return [] - - cited_line = " ".join(lines[line_no - 1].split()).lower() - normalized_quote = " ".join(quote.split()).lower() - if normalized_quote not in cited_line: - return [f"{prefix}: quote not found on line {line_no}"] - - return [] - - -def validate_entry(entry, index, file_cache): - errors = [] - - missing = REQUIRED_FIELDS - set(entry.keys()) - if missing: - errors.append(f"entry[{index}]: missing required fields: {', '.join(sorted(missing))}") - - for field in NON_EMPTY_STRING_FIELDS: - val = entry.get(field) - if not isinstance(val, str) or not val: - errors.append(f"entry[{index}]: '{field}' must be a non-empty string") - - sev = entry.get("severity") - if isinstance(sev, bool) or not isinstance(sev, int) or not (1 <= sev <= 5): - errors.append(f"entry[{index}]: 'severity' must be an integer 1-5, got {sev!r}") - - infra = entry.get("infrastructure_failure") - if not isinstance(infra, bool): - errors.append(f"entry[{index}]: 'infrastructure_failure' must be a boolean, got {type(infra).__name__}") - - layer = entry.get("stack_layer") - if not isinstance(layer, str) or layer not in VALID_STACK_LAYERS: - errors.append(f"entry[{index}]: 'stack_layer' must be one of {sorted(VALID_STACK_LAYERS)}, got {layer!r}") - - conf = entry.get("confidence") - if not isinstance(conf, str) or conf not in VALID_CONFIDENCE: - errors.append(f"entry[{index}]: 'confidence' must be one of {sorted(VALID_CONFIDENCE)}, got {conf!r}") - - chain = entry.get("causal_chain") - if not isinstance(chain, list): - if chain is not None: - errors.append(f"entry[{index}]: 'causal_chain' must be an array") - else: - errors.append(f"entry[{index}]: 'causal_chain' must be a non-empty array, got null") - elif not chain: - errors.append(f"entry[{index}]: 'causal_chain' must be a non-empty array") - else: - for ci, link in enumerate(chain): - if not isinstance(link, dict): - errors.append(f"entry[{index}].causal_chain[{ci}]: must be an object") - continue - if "cause" not in link: - errors.append(f"entry[{index}].causal_chain[{ci}]: missing required key 'cause'") - if "evidence" not in link: - errors.append(f"entry[{index}].causal_chain[{ci}]: missing required key 'evidence'") - if "quote" not in link: - errors.append(f"entry[{index}].causal_chain[{ci}]: missing required key 'quote'") - evidence = link.get("evidence", "") - quote = link.get("quote", "") - if isinstance(evidence, str) and evidence: - errors.extend(validate_evidence( - evidence, quote, - f"entry[{index}].causal_chain[{ci}]", file_cache)) - - for field in ("analysis_gaps", "scenarios"): - val = entry.get(field) - if val is not None and not isinstance(val, list): - errors.append(f"entry[{index}]: '{field}' must be an array") - - return errors - - -def validate_json_text(text): - try: - data = json.loads(text) - except json.JSONDecodeError as e: - if "--- STRUCTURED SUMMARY ---" in text: - return [ - "Output contains prose and STRUCTURED SUMMARY markers. " - "Your entire response must be a valid JSON array only — no prose, no markers." - ] - return [f"Output is not valid JSON: {e}. Your entire response must be a valid JSON array."] - - if isinstance(data, dict): - return [ - "Output is a JSON object, not an array. " - "Wrap your output in [...] — single failures must still be a JSON array." - ] - elif not isinstance(data, list): - return [f"Expected a JSON array, got {type(data).__name__}"] - - if not data: - return ["JSON array is empty. Expected at least one failure entry."] - - file_cache = {} - all_errors = [] - for i, entry in enumerate(data): - if not isinstance(entry, dict): - all_errors.append(f"entry[{i}]: expected an object, got {type(entry).__name__}") - continue - all_errors.extend(validate_entry(entry, i, file_cache)) - - return all_errors - - -def validate_message(message): - if not message or not message.strip(): - return ["Agent produced empty output. Expected a JSON array."] - - return validate_json_text(message.strip()) - - -def main(): - try: - payload = json.load(sys.stdin) - except (json.JSONDecodeError, ValueError): - print("WARNING: validate-rca-output: malformed JSON on stdin, skipping validation", file=sys.stderr) - sys.exit(0) - - if not isinstance(payload, dict): - print("WARNING: validate-rca-output: expected dict payload, skipping validation", file=sys.stderr) - sys.exit(0) - - message = payload.get("last_assistant_message", "") - errors = validate_message(message) - - if errors: - reason = "RCA output validation failed:\n" + "\n".join(f" - {e}" for e in errors) - json.dump({"decision": "block", "reason": reason}, sys.stdout) - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/plugins/microshift-ci/scripts/validate-rca-output.py b/plugins/microshift-ci/scripts/validate-rca-output.py new file mode 120000 index 00000000..8367a04f --- /dev/null +++ b/plugins/microshift-ci/scripts/validate-rca-output.py @@ -0,0 +1 @@ +../../shared/scripts/validate-rca-output.py \ No newline at end of file diff --git a/plugins/microshift-ci/skills/prow-job/SKILL.md b/plugins/microshift-ci/skills/prow-job/SKILL.md index 5b4fe9a7..e6101431 100644 --- a/plugins/microshift-ci/skills/prow-job/SKILL.md +++ b/plugins/microshift-ci/skills/prow-job/SKILL.md @@ -3,7 +3,7 @@ name: microshift-ci:prow-job argument-hint: description: Download Prow job artifacts, identify root cause of failure, and produce a structured error report user-invocable: true -allowed-tools: Skill, Bash, Read, Write, Glob, Grep, Agent +allowed-tools: Bash, Read, Write, Agent --- # microshift-ci:prow-job diff --git a/plugins/shared/scripts/validate-rca-output.py b/plugins/shared/scripts/validate-rca-output.py new file mode 100755 index 00000000..6094194e --- /dev/null +++ b/plugins/shared/scripts/validate-rca-output.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""SubagentStop hook validator for prow-job-analyzer agent output. + +Reads the hook payload from stdin, extracts last_assistant_message, +and validates it against the expected JSON schema. Returns a block +decision with specific corrections when validation fails. +""" + +import json +import os +import re +import sys + +REQUIRED_FIELDS = { + "severity", "stack_layer", "step_name", "error_signature", + "root_cause", "raw_error", "infrastructure_failure", + "job_url", "job_name", "release", "remediation", "finished", + "causal_chain", "confidence", "analysis_gaps", "scenarios", +} + +NON_EMPTY_STRING_FIELDS = { + "error_signature", "raw_error", "job_url", "job_name", "finished", + "step_name", "root_cause", "remediation", "release", +} + +# Keep in sync with prow-job-analyzer.md (field descriptions, +# severity rubric, and JSON schema) in each plugin. +VALID_CONFIDENCE = {"high", "medium", "low"} +VALID_STACK_LAYERS = { + "AWS Infra", "External Infrastructure", "build phase", "deploy phase", + "test setup phase", "Test Configuration", "test", "teardown", +} + + +BINARY_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".tar.xz", ".gz", ".bz2", ".xz", ".zip") + + +def _read_lines(path, cache): + """Read file lines with caching to avoid re-reading large build logs.""" + if path in cache: + return cache[path] + try: + with open(path, errors="replace") as f: + lines = f.readlines() + except OSError: + lines = None + cache[path] = lines + return lines + + +def validate_evidence(evidence, quote, prefix, file_cache): + """Validate that a causal_chain evidence citation is real. + + Checks: format (absolute_path:line), file exists, line in range, + quote appears on cited line. Returns a list of error strings. + """ + m = re.fullmatch(r"(.+):(\d+)", evidence) + if not m: + return [f"{prefix}: evidence must be absolute_path:line_number, got: {evidence}"] + + path, line_no = m.group(1), int(m.group(2)) + + if not os.path.isabs(path): + return [f"{prefix}: evidence path must be absolute, got: {path}"] + + if not os.path.isfile(path): + return [f"{prefix}: evidence file not found: {path}"] + + if any(path.endswith(ext) for ext in BINARY_EXTENSIONS): + return [] + + lines = _read_lines(path, file_cache) + if lines is None: + return [f"{prefix}: evidence file could not be read: {path}"] + + if line_no < 1 or line_no > len(lines): + return [f"{prefix}: evidence cites line {line_no} but file has only {len(lines)} lines"] + + if not isinstance(quote, str) or not quote: + return [f"{prefix}: 'quote' must be a non-empty string"] + + cited_line = " ".join(lines[line_no - 1].split()).lower() + normalized_quote = " ".join(quote.split()).lower() + if normalized_quote not in cited_line: + return [f"{prefix}: quote not found on line {line_no}"] + + return [] + + +def validate_entry(entry, index, file_cache): + errors = [] + + missing = REQUIRED_FIELDS - set(entry.keys()) + if missing: + errors.append(f"entry[{index}]: missing required fields: {', '.join(sorted(missing))}") + + for field in NON_EMPTY_STRING_FIELDS: + val = entry.get(field) + if not isinstance(val, str) or not val: + errors.append(f"entry[{index}]: '{field}' must be a non-empty string") + + sev = entry.get("severity") + if isinstance(sev, bool) or not isinstance(sev, int) or not (1 <= sev <= 5): + errors.append(f"entry[{index}]: 'severity' must be an integer 1-5, got {sev!r}") + + infra = entry.get("infrastructure_failure") + if not isinstance(infra, bool): + errors.append(f"entry[{index}]: 'infrastructure_failure' must be a boolean, got {type(infra).__name__}") + + layer = entry.get("stack_layer") + if not isinstance(layer, str) or layer not in VALID_STACK_LAYERS: + errors.append(f"entry[{index}]: 'stack_layer' must be one of {sorted(VALID_STACK_LAYERS)}, got {layer!r}") + + conf = entry.get("confidence") + if not isinstance(conf, str) or conf not in VALID_CONFIDENCE: + errors.append(f"entry[{index}]: 'confidence' must be one of {sorted(VALID_CONFIDENCE)}, got {conf!r}") + + chain = entry.get("causal_chain") + if not isinstance(chain, list): + errors.append(f"entry[{index}]: 'causal_chain' must be a non-empty array, got {type(chain).__name__}") + elif not chain: + errors.append(f"entry[{index}]: 'causal_chain' must be a non-empty array") + else: + for ci, link in enumerate(chain): + if not isinstance(link, dict): + errors.append(f"entry[{index}].causal_chain[{ci}]: must be an object") + continue + for key in ("cause", "evidence", "quote"): + val = link.get(key) + if not isinstance(val, str) or not val: + errors.append(f"entry[{index}].causal_chain[{ci}]: '{key}' must be a non-empty string") + evidence = link.get("evidence", "") + quote = link.get("quote", "") + if isinstance(evidence, str) and evidence: + errors.extend(validate_evidence( + evidence, quote, + f"entry[{index}].causal_chain[{ci}]", file_cache)) + + for field in ("analysis_gaps", "scenarios"): + val = entry.get(field) + if not isinstance(val, list): + errors.append(f"entry[{index}]: '{field}' must be an array, got {type(val).__name__}") + elif any(not isinstance(item, str) for item in val): + errors.append(f"entry[{index}]: '{field}' items must all be strings") + + scenarios = entry.get("scenarios") + layer = entry.get("stack_layer", "") + if isinstance(scenarios, list) and not scenarios and layer == "test": + errors.append( + f"entry[{index}]: 'scenarios' is empty but stack_layer is 'test' — " + "populate with the names of the failing test cases" + ) + + return errors + + +def validate_json_text(text): + try: + data = json.loads(text) + except json.JSONDecodeError as e: + if "--- STRUCTURED SUMMARY ---" in text: + return [ + "Output contains prose and STRUCTURED SUMMARY markers. " + "Your entire response must be a valid JSON array only — no prose, no markers." + ] + return [f"Output is not valid JSON: {e}. Your entire response must be a valid JSON array."] + + if isinstance(data, dict): + return [ + "Output is a JSON object, not an array. " + "Wrap your output in [...] — single failures must still be a JSON array." + ] + elif not isinstance(data, list): + return [f"Expected a JSON array, got {type(data).__name__}"] + + if not data: + return ["JSON array is empty. Expected at least one failure entry."] + + file_cache = {} + all_errors = [] + for i, entry in enumerate(data): + if not isinstance(entry, dict): + all_errors.append(f"entry[{i}]: expected an object, got {type(entry).__name__}") + continue + all_errors.extend(validate_entry(entry, i, file_cache)) + + return all_errors + + +def validate_message(message): + if not message or not message.strip(): + return ["Agent produced empty output. Expected a JSON array."] + + return validate_json_text(message.strip()) + + +def main(): + try: + payload = json.load(sys.stdin) + except (json.JSONDecodeError, ValueError): + print("WARNING: validate-rca-output: malformed JSON on stdin, skipping validation", file=sys.stderr) + sys.exit(0) + + if not isinstance(payload, dict): + print("WARNING: validate-rca-output: expected dict payload, skipping validation", file=sys.stderr) + sys.exit(0) + + message = payload.get("last_assistant_message", "") + errors = validate_message(message) + + if errors: + reason = "RCA output validation failed:\n" + "\n".join(f" - {e}" for e in errors) + json.dump({"decision": "block", "reason": reason}, sys.stdout) + + sys.exit(0) + + +if __name__ == "__main__": + main()