diff --git a/.gitignore b/.gitignore index 42eab3b1..d28fa9f8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ data/* logs/* coverage_output/* sbom/* +scripts/pr-check/config.env requirements.txt **/__pycache__/* integration-tests/Output/* diff --git a/README.md b/README.md index c075fee1..c48798de 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Replace `HEAD` with a hash, branch, or `main~3` as needed. - [Build](#build) - [Build from an LLVM pull request](#build-from-an-llvm-pull-request) - [Workflow 2: PR coverage gap detection](#workflow-2-pr-coverage-gap-detection) + - [Periodic PR checking](#periodic-pr-checking) - [Run integration tests](#run-integration-tests) - [Run a container](#run-a-container) - [Tests](#tests) @@ -404,6 +405,21 @@ If the image `fuzz-fill-test:llvm-pr-` already exists, omit `--build-image` t Main output: `/commit_lines_report/target_lines_uncovered.csv`. See [Workflow 2](#workflow-2-uncovered-lines-in-a-commit) for report semantics. +### Periodic PR checking + +To automatically check open AMDGPU and SPIR-V PRs on a schedule, use [`scripts/pr-check/check-llvm-prs.sh`](scripts/pr-check/check-llvm-prs.sh). It discovers PRs via `gh`, re-runs gap detection when a PR head SHA changes, and writes local reports under `data/pr-check/reports/`. + +For a **daily AMDGPU-only** cron job, use [`scripts/pr-check/run-daily-amdgpu.sh`](scripts/pr-check/run-daily-amdgpu.sh) (see [`scripts/pr-check/README.md`](scripts/pr-check/README.md)). + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Set LLVM_REPO in config.env, then: +./scripts/pr-check/check-llvm-prs.sh --discover-only +./scripts/pr-check/check-llvm-prs.sh +``` + +See [`scripts/pr-check/README.md`](scripts/pr-check/README.md) for cron setup, configuration options, and output layout. + ### Run integration tests ```bash diff --git a/pyproject.toml b/pyproject.toml index b1c43365..9be86551 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ reduce = "reduce.__main__:main" llvm-test-suite-coverage = "coverage.__main__:main" added-lines = "added_lines.__main__:main" +pr-check = "pr_check.__main__:main" [tool.setuptools] package-dir = { "" = "src" } diff --git a/scripts/build-llvm-sancov.sh b/scripts/build-llvm-sancov.sh index 15bf88c6..99fc347d 100755 --- a/scripts/build-llvm-sancov.sh +++ b/scripts/build-llvm-sancov.sh @@ -158,7 +158,8 @@ LLVM_CMAKE_BASE=( -G Ninja -DCMAKE_C_COMPILER="$C_COMPILER" -DCMAKE_CXX_COMPILER="$CXX_COMPILER" - -DLLVM_TARGETS_TO_BUILD="X86;AMDGPU;SPIRV" + -DLLVM_TARGETS_TO_BUILD="X86;AMDGPU" + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV" -DLLVM_ENABLE_PROJECTS="" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_SPLIT_DWARF=ON diff --git a/scripts/docker/pr-cov-gaps-detection.sh b/scripts/docker/pr-cov-gaps-detection.sh index 35dc9c2b..edbcf5f9 100755 --- a/scripts/docker/pr-cov-gaps-detection.sh +++ b/scripts/docker/pr-cov-gaps-detection.sh @@ -38,8 +38,8 @@ Required: Options: --build-image Build PR image via build-image-pr.sh before detection - --keep-image Keep the PR image after detection (default: remove it - when --build-image was used) + --keep-image Keep the PR image after detection when used with + --build-image (no effect when the image is pre-built) --llvm-repo Local llvm-project clone (required with --build-image) --backend-tests amdgpu or spirv (required with --build-image) --github-repo @@ -215,8 +215,8 @@ if [[ -z "$output_dir" ]]; then fi if [[ "$build_image" -eq 0 ]]; then - if [[ -n "$llvm_repo" || -n "$backend_tests" || -n "$github_repo" || "$keep_image" -eq 1 ]]; then - echo "error: --llvm-repo, --backend-tests, --github-repo, and --keep-image require --build-image" >&2 + if [[ -n "$llvm_repo" || -n "$backend_tests" || -n "$github_repo" ]]; then + echo "error: --llvm-repo, --backend-tests, and --github-repo require --build-image" >&2 exit 1 fi else diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md new file mode 100644 index 00000000..8290fb08 --- /dev/null +++ b/scripts/pr-check/README.md @@ -0,0 +1,229 @@ +# Periodic LLVM PR coverage gap checking + +Automatically discover open AMDGPU and SPIR-V PRs on `llvm/llvm-project`, run [Workflow 2](../docker/pr-cov-gaps-detection.sh) coverage-gap detection when a PR's head SHA changes, and write local reports under `data/pr-check/reports/`. + +## Prerequisites + +- [Docker](https://docs.docker.com/) with BuildKit +- [GitHub CLI](https://cli.github.com/) (`gh auth login`) +- `git` +- A local `llvm-project` clone (used as a reference for faster PR fetches; PR content is fetched from GitHub) +- Python 3.10+ with fuzz-fill dependencies installed (`pip install -e .`) + +## One-time setup + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Edit config.env and set LLVM_REPO +``` + +`config.env` is local-only (not committed). Paths in the example are relative to the fuzz-fill repo root. + +## Usage + +Run from the fuzz-fill repo root: + +```bash +# List open PRs touching AMDGPU or SPIR-V target paths +./scripts/pr-check/check-llvm-prs.sh --discover-only + +# Show planned work (PR/backend pairs needing a fresh check) +./scripts/pr-check/check-llvm-prs.sh --plan-only + +# Refresh latest.json / latest.md from state (no checks, no LLVM_REPO needed) +./scripts/pr-check/get-latest.sh + +# Same as get-latest.sh, via the main orchestrator flag +./scripts/pr-check/check-llvm-prs.sh --report-only + +# Discover, plan, run up to PR_CHECK_MAX_PER_RUN checks, and write reports +./scripts/pr-check/check-llvm-prs.sh + +# AMDGPU only, or drain all pending checks in one run +./scripts/pr-check/check-llvm-prs.sh --backends amdgpu +./scripts/pr-check/check-llvm-prs.sh --backends amdgpu --drain-queue +``` + +Override config path: + +```bash +./scripts/pr-check/check-llvm-prs.sh --config /path/to/my-config.env +``` + +Daily cron entry point (AMDGPU only, drains the full queue): + +```bash +./scripts/pr-check/run-daily-amdgpu.sh +``` + +## What each run does + +1. **Discover** — `gh search prs` for open PRs with label `backend:AMDGPU` and/or `backend:SPIRV`, then keep only those with changed files under `llvm/lib/Target/AMDGPU/` or `llvm/lib/Target/SPIRV/` +2. **Plan** — compare against `state.json`; queue entries that are new or have a changed head SHA +3. **Check** — for each planned `(PR, backend)` pair (up to `PR_CHECK_MAX_PER_RUN`): + - Build Docker image `fuzz-fill-test:llvm-pr--` via [`build-image-pr.sh`](../docker/build-image-pr.sh) + - Run [`pr-cov-gaps-detection.sh`](../docker/pr-cov-gaps-detection.sh) + - Record `gaps`, `clean`, or `failed` in state +4. **Report** — write `data/pr-check/reports/latest.json`, `latest.md`, and `new-prs.md` + +### CLI options + +| Flag / env var | Default | Purpose | +|----------------|---------|---------| +| `--backends` / `PR_CHECK_BACKENDS` | amdgpu, spirv | Limit discovery to specific backends | +| `--max-age-days` / `PR_CHECK_MAX_AGE_DAYS` | 14 | Only PRs opened within the last N days | +| `--max-per-run` / `PR_CHECK_MAX_PER_RUN` | 1 | Max checks per invocation (ignored with `--drain-queue`) | +| `--drain-queue` | off | Process all pending checks one at a time until empty | + +## Output layout + +| Path | Description | +|------|-------------| +| `data/pr-check/state.json` | Persistent check history keyed by `":"` | +| `data/pr-check/runs/-/` | Per-check artifacts (`baseline/`, `commit_lines_report/`, …) | +| `data/pr-check/reports/latest.json` | Machine-readable summary of all checked PRs | +| `data/pr-check/reports/latest.md` | Human-readable gap report with PR links | +| `data/pr-check/reports/new-prs.md` | PRs new or updated since the previous report (diff vs prior `latest.json`) | +| `data/pr-check/reports/runs/.json` | Snapshot from each report generation | + +A PR has **coverage gaps** when `target_lines_uncovered.csv` is non-empty (added lines not covered by the regression suite). + +## Refreshing reports + +Reports are built from `state.json`, not by scanning `runs/`. After a failed or interrupted check run, or anytime you want the summary synced to current state: + +```bash +./scripts/pr-check/get-latest.sh +``` + +Use this when `latest.md` looks stale (e.g. fewer entries than `state.json`, or checks finished but the orchestrator exited early). The markdown body lists only PRs **with gaps**; clean PRs appear in the summary line and in `latest.json`. The same refresh also updates `new-prs.md` by diffing the current state against the previous `latest.json` — useful after an interrupted run where checks completed but report generation did not. + +### Rebuilding `state.json` from run artifacts + +If `state.json` is missing or corrupted (for example after a disk-full write), rebuild it from completed directories under `data/pr-check/runs/` without re-running coverage checks: + +```bash +# Preview what would be written +./scripts/pr-check/rebuild-state.sh --dry-run + +# Rebuild state, then refresh reports +./scripts/pr-check/rebuild-state.sh +./scripts/pr-check/get-latest.sh +``` + +The rebuild: + +1. Scans `data/pr-check/runs/-/` and skips incomplete directories (no `commit_lines_report/target_lines_uncovered.csv`). +2. Recomputes `gap_count`, `lit_failure_count`, and `status` from on-disk artifacts. +3. Fills `title`, `head_sha`, and `checked_at` from `reports/latest.json` and `reports/runs/*.json` when available. +4. Falls back to `gh pr view` for any remaining runs (uses the **current** PR head SHA — if a PR was updated after the run, `--plan-only` may queue a re-check). + +Incomplete or empty run directories (for example a check interrupted before detection finished) are skipped and are not added to state. + +The main orchestrator also refreshes reports at normal exit. An EXIT trap regenerates the report if at least one check ran but the process did not finish cleanly (Ctrl+C, OOM, or unexpected early exit during `--drain-queue`). + +## Daily AMDGPU cron + +For a once-daily job that checks all pending AMDGPU PRs opened within the last 14 days: + +### One-time setup + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Set LLVM_REPO to an absolute path (required for cron) +# Ensure gh auth login and Docker work for the cron user +pip install -e . +``` + +`config.env` is local-only (not committed). Use absolute paths in cron configs. + +### Test before scheduling + +```bash +# See what would run (no LLVM builds) +./scripts/pr-check/check-llvm-prs.sh \ + --config scripts/pr-check/config.env \ + --backends amdgpu \ + --plan-only + +# Full daily run (builds LLVM Docker images — can take hours on first backfill) +./scripts/pr-check/run-daily-amdgpu.sh +``` + +### Install cron + +See [`cron/amdgpu-daily.example`](cron/amdgpu-daily.example) for a ready-to-edit crontab snippet. Typical install: + +```bash +crontab -e +# Add the line from cron/amdgpu-daily.example (with your paths) +``` + +The wrapper [`run-daily-amdgpu.sh`](run-daily-amdgpu.sh) appends logs to `logs/pr-check/amdgpu-daily.log` (gitignored, no sudo required) and uses `flock` to skip if a prior run is still going. + +### Expected behavior + +| Run | What happens | +|-----|--------------| +| **First daily run** | Discovers all open AMDGPU PRs from the last 14 days; checks every PR not yet in `state.json` (or with a changed head SHA). May take many hours. | +| **Subsequent days** | Same 14-day discovery window, but most PRs are skipped (unchanged SHA). Only new or updated PRs are checked. | + +Results land in `data/pr-check/reports/latest.md`. + +## Lightweight polling cron + +For frequent runs that process one PR at a time (both backends): + +```cron +# Every 6 hours: run at most one PR/backend check, then refresh reports +0 */6 * * * cd /path/to/fuzz-fill && ./scripts/pr-check/check-llvm-prs.sh >> logs/pr-check/polling.log 2>&1 +``` + +Keep `PR_CHECK_MAX_PER_RUN=1` unless you have enough CPU/time for multiple LLVM Docker builds per invocation. + +## systemd timer example + +`/etc/systemd/system/fuzz-fill-pr-check.service`: + +```ini +[Unit] +Description=fuzz-fill periodic LLVM PR coverage gap check + +[Service] +Type=oneshot +WorkingDirectory=/path/to/fuzz-fill +ExecStart=/path/to/fuzz-fill/scripts/pr-check/check-llvm-prs.sh +User=your-user +``` + +`/etc/systemd/system/fuzz-fill-pr-check.timer`: + +```ini +[Unit] +Description=Run fuzz-fill PR coverage checks every 6 hours + +[Timer] +OnCalendar=*-*-* 00,06,12,18:00:00 +Persistent=true + +[Install] +WantedBy=timers.target +``` + +Enable with: + +```bash +sudo systemctl enable --now fuzz-fill-pr-check.timer +``` + +## Low-level CLI + +The orchestrator wraps [`src/pr_check/checker.py`](../../src/pr_check/checker.py): + +```bash +PYTHONPATH=src python3 -m pr_check discover +PYTHONPATH=src python3 -m pr_check plan --state-file data/pr-check/state.json +PYTHONPATH=src python3 -m pr_check report \ + --state-file data/pr-check/state.json \ + --report-dir data/pr-check/reports +``` diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh new file mode 100755 index 00000000..55ac06c6 --- /dev/null +++ b/scripts/pr-check/check-llvm-prs.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash +# Periodic orchestrator for LLVM AMDGPU/SPIR-V PR coverage-gap detection. +# +# Discovers open target PRs, plans work against state.json, and runs +# pr-cov-gaps-detection.sh for PR/backend pairs whose head SHA changed. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +LLVM_REPO="${LLVM_REPO:-}" +PR_CHECK_JOBS="${PR_CHECK_JOBS:-$(nproc)}" +PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" +PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" +PR_CHECK_MAX_AGE_DAYS="${PR_CHECK_MAX_AGE_DAYS:-14}" +PR_CHECK_BACKENDS="${PR_CHECK_BACKENDS:-}" +PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" +PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" +PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +PR_CHECK_LOG_LEVEL="${PR_CHECK_LOG_LEVEL:-info}" +GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" +IMAGE_NAME="${IMAGE_NAME:-fuzz-fill-test}" + +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" + +discover_only=0 +plan_only=0 +report_only=0 +drain_queue=0 + +usage() { + cat < Load environment overrides from a file + (default: ${SCRIPT_DIR}/config.env if present) + --llvm-repo Local llvm-project clone (required unless set in config) + --state-file State file (default: ${PR_CHECK_STATE_FILE}) + --output-root Per-run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) + --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) + --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) + --drain-queue Process all pending checks (one at a time until queue empty) + --max-age-days Only PRs opened in the last N days (default: ${PR_CHECK_MAX_AGE_DAYS}) + --backends Comma-separated backends to check (default: amdgpu,spirv) + --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) + --log-level Python log level: debug, info, warning, error + (default: ${PR_CHECK_LOG_LEVEL}) + -v, --verbose Shorthand for --log-level debug + --github-repo + GitHub repo hosting PRs (default: ${GITHUB_REPO}) + --help, -h Show this help + +Configuration: + Copy ${SCRIPT_DIR}/config.example.env to ${SCRIPT_DIR}/config.env and set + LLVM_REPO before running under cron. + +Examples: + $(basename "$0") --discover-only + $(basename "$0") --plan-only + $(basename "$0") --llvm-repo /path/to/llvm-project +EOF +} + +load_config() { + if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck source=/dev/null + source "$CONFIG_FILE" + fi +} + +run_pr_check() { + PYTHONPATH="${REPO_ROOT}/src" python3 -m pr_check \ + "$@" \ + --log-level "$PR_CHECK_LOG_LEVEL" +} + +report_generated=0 + +generate_report() { + report_generated=1 + echo "=== coverage gap report ===" + run_pr_check report \ + --github-repo "$GITHUB_REPO" \ + --state-file "$PR_CHECK_STATE_FILE" \ + --report-dir "$PR_CHECK_REPORT_DIR" +} + +cleanup_report() { + if [[ "$report_generated" -eq 0 && "$total_checks" -gt 0 ]]; then + generate_report || true + fi +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +validate_positive_int() { + local name="$1" + local value="$2" + if [[ ! "$value" =~ ^[0-9]+$ ]] || [[ "$value" -eq 0 ]]; then + echo "error: ${name} must be a positive integer: ${value}" >&2 + exit 1 + fi +} + +validate_runtime_config() { + if [[ -z "$LLVM_REPO" ]]; then + echo "error: LLVM_REPO is required (set in config.env or pass --llvm-repo)" >&2 + exit 1 + fi + if [[ ! -d "$LLVM_REPO" ]]; then + echo "error: LLVM_REPO is not a directory: ${LLVM_REPO}" >&2 + exit 1 + fi + validate_positive_int "PR_CHECK_JOBS" "$PR_CHECK_JOBS" + validate_positive_int "PR_CHECK_MAX_PER_RUN" "$PR_CHECK_MAX_PER_RUN" + validate_positive_int "PR_CHECK_SEARCH_LIMIT" "$PR_CHECK_SEARCH_LIMIT" + validate_positive_int "PR_CHECK_MAX_AGE_DAYS" "$PR_CHECK_MAX_AGE_DAYS" +} + +record_failed_run() { + local pr_number="$1" + local backend="$2" + local title="$3" + local head_sha="$4" + local output_dir="$5" + local error_message="$6" + + run_pr_check record \ + --state-file "$PR_CHECK_STATE_FILE" \ + --pr-number "$pr_number" \ + --backend "$backend" \ + --title "$title" \ + --head-sha "$head_sha" \ + --status failed \ + --gap-count 0 \ + --lit-failure-count 0 \ + --output-dir "$output_dir" \ + --error "$error_message" +} + +run_one_work_item() { + local pr_number="$1" + local backend="$2" + local title="$3" + local head_sha="$4" + + local output_dir="${PR_CHECK_OUTPUT_ROOT}/${pr_number}-${backend}" + local image_tag="llvm-pr-${pr_number}-${backend}" + local image_ref="${IMAGE_NAME}:${image_tag}" + + mkdir -p "$output_dir" + output_dir="$(realpath "$output_dir")" + + echo "=== PR #${pr_number} (${backend}) ===" + echo "Title: ${title}" + echo "Head: ${head_sha}" + echo "Output: ${output_dir}" + + set +e + "${REPO_ROOT}/scripts/docker/build-image-pr.sh" \ + --llvm-repo "$LLVM_REPO" \ + --pr-id "$pr_number" \ + --allowlist "$backend" \ + --github-repo "$GITHUB_REPO" \ + --tag "$image_tag" \ + -j "$PR_CHECK_JOBS" + build_status=$? + + if [[ "$build_status" -ne 0 ]]; then + set -e + record_failed_run "$pr_number" "$backend" "$title" "$head_sha" "$output_dir" "docker image build failed" + return "$build_status" + fi + + "${REPO_ROOT}/scripts/docker/pr-cov-gaps-detection.sh" \ + --image "$image_ref" \ + --output-dir "$output_dir" \ + --keep-image \ + -j "$PR_CHECK_JOBS" + detect_status=$? + set -e + + if [[ "$detect_status" -ne 0 ]]; then + record_failed_run "$pr_number" "$backend" "$title" "$head_sha" "$output_dir" "coverage-gap detection failed" + return "$detect_status" + fi + + local evaluation + evaluation="$(run_pr_check evaluate-output --output-dir "$output_dir")" + local gap_count lit_failure_count status + gap_count="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["gap_count"])' <<<"$evaluation")" + lit_failure_count="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["lit_failure_count"])' <<<"$evaluation")" + status="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])' <<<"$evaluation")" + + run_pr_check record \ + --state-file "$PR_CHECK_STATE_FILE" \ + --pr-number "$pr_number" \ + --backend "$backend" \ + --title "$title" \ + --head-sha "$head_sha" \ + --status "$status" \ + --gap-count "$gap_count" \ + --lit-failure-count "$lit_failure_count" \ + --output-dir "$output_dir" + + echo "Result: status=${status} gap_count=${gap_count} lit_failures=${lit_failure_count}" +} + +plan_work_json() { + local max_items="$1" + local plan_args=( + "${common_args[@]}" + --state-file "$PR_CHECK_STATE_FILE" + --limit "$PR_CHECK_SEARCH_LIMIT" + ) + if [[ -n "$max_items" ]]; then + plan_args+=(--max-items "$max_items") + fi + run_pr_check plan "${plan_args[@]}" +} + +work_count_from_json() { + local work_json="$1" + python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$work_json" +} + +run_work_items() { + local work_json="$1" + + while IFS= read -r item; do + local pr_number backend title head_sha reason + pr_number="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["pr_number"])' "$item")" + backend="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["backend"])' "$item")" + title="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["title"])' "$item")" + head_sha="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["head_sha"])' "$item")" + reason="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["reason"])' "$item")" + + echo + echo ">>> Running check (${reason}): #${pr_number} ${backend}" + if ! run_one_work_item "$pr_number" "$backend" "$title" "$head_sha"; then + failures=$((failures + 1)) + fi + done < <(python3 -c 'import json,sys; print("\n".join(json.dumps(x) for x in json.load(sys.stdin)))' <<<"$work_json") +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --discover-only) + discover_only=1 + shift + ;; + --plan-only) + plan_only=1 + shift + ;; + --report-only) + report_only=1 + shift + ;; + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --llvm-repo) + [[ $# -ge 2 ]] || { echo "error: --llvm-repo requires a value" >&2; exit 2; } + LLVM_REPO="$2" + shift 2 + ;; + --state-file) + [[ $# -ge 2 ]] || { echo "error: --state-file requires a value" >&2; exit 2; } + PR_CHECK_STATE_FILE="$2" + shift 2 + ;; + --output-root) + [[ $# -ge 2 ]] || { echo "error: --output-root requires a value" >&2; exit 2; } + PR_CHECK_OUTPUT_ROOT="$2" + shift 2 + ;; + --report-dir) + [[ $# -ge 2 ]] || { echo "error: --report-dir requires a value" >&2; exit 2; } + PR_CHECK_REPORT_DIR="$2" + shift 2 + ;; + --max-per-run) + [[ $# -ge 2 ]] || { echo "error: --max-per-run requires a value" >&2; exit 2; } + PR_CHECK_MAX_PER_RUN="$2" + shift 2 + ;; + --drain-queue) + drain_queue=1 + shift + ;; + --max-age-days) + [[ $# -ge 2 ]] || { echo "error: --max-age-days requires a value" >&2; exit 2; } + PR_CHECK_MAX_AGE_DAYS="$2" + shift 2 + ;; + --backends) + [[ $# -ge 2 ]] || { echo "error: --backends requires a value" >&2; exit 2; } + PR_CHECK_BACKENDS="$2" + shift 2 + ;; + --jobs) + [[ $# -ge 2 ]] || { echo "error: --jobs requires a value" >&2; exit 2; } + PR_CHECK_JOBS="$2" + shift 2 + ;; + --log-level) + [[ $# -ge 2 ]] || { echo "error: --log-level requires a value" >&2; exit 2; } + PR_CHECK_LOG_LEVEL="$2" + shift 2 + ;; + -v|--verbose) + PR_CHECK_LOG_LEVEL="debug" + shift + ;; + --github-repo) + [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; } + GITHUB_REPO="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ $# -gt 0 ]]; then + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 +fi + +load_config +require_command python3 +require_command docker +require_command gh +require_command git + +mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" "$PR_CHECK_OUTPUT_ROOT" "$PR_CHECK_REPORT_DIR" + +common_args=( + --github-repo "$GITHUB_REPO" + --max-age-days "$PR_CHECK_MAX_AGE_DAYS" +) +if [[ -n "$PR_CHECK_BACKENDS" ]]; then + common_args+=(--backends "$PR_CHECK_BACKENDS") +fi + +if [[ "$report_only" -eq 1 ]]; then + generate_report + exit 0 +fi + +if [[ "$discover_only" -eq 1 ]]; then + echo "=== discover open PRs (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 + run_pr_check discover "${common_args[@]}" --limit "$PR_CHECK_SEARCH_LIMIT" + exit 0 +fi + +if [[ "$plan_only" -eq 1 ]]; then + echo "=== plan PR checks (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 + run_pr_check plan \ + "${common_args[@]}" \ + --state-file "$PR_CHECK_STATE_FILE" \ + --limit "$PR_CHECK_SEARCH_LIMIT" \ + --max-items "$PR_CHECK_MAX_PER_RUN" + exit 0 +fi + +validate_runtime_config + +echo "=== automated PR check run (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 + +failures=0 +total_checks=0 +trap cleanup_report EXIT + +if [[ "$drain_queue" -eq 1 ]]; then + echo "Draining queue (one check at a time until empty)..." >&2 + while true; do + work_json="$(plan_work_json 1)" + work_count="$(work_count_from_json "$work_json")" + if [[ "$work_count" -eq 0 ]]; then + break + fi + + total_checks=$((total_checks + work_count)) + echo "Planned check ${total_checks} (draining queue)..." >&2 + run_work_items "$work_json" + done + + if [[ "$total_checks" -eq 0 ]]; then + echo "No PR/backend pairs need checking." + else + echo + echo "Completed ${total_checks} check(s) while draining queue." + fi +else + echo "Discovering and planning work (max ${PR_CHECK_MAX_PER_RUN} check(s) this run)..." >&2 + + work_json="$(plan_work_json "$PR_CHECK_MAX_PER_RUN")" + work_count="$(work_count_from_json "$work_json")" + if [[ "$work_count" -eq 0 ]]; then + echo "No PR/backend pairs need checking." + generate_report + exit 0 + fi + + echo "Planned ${work_count} PR/backend check(s)." + total_checks=$work_count + run_work_items "$work_json" +fi + +if [[ "$total_checks" -eq 0 ]]; then + generate_report + exit 0 +fi + +if [[ "$failures" -gt 0 ]]; then + echo + echo "${failures} check(s) failed (see state file for details)." + generate_report + exit 1 +fi + +echo +echo "All planned checks completed successfully." +generate_report diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env new file mode 100644 index 00000000..de1721ef --- /dev/null +++ b/scripts/pr-check/config.example.env @@ -0,0 +1,38 @@ +# Copy this file to scripts/pr-check/config.env and adjust paths. +# Run check-llvm-prs.sh from the fuzz-fill repo root (or use absolute paths below). + +# Local llvm-project clone used as a git reference for PR image builds. +# PR content is still fetched fresh from GitHub; the clone can be stale. +LLVM_REPO=/path/to/llvm-project + +# Parallel jobs for Docker LLVM builds and llvm-lit baseline runs. +PR_CHECK_JOBS=$(nproc) + +# Max PR/backend checks per cron invocation (each first-time build compiles LLVM). +# Ignored when using run-daily-amdgpu.sh (it passes --drain-queue). +PR_CHECK_MAX_PER_RUN=1 + +# Max open PRs returned per backend search (AMDGPU and SPIR-V). +PR_CHECK_SEARCH_LIMIT=100 + +# Only include PRs opened within this many days (GitHub created:> filter). +# Used by the daily AMDGPU cron wrapper and --max-age-days on check-llvm-prs.sh. +PR_CHECK_MAX_AGE_DAYS=14 + +# Optional: limit discovery/planning to specific backends (comma-separated). +# Example for AMDGPU-only: PR_CHECK_BACKENDS=amdgpu +# PR_CHECK_BACKENDS= + +# Host paths for state, per-run artifacts, and aggregated reports. +PR_CHECK_OUTPUT_ROOT=data/pr-check/runs +PR_CHECK_STATE_FILE=data/pr-check/state.json +PR_CHECK_REPORT_DIR=data/pr-check/reports + +# Optional: log level for pr_check progress messages on stderr (debug, info, warning, error). +PR_CHECK_LOG_LEVEL=info + +# Optional: override the GitHub repo hosting PRs (default: llvm/llvm-project). +# GITHUB_REPO=llvm/llvm-project + +# Optional: personal access token for higher GitHub API rate limits. +# GITHUB_TOKEN= diff --git a/scripts/pr-check/cron/amdgpu-daily.example b/scripts/pr-check/cron/amdgpu-daily.example new file mode 100644 index 00000000..251d543e --- /dev/null +++ b/scripts/pr-check/cron/amdgpu-daily.example @@ -0,0 +1,42 @@ +# Example crontab entry for daily AMDGPU PR coverage-gap checking. +# +# Install: +# 1. Copy scripts/pr-check/config.example.env to scripts/pr-check/config.env +# 2. Set LLVM_REPO to an absolute path in config.env +# 3. Ensure gh auth login and Docker work for the cron user +# 4. Install the schedule into *your* crontab (runs as your user, not root): +# a. Open your crontab file in an editor: +# crontab -e +# (First time: pick an editor, e.g. nano. Opens ~/.crontab or a temp file.) +# b. Copy the cron line from step 5 below (the line starting with "0 2"). +# c. Replace /home/USER with your home directory, e.g. for user agorzyns: +# /home/agorzyns/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh +# Tip: run `realpath scripts/pr-check/run-daily-amdgpu.sh` from the repo root. +# d. Paste the edited line as a new line at the end of the file. +# e. Save and exit the editor (in nano: Ctrl+O, Enter, Ctrl+X). +# f. Confirm it was installed: +# crontab -l +# You should see your new line listed. +# +# 5. Cron line to paste (edit the path first — see 4c): +# Logs append to logs/pr-check/amdgpu-daily.log under the fuzz-fill repo +# (already gitignored — no sudo or /var/log needed). +# +# Optional: email on failure (if mail is configured on the host): +# MAILTO=you@example.com + +# Daily at 02:00 UTC — drain all pending AMDGPU PRs opened in the last 14 days. +0 2 * * * /home/USER/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh + +# Cron schedule format: minute hour day-of-month month day-of-week command +# "0 2 * * *" = every day at 02:00 (server local time; check `date` if unsure). +# Notes: +# - Use an absolute path to run-daily-amdgpu.sh. +# - First run may take many hours (backfill within the 14-day window). +# - Subsequent runs only check new PRs or PRs with changed head SHAs. +# - Results: data/pr-check/reports/latest.md under the fuzz-fill repo. +# - If a prior run is still going, flock skips the new invocation (exit 0). +# - Dry run without builds: +# /home/USER/local/dev/fuzz-fill/scripts/pr-check/check-llvm-prs.sh \ +# --config /home/USER/local/dev/fuzz-fill/scripts/pr-check/config.env \ +# --backends amdgpu --plan-only diff --git a/scripts/pr-check/get-latest.sh b/scripts/pr-check/get-latest.sh new file mode 100755 index 00000000..63cd1303 --- /dev/null +++ b/scripts/pr-check/get-latest.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Regenerate latest.json / latest.md from data/pr-check/state.json. +# +# Does not run PR checks or require LLVM_REPO — only refreshes the report +# summary from whatever is already recorded in state. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" +ORCHESTRATOR="${SCRIPT_DIR}/check-llvm-prs.sh" +REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" + +usage() { + cat < Load path overrides from a config file + (default: ${CONFIG_FILE} if present) + --help, -h Show this help + +Environment: + PR_CHECK_CONFIG Override config file path (default: ${CONFIG_FILE}) + PR_CHECK_REPORT_DIR Report output directory (default: data/pr-check/reports) + +Examples: + $(basename "$0") + $(basename "$0") --config /path/to/config.env +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ ! -x "$ORCHESTRATOR" ]]; then + echo "error: orchestrator not found or not executable: ${ORCHESTRATOR}" >&2 + exit 1 +fi + +orchestrator_args=(--report-only) +if [[ -f "$CONFIG_FILE" ]]; then + orchestrator_args+=(--config "$CONFIG_FILE") + # shellcheck source=/dev/null + source "$CONFIG_FILE" + REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +fi + +cd "$REPO_ROOT" +"${ORCHESTRATOR}" "${orchestrator_args[@]}" + +echo +echo "Report updated:" +echo " ${REPORT_DIR}/latest.md" +echo " ${REPORT_DIR}/latest.json" +echo " ${REPORT_DIR}/new-prs.md" diff --git a/scripts/pr-check/rebuild-state.sh b/scripts/pr-check/rebuild-state.sh new file mode 100755 index 00000000..088ca228 --- /dev/null +++ b/scripts/pr-check/rebuild-state.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Rebuild data/pr-check/state.json from on-disk run artifacts. +# +# Scans data/pr-check/runs/-/, re-evaluates gap/LIT counts, and +# fills title/head_sha from saved reports (latest.json, runs/*.json) or GitHub. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" +PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" +PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +PR_CHECK_LOG_LEVEL="${PR_CHECK_LOG_LEVEL:-info}" +GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" + +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" + +usage() { + cat < Output state file (default: ${PR_CHECK_STATE_FILE}) + --output-root Run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) + --report-dir Report directory (default: ${PR_CHECK_REPORT_DIR}) + --config Load environment overrides from a file + --help, -h Show this help + +Examples: + $(basename "$0") --dry-run + $(basename "$0") + $(basename "$0") && ./scripts/pr-check/get-latest.sh +EOF +} + +load_config() { + if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck source=/dev/null + source "$CONFIG_FILE" + fi +} + +dry_run=0 +extra_args=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + dry_run=1 + extra_args+=(--dry-run) + shift + ;; + --no-fetch-pr-metadata) + extra_args+=(--no-fetch-pr-metadata) + shift + ;; + --state-file) + [[ $# -ge 2 ]] || { echo "error: --state-file requires a value" >&2; exit 2; } + PR_CHECK_STATE_FILE="$2" + shift 2 + ;; + --output-root) + [[ $# -ge 2 ]] || { echo "error: --output-root requires a value" >&2; exit 2; } + PR_CHECK_OUTPUT_ROOT="$2" + shift 2 + ;; + --report-dir) + [[ $# -ge 2 ]] || { echo "error: --report-dir requires a value" >&2; exit 2; } + PR_CHECK_REPORT_DIR="$2" + shift 2 + ;; + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +load_config +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +require_command python3 +if [[ "${extra_args[*]}" != *"--no-fetch-pr-metadata"* ]]; then + require_command gh +fi + +mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" + +PYTHONPATH="${REPO_ROOT}/src" python3 -m pr_check rebuild-state \ + --state-file "$PR_CHECK_STATE_FILE" \ + --output-root "$PR_CHECK_OUTPUT_ROOT" \ + --report-dir "$PR_CHECK_REPORT_DIR" \ + --github-repo "$GITHUB_REPO" \ + --log-level "$PR_CHECK_LOG_LEVEL" \ + "${extra_args[@]}" diff --git a/scripts/pr-check/run-daily-amdgpu.sh b/scripts/pr-check/run-daily-amdgpu.sh new file mode 100755 index 00000000..4408db94 --- /dev/null +++ b/scripts/pr-check/run-daily-amdgpu.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Daily cron entry point for AMDGPU-only LLVM PR coverage-gap checking. +# +# Uses flock to avoid overlapping runs, loads local config.env, and drains +# all pending AMDGPU PR checks opened within the last 14 days. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" +LOCK_FILE="${REPO_ROOT}/data/pr-check/.amdgpu-daily.lock" +LOG_FILE="${PR_CHECK_LOG_FILE:-${REPO_ROOT}/logs/pr-check/amdgpu-daily.log}" +ORCHESTRATOR="${SCRIPT_DIR}/check-llvm-prs.sh" + +log() { + printf '[%s] %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" >&2 +} + +usage() { + cat <&2 + usage >&2 + exit 2 +fi + +# Cron runs with a minimal PATH; include common install locations. +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${PATH:-}" + +mkdir -p "$(dirname "$LOG_FILE")" +exec > >(tee -a "$LOG_FILE") 2>&1 + +if [[ ! -x "$ORCHESTRATOR" ]]; then + log "error: orchestrator not found or not executable: ${ORCHESTRATOR}" + exit 1 +fi + +if [[ ! -f "$CONFIG_FILE" ]]; then + log "error: config file not found: ${CONFIG_FILE}" + log "Copy ${SCRIPT_DIR}/config.example.env to ${CONFIG_FILE} and set LLVM_REPO." + exit 1 +fi + +# shellcheck source=/dev/null +source "$CONFIG_FILE" + +if [[ -z "${LLVM_REPO:-}" ]]; then + log "error: LLVM_REPO is not set in ${CONFIG_FILE}" + exit 1 +fi + +if [[ "${LLVM_REPO}" != /* ]]; then + LLVM_REPO="${REPO_ROOT}/${LLVM_REPO}" + export LLVM_REPO + log "Resolved relative LLVM_REPO to ${LLVM_REPO}" +fi + +mkdir -p "$(dirname "$LOCK_FILE")" + +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log "Another daily AMDGPU PR check is already running; exiting." + exit 0 +fi + +log "Starting daily AMDGPU PR check (repo=${REPO_ROOT})" +cd "$REPO_ROOT" + +"${ORCHESTRATOR}" \ + --config "$CONFIG_FILE" \ + --backends amdgpu \ + --max-age-days "${PR_CHECK_MAX_AGE_DAYS:-14}" \ + --drain-queue + +log "Daily AMDGPU PR check finished." diff --git a/src/coverage/sancov.py b/src/coverage/sancov.py index 37fbe7d5..26118419 100644 --- a/src/coverage/sancov.py +++ b/src/coverage/sancov.py @@ -15,6 +15,7 @@ logger = get_logger("coverage.sancov") + class Sancov: @staticmethod @@ -352,16 +353,16 @@ def merge(self) -> None: with log_timing(logger, f"sancov merge ({self.suffix})"): merged_out = self.get_merged_sancov_path() - raw_files = list(self.raw_sancov_dir.glob(f"{self.suffix}.*.sancov")) + raw_files = sorted(self.raw_sancov_dir.glob(f"{self.suffix}.*.sancov")) if not raw_files: raise FileNotFoundError(f"No sancov files found for suffix: {self.suffix}") - """Merge raw ``.sancov`` files with repeated ``sancov -union`` (batched).""" if len(raw_files) == 1: shutil.copy(raw_files[0], merged_out) return - sancov = self.sancov_bin + logger.info("Merging %d raw sancov file(s) via sancov -union", len(raw_files)) + batch = self.union_batch layer = list(raw_files) with tempfile.TemporaryDirectory(dir=merged_out.parent) as tmp: @@ -375,17 +376,25 @@ def merge(self) -> None: nxt.append(chunk[0]) continue out_f = tmp_path / f"u_{round_idx}_{len(nxt)}.sancov" - run_subprocess( - logger, - [str(sancov), "-union"] - + [str(p) for p in chunk] - + ["--output", str(out_f)], - check=True, - ) + self._union_raw_sancov_chunk(chunk, out_f) nxt.append(out_f) layer = nxt round_idx += 1 shutil.copy(layer[0], merged_out) + + def _union_raw_sancov_chunk(self, inputs: list[Path], output: Path) -> None: + result = run_subprocess( + logger, + [str(self.sancov_bin), "-union", *[str(path) for path in inputs], "--output", str(output)], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + stderr = (result.stderr or result.stdout or "").strip() + raise RuntimeError( + f"sancov -union failed for {len(inputs)} file(s): {stderr or result.returncode}" + ) def symbolize( self, diff --git a/src/pr_check/__init__.py b/src/pr_check/__init__.py new file mode 100644 index 00000000..90fe763c --- /dev/null +++ b/src/pr_check/__init__.py @@ -0,0 +1 @@ +"""Periodic LLVM PR coverage gap checking helpers.""" diff --git a/src/pr_check/__main__.py b/src/pr_check/__main__.py new file mode 100644 index 00000000..75a273f1 --- /dev/null +++ b/src/pr_check/__main__.py @@ -0,0 +1,4 @@ +from .checker import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py new file mode 100644 index 00000000..a221f5fb --- /dev/null +++ b/src/pr_check/checker.py @@ -0,0 +1,1662 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Discover, plan, and record periodic LLVM PR coverage-gap checks.""" + +from __future__ import annotations + +import argparse +import json +import logging +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlencode + +import pandas as pd + +STATE_VERSION = 1 +DEFAULT_GITHUB_REPO = "llvm/llvm-project" +DEFAULT_SEARCH_LIMIT = 100 +DEFAULT_MAX_PR_AGE_DAYS = 14 + +BACKEND_SEARCH_QUERIES: dict[str, list[str]] = { + "amdgpu": [ + 'label:"backend:AMDGPU"', + ], + "spirv": [ + 'label:"backend:SPIR-V"', + ], +} + +BACKEND_TARGET_PATH_PREFIXES: dict[str, str] = { + "amdgpu": "llvm/lib/Target/AMDGPU/", + "spirv": "llvm/lib/Target/SPIRV/", +} + +SEARCH_JSON_FIELDS = ["number", "title", "updatedAt"] +PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt", "files"] +LIT_FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"}) + +LOG_FORMAT = "%(levelname)-8s %(message)s" +log = logging.getLogger("pr_check") + + +class PrCheckerError(Exception): + """Error from PR checker operations.""" + + +@dataclass(frozen=True) +class DiscoveredPr: + """An open PR that touches at least one tracked backend.""" + + pr_number: int + title: str + head_sha: str + updated_at: str + backends: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class WorkItem: + """A PR/backend pair that should be checked.""" + + pr_number: int + backend: str + title: str + head_sha: str + reason: str + + +@dataclass +class StateEntry: + """Persistent record for one PR/backend check.""" + + pr_number: int + backend: str + title: str + head_sha: str + status: str + gap_count: int + lit_failure_count: int + checked_at: str + output_dir: str + error: str | None = None + + +def entry_key(pr_number: int, backend: str) -> str: + return f"{pr_number}:{backend}" + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def configure_logging(level: str = "info") -> None: + """Send log records to stderr so stdout stays free for JSON payloads.""" + numeric_level = getattr(logging, level.upper(), None) + if not isinstance(numeric_level, int): + raise PrCheckerError(f"invalid log level: {level}") + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + + root = logging.getLogger("pr_check") + root.handlers.clear() + root.setLevel(numeric_level) + root.addHandler(handler) + root.propagate = False + + +def _validate_max_age_days(max_age_days: int) -> None: + if max_age_days <= 0: + raise PrCheckerError(f"max_age_days must be a positive integer: {max_age_days}") + + +def _parse_backends_arg(raw: str | None) -> list[str] | None: + """Parse a comma-separated backend list from CLI input.""" + if raw is None: + return None + names = [part.strip() for part in raw.split(",") if part.strip()] + if not names: + raise PrCheckerError("backends must list at least one backend when provided") + return _validate_backends(names) + + +def _validate_backends(backends: list[str]) -> list[str]: + unknown = sorted(set(backends) - set(BACKEND_SEARCH_QUERIES)) + if unknown: + raise PrCheckerError( + f"unknown backend(s): {', '.join(unknown)} " + f"(known: {', '.join(sorted(BACKEND_SEARCH_QUERIES))})" + ) + seen: set[str] = set() + resolved: list[str] = [] + for backend in backends: + if backend not in seen: + seen.add(backend) + resolved.append(backend) + return resolved + + +def _backends_to_search(backends: list[str] | None) -> list[str]: + if backends is None: + return list(BACKEND_SEARCH_QUERIES) + return _validate_backends(backends) + + +def _search_created_since(max_age_days: int) -> str: + """Return a YYYY-MM-DD date for GitHub ``created:>`` search qualifiers.""" + _validate_max_age_days(max_age_days) + cutoff = datetime.now(timezone.utc).date() - timedelta(days=max_age_days) + return cutoff.isoformat() + + +def _backend_search_terms(backend: str, *, max_age_days: int) -> list[str]: + """Build gh search terms for one backend, limited to recently opened PRs.""" + created_since = _search_created_since(max_age_days) + age_filter = f"created:>{created_since}" + return [f"{term} {age_filter}" for term in BACKEND_SEARCH_QUERIES[backend]] + + +def _pr_changed_paths(pr_view: dict[str, Any]) -> list[str]: + files = pr_view.get("files") + if not isinstance(files, list): + return [] + paths: list[str] = [] + for entry in files: + if not isinstance(entry, dict): + continue + path = entry.get("path") + if isinstance(path, str) and path: + paths.append(path) + return paths + + +def _pr_touches_backend_path(pr_view: dict[str, Any], backend: str) -> bool: + prefix = BACKEND_TARGET_PATH_PREFIXES[backend] + return any(path.startswith(prefix) for path in _pr_changed_paths(pr_view)) + + +def _merge_search_items(items_list: list[list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Merge GitHub search hits, keeping one row per PR number.""" + merged: dict[int, dict[str, Any]] = {} + for items in items_list: + for item in items: + number = item.get("number") + if number is None: + continue + pr_number = int(number) + existing = merged.get(pr_number) + if existing is None: + merged[pr_number] = dict(item) + continue + if item.get("title") and not existing.get("title"): + existing["title"] = item["title"] + if (item.get("updatedAt") or "") > (existing.get("updatedAt") or ""): + existing["updatedAt"] = item["updatedAt"] + return [merged[pr_number] for pr_number in sorted(merged)] + + +def _require_gh() -> str: + from shutil import which + + gh_path = which("gh") + if gh_path is None: + raise PrCheckerError("required command not found: gh") + return gh_path + + +def _run_gh(args: list[str], *, timeout: int = 120) -> str: + gh_path = _require_gh() + log.debug("running gh %s", " ".join(args)) + try: + result = subprocess.run( + [gh_path, *args], + capture_output=True, + text=True, + encoding="utf-8", + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise PrCheckerError(f"gh command timed out after {timeout}s: {' '.join(args)}") from exc + except OSError as exc: + raise PrCheckerError(f"failed to run gh: {exc}") from exc + + if result.returncode != 0: + stderr = (result.stderr or "").strip() or "(no error message)" + raise PrCheckerError(f"gh {' '.join(args)} failed: {stderr}") + + return result.stdout + + +def _gh_json(args: list[str], *, timeout: int = 120) -> Any: + stdout = _run_gh([*args, "--json", ",".join(_json_fields_for_command(args))], timeout=timeout) + if not stdout.strip(): + return [] + try: + return json.loads(stdout) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"gh returned invalid JSON: {exc.msg}") from exc + + +def _gh_api_json(endpoint: str) -> Any: + stdout = _run_gh(["api", endpoint]) + if not stdout.strip(): + return {} + try: + return json.loads(stdout) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"gh api returned invalid JSON: {exc.msg}") from exc + + +def _normalize_search_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Map GitHub issue-search items to the fields used elsewhere in this module.""" + normalized: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + normalized.append( + { + "number": item.get("number"), + "title": item.get("title") or "", + "updatedAt": item.get("updated_at") or "", + } + ) + return normalized + + +def _json_fields_for_command(args: list[str]) -> list[str]: + if args and args[0] == "search": + return SEARCH_JSON_FIELDS + if args and args[0] == "pr" and len(args) >= 2 and args[1] == "view": + return PR_VIEW_JSON_FIELDS + raise PrCheckerError(f"unsupported gh command for JSON parsing: {' '.join(args)}") + + +def _search_issues( + search_query: str, + *, + limit: int, +) -> list[dict[str, Any]]: + payload = _gh_api_json( + "search/issues?" + + urlencode( + { + "q": search_query, + "per_page": str(min(limit, 100)), + } + ) + ) + if not isinstance(payload, dict): + raise PrCheckerError(f"unexpected gh api search payload: {type(payload)!r}") + return _normalize_search_items(payload.get("items", [])) + + +def _search_backend_prs( + backend: str, + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, +) -> list[dict[str, Any]]: + search_terms = _backend_search_terms(backend, max_age_days=max_age_days) + per_term_results: list[list[dict[str, Any]]] = [] + for term in search_terms: + search_query = f"repo:{github_repo} is:pr is:open {term}" + log.info( + "Searching %s PRs on %s (query=%r, limit=%d, max_age_days=%d)", + backend, + github_repo, + search_query, + limit, + max_age_days, + ) + items = _search_issues(search_query, limit=limit) + log.info("Found %d open %s PR(s) for search term", len(items), backend) + per_term_results.append(items) + + items = _merge_search_items(per_term_results)[:limit] + log.info("Found %d unique open %s PR(s) after merging search terms", len(items), backend) + return items + + +def _view_pr(pr_number: int, github_repo: str) -> dict[str, Any]: + payload = _gh_json(["pr", "view", str(pr_number), "--repo", github_repo]) + if not isinstance(payload, dict): + raise PrCheckerError(f"unexpected gh pr view payload for #{pr_number}: {type(payload)!r}") + return payload + + +def _search_results_dataframe( + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, + backends: list[str] | None = None, +) -> pd.DataFrame: + """Run backend searches and return one row per (PR, backend) match.""" + frames: list[pd.DataFrame] = [] + for backend in _backends_to_search(backends): + items = _search_backend_prs( + backend, + github_repo=github_repo, + limit=limit, + max_age_days=max_age_days, + ) + if not items: + continue + frame = pd.DataFrame(items) + frame["backend"] = backend + frames.append(frame) + + if not frames: + return pd.DataFrame(columns=["number", "title", "updatedAt", "backend"]) + + return pd.concat(frames, ignore_index=True) + + +def _non_empty_first(values: pd.Series) -> str: + for value in values: + if pd.notna(value) and str(value): + return str(value) + return "" + + +def _aggregate_search_results(search_df: pd.DataFrame) -> pd.DataFrame: + """Collapse per-backend search hits to one row per PR.""" + if search_df.empty: + return pd.DataFrame(columns=["pr_number", "title", "updated_at", "backends"]) + + return ( + search_df.groupby("number", sort=True) + .agg( + title=("title", _non_empty_first), + updated_at=("updatedAt", "max"), + backends=("backend", lambda values: sorted(set(values))), + ) + .reset_index() + .rename(columns={"number": "pr_number"}) + ) + + +def discover_prs( + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, + backends: list[str] | None = None, +) -> list[DiscoveredPr]: + """Find open PRs with changed files under AMDGPU/SPIR-V target directories.""" + search_backends = _backends_to_search(backends) + log.info( + "Discovering open PRs on %s (limit=%d per backend, max_age_days=%d, backends=%s)", + github_repo, + limit, + max_age_days, + ", ".join(search_backends), + ) + search_df = _search_results_dataframe( + github_repo=github_repo, + limit=limit, + max_age_days=max_age_days, + backends=backends, + ) + grouped = _aggregate_search_results(search_df) + if grouped.empty: + log.info("No matching open PRs found") + return [] + + total = len(grouped) + log.info( + "Search returned %d unique PR(s); resolving PR metadata and filtering by target paths", + total, + ) + + discovered: list[DiscoveredPr] = [] + skipped_without_target_paths = 0 + for index, row in enumerate(grouped.itertuples(index=False), start=1): + pr_number = int(row.pr_number) + if index == 1 or index == total or index % 10 == 0: + log.info("Inspecting PR %d/%d: #%d", index, total, pr_number) + else: + log.debug("Inspecting PR %d/%d: #%d", index, total, pr_number) + + pr_view = _view_pr(pr_number, github_repo) + head_sha = pr_view.get("headRefOid") + if not head_sha: + raise PrCheckerError(f"could not resolve headRefOid for {github_repo}#{pr_number}") + + matched_backends = [ + backend + for backend in row.backends + if _pr_touches_backend_path(pr_view, backend) + ] + if not matched_backends: + skipped_without_target_paths += 1 + log.debug( + "Skipping #%d: no changed files under tracked target path(s) for %s", + pr_number, + ", ".join(row.backends), + ) + continue + + discovered.append( + DiscoveredPr( + pr_number=pr_number, + title=pr_view.get("title") or row.title or "", + head_sha=head_sha, + updated_at=pr_view.get("updatedAt") or row.updated_at or "", + backends=matched_backends, + ) + ) + + log.info( + "Discovery complete: %d PR(s) with target-path changes (%d skipped without target-path changes)", + len(discovered), + skipped_without_target_paths, + ) + return discovered + + +def load_state(path: Path) -> dict[str, Any]: + if not path.exists(): + log.info("State file not found, starting fresh: %s", path) + return {"version": STATE_VERSION, "entries": {}} + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"invalid state file {path}: {exc.msg}") from exc + + if not isinstance(payload, dict): + raise PrCheckerError(f"invalid state file {path}: expected object at top level") + if payload.get("version") != STATE_VERSION: + raise PrCheckerError( + f"unsupported state version in {path}: {payload.get('version')!r} (expected {STATE_VERSION})" + ) + if not isinstance(payload.get("entries"), dict): + raise PrCheckerError(f"invalid state file {path}: missing entries object") + + log.info("Loaded state from %s (%d entries)", path, len(payload["entries"])) + return payload + + +def save_state(path: Path, state: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + log.info("Saved state to %s (%d entries)", path, len(state.get("entries", {}))) + + +def _state_entry_from_dict(raw: dict[str, Any]) -> StateEntry: + return StateEntry( + pr_number=int(raw["pr_number"]), + backend=str(raw["backend"]), + title=str(raw.get("title", "")), + head_sha=str(raw["head_sha"]), + status=str(raw["status"]), + gap_count=int(raw.get("gap_count", 0)), + lit_failure_count=int(raw.get("lit_failure_count", 0)), + checked_at=str(raw.get("checked_at", "")), + output_dir=str(raw.get("output_dir", "")), + error=raw.get("error"), + ) + + +def plan_work(discovered: list[DiscoveredPr], state: dict[str, Any]) -> list[WorkItem]: + """Return PR/backend pairs that need a fresh coverage run.""" + entries: dict[str, Any] = state["entries"] + work: list[WorkItem] = [] + pair_count = 0 + skipped_up_to_date = 0 + new_count = 0 + head_changed_count = 0 + + log.info( + "Planning work from %d discovered PR(s) against %d state entries", + len(discovered), + len(entries), + ) + + for pr in discovered: + for backend in pr.backends: + pair_count += 1 + key = entry_key(pr.pr_number, backend) + existing = entries.get(key) + if existing is None: + new_count += 1 + work.append( + WorkItem( + pr_number=pr.pr_number, + backend=backend, + title=pr.title, + head_sha=pr.head_sha, + reason="new", + ) + ) + log.debug("Queued #%d (%s): new", pr.pr_number, backend) + continue + + previous = _state_entry_from_dict(existing) + if previous.head_sha != pr.head_sha: + head_changed_count += 1 + work.append( + WorkItem( + pr_number=pr.pr_number, + backend=backend, + title=pr.title, + head_sha=pr.head_sha, + reason="head_changed", + ) + ) + log.debug( + "Queued #%d (%s): head changed %s -> %s", + pr.pr_number, + backend, + previous.head_sha[:12], + pr.head_sha[:12], + ) + else: + skipped_up_to_date += 1 + log.debug( + "Skipping #%d (%s): already checked at head %s", + pr.pr_number, + backend, + pr.head_sha[:12], + ) + + work.sort(key=lambda item: (item.pr_number, item.backend)) + log.info( + "Plan summary: %d PR/backend pair(s) scanned, %d queued (%d new, %d head_changed), %d up-to-date", + pair_count, + len(work), + new_count, + head_changed_count, + skipped_up_to_date, + ) + for item in work: + log.info( + " -> #%d (%s) [%s] head=%s", + item.pr_number, + item.backend, + item.reason, + item.head_sha[:12], + ) + return work + + +def record_result( + state: dict[str, Any], + *, + pr_number: int, + backend: str, + title: str, + head_sha: str, + status: str, + gap_count: int, + lit_failure_count: int, + output_dir: str, + error: str | None = None, +) -> None: + key = entry_key(pr_number, backend) + log.info( + "Recording result for #%d (%s): status=%s gap_count=%d lit_failures=%d", + pr_number, + backend, + status, + gap_count, + lit_failure_count, + ) + state["entries"][key] = { + "pr_number": pr_number, + "backend": backend, + "title": title, + "head_sha": head_sha, + "status": status, + "gap_count": gap_count, + "lit_failure_count": lit_failure_count, + "checked_at": utc_now_iso(), + "output_dir": output_dir, + "error": error, + } + + +def _discovered_to_json(discovered: list[DiscoveredPr]) -> list[dict[str, Any]]: + return [ + { + "pr_number": pr.pr_number, + "title": pr.title, + "head_sha": pr.head_sha, + "updated_at": pr.updated_at, + "backends": pr.backends, + } + for pr in discovered + ] + + +def _work_to_json(work: list[WorkItem]) -> list[dict[str, Any]]: + return [asdict(item) for item in work] + + +def _read_gap_csv(gap_csv: Path, *, max_rows: int | None = None) -> pd.DataFrame: + """Load a target_lines_uncovered.csv file as a normalized DataFrame.""" + if not gap_csv.is_file(): + return pd.DataFrame(columns=["file", "line_no", "text"]) + + frame = pd.read_csv(gap_csv, nrows=max_rows) + if frame.empty: + return pd.DataFrame(columns=["file", "line_no", "text"]) + + for column in ("file", "line_no", "text"): + if column not in frame.columns: + frame[column] = "" + + normalized = frame[["file", "line_no", "text"]].fillna("").astype(str) + return normalized.apply(lambda series: series.str.strip()) + + +def count_csv_data_rows(path: Path) -> int: + """Return the number of data rows in a CSV file (excluding the header).""" + return len(_read_gap_csv(path)) + + +def load_gap_rows(gap_csv: Path, *, max_rows: int = 20) -> list[dict[str, str]]: + """Read uncovered line rows from a target_lines_uncovered.csv file.""" + if max_rows <= 0: + return [] + + frame = _read_gap_csv(gap_csv, max_rows=max_rows) + if frame.empty: + return [] + + return frame.to_dict(orient="records") + + +def count_lit_failures(path: Path) -> int: + """Count LIT tests that failed during baseline collection.""" + if not path.is_file(): + return 0 + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return 0 + + tests = payload.get("tests", []) + if not isinstance(tests, list): + return 0 + + return sum( + 1 + for test in tests + if isinstance(test, dict) and test.get("code") in LIT_FAILURE_CODES + ) + + +def evaluate_output_dir(output_dir: Path) -> dict[str, Any]: + """Summarize a completed pr-cov-gaps-detection output directory.""" + gap_csv = output_dir / "commit_lines_report" / "target_lines_uncovered.csv" + lit_failures_json = output_dir / "baseline" / "lit_failures.json" + gap_count = count_csv_data_rows(gap_csv) + lit_failure_count = count_lit_failures(lit_failures_json) + return { + "gap_count": gap_count, + "lit_failure_count": lit_failure_count, + "status": "gaps" if gap_count > 0 else "clean", + "gap_report": str(gap_csv), + "lit_failures_json": str(lit_failures_json), + } + + +_STATE_ENTRY_FIELDS = ( + "pr_number", + "backend", + "title", + "head_sha", + "status", + "gap_count", + "lit_failure_count", + "checked_at", + "output_dir", + "error", +) + + +def parse_run_dir_name(name: str) -> tuple[int, str]: + """Parse a run directory name ``-``.""" + if "-" not in name: + raise PrCheckerError(f"invalid run directory name (expected -): {name!r}") + backend = name.rsplit("-", 1)[1] + if backend not in BACKEND_SEARCH_QUERIES: + raise PrCheckerError( + f"invalid run directory name {name!r}: unknown backend {backend!r}" + ) + try: + pr_number = int(name[: -(len(backend) + 1)]) + except ValueError as exc: + raise PrCheckerError(f"invalid run directory name (bad PR number): {name!r}") from exc + if pr_number <= 0: + raise PrCheckerError(f"invalid run directory name (bad PR number): {name!r}") + return pr_number, backend + + +def is_evaluable_run(output_dir: Path) -> bool: + """Return True when *output_dir* has the artifacts produced by a completed check.""" + return (output_dir / "commit_lines_report" / "target_lines_uncovered.csv").is_file() + + +def output_dir_checked_at(output_dir: Path) -> str: + """Approximate check completion time from the newest artifact under *output_dir*.""" + newest_mtime = output_dir.stat().st_mtime + for path in output_dir.rglob("*"): + if path.is_file(): + newest_mtime = max(newest_mtime, path.stat().st_mtime) + return datetime.fromtimestamp(newest_mtime, tz=timezone.utc).replace(microsecond=0).isoformat() + + +def _report_entry_to_state_entry(entry: dict[str, Any]) -> dict[str, Any]: + return {field: entry.get(field) for field in _STATE_ENTRY_FIELDS} + + +def _checked_at_sort_key(checked_at: str) -> str: + return checked_at or "" + + +def load_report_entry_index(report_dir: Path | None) -> dict[str, dict[str, Any]]: + """Load PR/backend metadata from ``latest.json`` and ``runs/*.json`` report snapshots.""" + if report_dir is None: + return {} + + candidates: list[Path] = [] + latest_json = report_dir / "latest.json" + if latest_json.is_file(): + candidates.append(latest_json) + runs_dir = report_dir / "runs" + if runs_dir.is_dir(): + candidates.extend(sorted(runs_dir.glob("*.json"))) + + indexed: dict[str, dict[str, Any]] = {} + for path in candidates: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + log.warning("Skipping invalid report file %s: %s", path, exc.msg) + continue + if not isinstance(payload, dict): + log.warning("Skipping invalid report file %s: expected object at top level", path) + continue + + for raw in payload.get("all_entries", []): + if not isinstance(raw, dict): + continue + try: + key = _entry_key_from_record(raw) + state_entry = _report_entry_to_state_entry(raw) + except (KeyError, TypeError, ValueError): + log.warning("Skipping malformed report entry in %s", path) + continue + + existing = indexed.get(key) + if existing is None or _checked_at_sort_key(state_entry["checked_at"]) >= _checked_at_sort_key( + existing.get("checked_at", "") + ): + indexed[key] = state_entry + + log.info( + "Loaded metadata for %d PR/backend pair(s) from report files under %s", + len(indexed), + report_dir, + ) + return indexed + + +def fetch_pr_metadata(pr_number: int, *, github_repo: str) -> tuple[str, str]: + """Return ``(title, head_sha)`` for an open PR via ``gh pr view``.""" + payload = _view_pr(pr_number, github_repo) + head_sha = payload.get("headRefOid") + if not head_sha: + raise PrCheckerError(f"could not resolve headRefOid for {github_repo}#{pr_number}") + title = str(payload.get("title") or "") + return title, str(head_sha) + + +def rebuild_state_from_runs( + *, + output_root: Path, + github_repo: str = DEFAULT_GITHUB_REPO, + report_dir: Path | None = None, + existing_state: dict[str, Any] | None = None, + fetch_missing_pr_metadata: bool = True, +) -> tuple[dict[str, Any], dict[str, int]]: + """Rebuild ``state.json`` entries from on-disk run artifacts. + + Completed runs under *output_root* are re-evaluated with + :func:`evaluate_output_dir`. ``title``, ``head_sha``, and ``checked_at`` are + taken from prior report snapshots when available; remaining PRs can optionally + be resolved via ``gh pr view`` (current head SHA — see README caveat). + """ + if not output_root.is_dir(): + raise PrCheckerError(f"output root is not a directory: {output_root}") + + entries: dict[str, dict[str, Any]] = {} + if existing_state is not None: + for key, raw in existing_state.get("entries", {}).items(): + if isinstance(raw, dict): + entries[key] = dict(raw) + + report_index = load_report_entry_index(report_dir) + stats = { + "runs_seen": 0, + "runs_evaluated": 0, + "runs_skipped_incomplete": 0, + "metadata_from_reports": 0, + "metadata_from_existing_state": 0, + "metadata_from_github": 0, + "metadata_missing": 0, + } + + for run_dir in sorted(output_root.iterdir()): + if not run_dir.is_dir(): + continue + stats["runs_seen"] += 1 + if not is_evaluable_run(run_dir): + stats["runs_skipped_incomplete"] += 1 + log.info("Skipping incomplete run directory (no gap report): %s", run_dir.name) + continue + + pr_number, backend = parse_run_dir_name(run_dir.name) + key = entry_key(pr_number, backend) + evaluation = evaluate_output_dir(run_dir) + output_dir = str(run_dir.resolve()) + + entry: dict[str, Any] = { + "pr_number": pr_number, + "backend": backend, + "title": "", + "head_sha": "", + "status": evaluation["status"], + "gap_count": evaluation["gap_count"], + "lit_failure_count": evaluation["lit_failure_count"], + "checked_at": output_dir_checked_at(run_dir), + "output_dir": output_dir, + "error": None, + } + + metadata_source = report_index.get(key) + metadata_origin: str | None = None + if metadata_source is not None: + metadata_origin = "reports" + elif key in entries: + metadata_source = entries[key] + metadata_origin = "existing_state" + + if metadata_source is not None: + entry["title"] = str(metadata_source.get("title") or "") + entry["head_sha"] = str(metadata_source.get("head_sha") or "") + if metadata_source.get("checked_at"): + entry["checked_at"] = str(metadata_source["checked_at"]) + if metadata_source.get("status") == "failed": + entry["status"] = "failed" + entry["error"] = metadata_source.get("error") + if metadata_source.get("output_dir"): + entry["output_dir"] = str(metadata_source["output_dir"]) + if metadata_origin == "reports": + stats["metadata_from_reports"] += 1 + elif metadata_origin == "existing_state": + stats["metadata_from_existing_state"] += 1 + + if fetch_missing_pr_metadata and not entry["head_sha"]: + try: + title, head_sha = fetch_pr_metadata(pr_number, github_repo=github_repo) + except PrCheckerError as exc: + stats["metadata_missing"] += 1 + log.warning( + "Skipping %s: could not resolve PR metadata (%s)", + run_dir.name, + exc, + ) + continue + entry["title"] = title + entry["head_sha"] = head_sha + stats["metadata_from_github"] += 1 + log.info( + "Resolved PR metadata from GitHub for #%d (%s): head=%s", + pr_number, + backend, + head_sha[:12], + ) + elif not entry["head_sha"]: + stats["metadata_missing"] += 1 + log.warning( + "Skipping %s: missing head_sha (pass --fetch-pr-metadata or provide report snapshots)", + run_dir.name, + ) + continue + + entries[key] = entry + stats["runs_evaluated"] += 1 + + state = {"version": STATE_VERSION, "entries": entries} + log.info( + "Rebuild summary: %d run dir(s) seen, %d evaluated, %d incomplete skipped, " + "%d metadata from reports, %d from GitHub, %d missing", + stats["runs_seen"], + stats["runs_evaluated"], + stats["runs_skipped_incomplete"], + stats["metadata_from_reports"], + stats["metadata_from_github"], + stats["metadata_missing"], + ) + return state, stats + + +def pr_url(github_repo: str, pr_number: int) -> str: + return f"https://github.com/{github_repo}/pull/{pr_number}" + + +def _state_entries_dataframe(state: dict[str, Any]) -> pd.DataFrame: + """Convert state entries into a flat DataFrame for report aggregation.""" + rows: list[dict[str, Any]] = [] + for key, raw in state.get("entries", {}).items(): + if not isinstance(raw, dict): + continue + + entry = _state_entry_from_dict(raw) + rows.append( + { + "key": key, + "pr_number": entry.pr_number, + "backend": entry.backend, + "title": entry.title, + "head_sha": entry.head_sha, + "status": entry.status, + "gap_count": entry.gap_count, + "lit_failure_count": entry.lit_failure_count, + "checked_at": entry.checked_at, + "output_dir": entry.output_dir, + "error": entry.error, + } + ) + + columns = [ + "key", + "pr_number", + "backend", + "title", + "head_sha", + "status", + "gap_count", + "lit_failure_count", + "checked_at", + "output_dir", + "error", + ] + if not rows: + return pd.DataFrame(columns=columns) + + return pd.DataFrame(rows) + + +def _attach_sample_gaps(frame: pd.DataFrame, *, max_gap_lines: int) -> pd.DataFrame: + """Add a sample_gaps list column for entries that reported coverage gaps.""" + + def sample_gaps_for_row(row: pd.Series) -> list[dict[str, str]]: + if row["status"] != "gaps" or int(row["gap_count"]) <= 0: + return [] + gap_csv = Path(row["output_dir"]) / "commit_lines_report" / "target_lines_uncovered.csv" + return load_gap_rows(gap_csv, max_rows=max_gap_lines) + + enriched = frame.copy() + enriched["sample_gaps"] = enriched.apply(sample_gaps_for_row, axis=1) + return enriched + + +def _entry_key_from_record(entry: dict[str, Any]) -> str: + return entry_key(int(entry["pr_number"]), str(entry["backend"])) + + +def _index_report_entries(payload: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + """Map ``":"`` keys to entries from a report payload.""" + if payload is None: + return {} + indexed: dict[str, dict[str, Any]] = {} + for entry in payload.get("all_entries", []): + if isinstance(entry, dict): + indexed[_entry_key_from_record(entry)] = entry + return indexed + + +def diff_report_entries( + current: dict[str, Any], + previous: dict[str, Any] | None, +) -> set[str]: + """Return keys for entries that are new or updated since the previous report.""" + current_index = _index_report_entries(current) + if previous is None: + return set(current_index) + + previous_index = _index_report_entries(previous) + changed: set[str] = set() + for key, entry in current_index.items(): + prior = previous_index.get(key) + if prior is None: + changed.add(key) + continue + if entry.get("checked_at") != prior.get("checked_at"): + changed.add(key) + continue + if entry.get("head_sha") != prior.get("head_sha"): + changed.add(key) + return changed + + +def filter_report_payload(payload: dict[str, Any], keys: set[str]) -> dict[str, Any]: + """Return a copy of a report payload limited to the given entry keys.""" + if not keys: + return { + "generated_at": payload["generated_at"], + "github_repo": payload["github_repo"], + "summary": { + "total_entries": 0, + "with_gaps": 0, + "clean": 0, + "failed": 0, + }, + "entries_with_gaps": [], + "failed_entries": [], + "all_entries": [], + } + + def keep(entry: dict[str, Any]) -> bool: + return _entry_key_from_record(entry) in keys + + entries_with_gaps = [ + entry for entry in payload.get("entries_with_gaps", []) if keep(entry) + ] + failed_entries = [entry for entry in payload.get("failed_entries", []) if keep(entry)] + all_entries = [entry for entry in payload.get("all_entries", []) if keep(entry)] + + status_counts = {"gaps": 0, "clean": 0, "failed": 0} + for entry in all_entries: + status = entry.get("status") + if status in status_counts: + status_counts[status] += 1 + + return { + "generated_at": payload["generated_at"], + "github_repo": payload["github_repo"], + "summary": { + "total_entries": len(all_entries), + "with_gaps": status_counts["gaps"], + "clean": status_counts["clean"], + "failed": status_counts["failed"], + }, + "entries_with_gaps": entries_with_gaps, + "failed_entries": failed_entries, + "all_entries": all_entries, + } + + +def load_previous_report(latest_json: Path) -> dict[str, Any] | None: + """Load the previous latest.json report snapshot, if it exists.""" + if not latest_json.is_file(): + return None + try: + payload = json.loads(latest_json.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"invalid report file {latest_json}: {exc.msg}") from exc + if not isinstance(payload, dict): + raise PrCheckerError(f"invalid report file {latest_json}: expected object at top level") + return payload + + +def build_report_payload( + state: dict[str, Any], + *, + github_repo: str = DEFAULT_GITHUB_REPO, + max_gap_lines: int = 20, +) -> dict[str, Any]: + """Aggregate state entries into a report payload.""" + frame = _state_entries_dataframe(state) + if frame.empty: + return { + "generated_at": utc_now_iso(), + "github_repo": github_repo, + "summary": { + "total_entries": 0, + "with_gaps": 0, + "clean": 0, + "failed": 0, + }, + "entries_with_gaps": [], + "failed_entries": [], + "all_entries": [], + } + + frame = _attach_sample_gaps(frame, max_gap_lines=max_gap_lines) + frame["pr_url"] = frame["pr_number"].map(lambda number: pr_url(github_repo, int(number))) + + status_counts = frame["status"].value_counts() + with_gaps_frame = frame[frame["status"] == "gaps"].sort_values( + ["gap_count", "pr_number", "backend"], + ascending=[False, True, True], + ) + failed_frame = frame[frame["status"] == "failed"].sort_values(["pr_number", "backend"]) + all_entries_frame = frame.sort_values(["pr_number", "backend"]) + + return { + "generated_at": utc_now_iso(), + "github_repo": github_repo, + "summary": { + "total_entries": int(len(frame)), + "with_gaps": int(status_counts.get("gaps", 0)), + "clean": int(status_counts.get("clean", 0)), + "failed": int(status_counts.get("failed", 0)), + }, + "entries_with_gaps": with_gaps_frame.to_dict(orient="records"), + "failed_entries": failed_frame.to_dict(orient="records"), + "all_entries": all_entries_frame.to_dict(orient="records"), + } + + +def _markdown_escape_cell(value: str) -> str: + return value.replace("|", "\\|").replace("\n", " ") + + +def render_report_markdown(report: dict[str, Any], *, delta: bool = False) -> str: + """Render a human-readable Markdown summary.""" + title = ( + "# LLVM PR coverage gap report — new checks" + if delta + else "# LLVM PR coverage gap report" + ) + lines = [ + title, + "", + f"Generated: {report['generated_at']}", + f"Repository: {report['github_repo']}", + ] + summary = report["summary"] + if delta: + summary_line = ( + "New or updated since last report: " + f"{summary['total_entries']} PR/backend pair(s) " + f"({summary['with_gaps']} with gaps, " + f"{summary['clean']} clean, " + f"{summary['failed']} failed)" + ) + else: + summary_line = ( + "Entries: " + f"{summary['total_entries']} total, " + f"{summary['with_gaps']} with gaps, " + f"{summary['clean']} clean, " + f"{summary['failed']} failed" + ) + lines.extend([summary_line, ""]) + + entries_with_gaps = report.get("entries_with_gaps", []) + if entries_with_gaps: + lines.append("## PRs with coverage gaps") + lines.append("") + for entry in entries_with_gaps: + lines.append( + f"### #{entry['pr_number']} ({entry['backend']}) — " + f"{entry['gap_count']} uncovered line(s)" + ) + lines.append("") + lines.append(f"- **Title:** {entry['title']}") + lines.append(f"- **PR:** {entry['pr_url']}") + lines.append(f"- **Head:** `{entry['head_sha'][:12]}`") + lines.append(f"- **Checked:** {entry['checked_at']}") + if entry["lit_failure_count"] > 0: + lines.append( + f"- **Warning:** {entry['lit_failure_count']} LIT failure(s) during baseline" + ) + lines.append("") + sample_gaps = entry.get("sample_gaps", []) + if sample_gaps: + lines.extend(["| File | Line | Text |", "|------|------|------|"]) + for row in sample_gaps: + text = _markdown_escape_cell(row.get("text", "")) + file_path = _markdown_escape_cell(row.get("file", "")) + lines.append(f"| `{file_path}` | {row.get('line_no', '')} | `{text}` |") + remaining = entry["gap_count"] - len(sample_gaps) + if remaining > 0: + lines.extend( + [ + "", + ( + f"*…and {remaining} more line(s). See " + f"`{entry['output_dir']}/commit_lines_report/target_lines_uncovered.csv`*" + ), + ] + ) + lines.append("") + + failed_entries = report.get("failed_entries", []) + if failed_entries: + lines.extend(["## Failed checks", ""]) + for entry in failed_entries: + lines.append( + f"- #{entry['pr_number']} ({entry['backend']}): {entry.get('error') or 'unknown error'}" + ) + lines.append("") + + if not entries_with_gaps and not failed_entries: + if delta: + lines.append( + "No PRs with coverage gaps or failed checks since the last report." + ) + else: + lines.append("No PRs with coverage gaps or failed checks in the current state.") + lines.append("") + + return "\n".join(lines) + + +def write_reports( + report_dir: Path, + payload: dict[str, Any], + *, + write_run_snapshot: bool = True, +) -> dict[str, Path]: + """Write latest.json, latest.md, new-prs.md, and an optional timestamped snapshot.""" + report_dir.mkdir(parents=True, exist_ok=True) + + latest_json = report_dir / "latest.json" + latest_md = report_dir / "latest.md" + new_prs_md = report_dir / "new-prs.md" + + previous_payload = load_previous_report(latest_json) + changed_keys = diff_report_entries(payload, previous_payload) + delta_payload = filter_report_payload(payload, changed_keys) + log.info( + "Report delta: %d new or updated PR/backend pair(s) since last report", + len(changed_keys), + ) + + latest_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + latest_md.write_text(render_report_markdown(payload) + "\n", encoding="utf-8") + new_prs_md.write_text(render_report_markdown(delta_payload, delta=True) + "\n", encoding="utf-8") + + written = { + "latest_json": latest_json, + "latest_md": latest_md, + "new_prs_md": new_prs_md, + } + if write_run_snapshot: + runs_dir = report_dir / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + snapshot_name = payload["generated_at"].replace(":", "").replace("+00:00", "Z") + snapshot_path = runs_dir / f"{snapshot_name}.json" + snapshot_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + written["run_snapshot"] = snapshot_path + + log.info("Wrote report files under %s", report_dir) + for name, path in written.items(): + log.info(" %s: %s", name, path) + + return written + + +def cmd_discover(args: argparse.Namespace) -> int: + discovered = discover_prs( + github_repo=args.github_repo, + limit=args.limit, + max_age_days=args.max_age_days, + backends=_parse_backends_arg(args.backends), + ) + payload = _discovered_to_json(discovered) + log.info("Writing %d discovered PR(s) to stdout as JSON", len(payload)) + print(json.dumps(payload, indent=2)) + return 0 + + +def cmd_plan(args: argparse.Namespace) -> int: + discovered = discover_prs( + github_repo=args.github_repo, + limit=args.limit, + max_age_days=args.max_age_days, + backends=_parse_backends_arg(args.backends), + ) + state = load_state(args.state_file) + work = plan_work(discovered, state) + if args.max_items is not None: + if len(work) > args.max_items: + log.info( + "Capping planned work from %d item(s) to %d (--max-items)", + len(work), + args.max_items, + ) + work = work[: args.max_items] + log.info("Writing %d planned work item(s) to stdout as JSON", len(work)) + print(json.dumps(_work_to_json(work), indent=2)) + return 0 + + +def cmd_record(args: argparse.Namespace) -> int: + state = load_state(args.state_file) + record_result( + state, + pr_number=args.pr_number, + backend=args.backend, + title=args.title, + head_sha=args.head_sha, + status=args.status, + gap_count=args.gap_count, + lit_failure_count=args.lit_failure_count, + output_dir=args.output_dir, + error=args.error, + ) + save_state(args.state_file, state) + return 0 + + +def cmd_rebuild_state(args: argparse.Namespace) -> int: + existing_state: dict[str, Any] | None = None + if args.merge_existing_state and args.state_file.is_file(): + try: + existing_state = load_state(args.state_file) + except PrCheckerError as exc: + log.warning("Ignoring invalid existing state file %s: %s", args.state_file, exc) + + report_dir = args.report_dir + if report_dir is None: + report_dir = args.state_file.parent / "reports" + + state, stats = rebuild_state_from_runs( + output_root=args.output_root, + github_repo=args.github_repo, + report_dir=report_dir, + existing_state=existing_state, + fetch_missing_pr_metadata=args.fetch_pr_metadata, + ) + + if args.dry_run: + print( + json.dumps( + { + "dry_run": True, + "entry_count": len(state["entries"]), + "stats": stats, + }, + indent=2, + ) + ) + return 0 + + if args.state_file.is_file() and args.backup: + backup_path = args.state_file.with_suffix(args.state_file.suffix + ".bak") + backup_path.write_text(args.state_file.read_text(encoding="utf-8"), encoding="utf-8") + log.info("Backed up existing state to %s", backup_path) + + save_state(args.state_file, state) + print( + json.dumps( + { + "state_file": str(args.state_file), + "entry_count": len(state["entries"]), + "stats": stats, + }, + indent=2, + ) + ) + return 0 + + +def cmd_evaluate_output(args: argparse.Namespace) -> int: + log.info("Evaluating output directory: %s", args.output_dir) + payload = evaluate_output_dir(args.output_dir) + log.info( + "Evaluation: status=%s gap_count=%d lit_failures=%d", + payload["status"], + payload["gap_count"], + payload["lit_failure_count"], + ) + print(json.dumps(payload, indent=2)) + return 0 + + +def cmd_report(args: argparse.Namespace) -> int: + log.info("Generating coverage gap report") + state = load_state(args.state_file) + payload = build_report_payload( + state, + github_repo=args.github_repo, + max_gap_lines=args.max_gap_lines, + ) + summary = payload["summary"] + log.info( + "Report summary: %d total, %d with gaps, %d clean, %d failed", + summary["total_entries"], + summary["with_gaps"], + summary["clean"], + summary["failed"], + ) + written = write_reports( + args.report_dir, + payload, + write_run_snapshot=not args.no_run_snapshot, + ) + print( + json.dumps( + {key: str(path) for key, path in written.items()}, + indent=2, + ) + ) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + logging_parent = argparse.ArgumentParser(add_help=False) + logging_parent.add_argument( + "--log-level", + default="info", + choices=["debug", "info", "warning", "error"], + help="Log level for progress messages on stderr (default: info)", + ) + logging_parent.add_argument( + "-v", + "--verbose", + action="store_const", + const="debug", + dest="log_level", + help="Shorthand for --log-level debug", + ) + + github_repo_parent = argparse.ArgumentParser(add_help=False) + github_repo_parent.add_argument( + "--github-repo", + default=DEFAULT_GITHUB_REPO, + help=f"GitHub repo hosting PRs (default: {DEFAULT_GITHUB_REPO})", + ) + + search_parent = argparse.ArgumentParser(add_help=False) + search_parent.add_argument( + "--limit", + type=int, + default=DEFAULT_SEARCH_LIMIT, + help=f"Max PRs per backend search (default: {DEFAULT_SEARCH_LIMIT})", + ) + search_parent.add_argument( + "--max-age-days", + type=int, + default=DEFAULT_MAX_PR_AGE_DAYS, + help=( + "Only include PRs opened within this many days " + f"(default: {DEFAULT_MAX_PR_AGE_DAYS})" + ), + ) + search_parent.add_argument( + "--backends", + default=None, + help=( + "Comma-separated backend names to search " + f"(default: {', '.join(sorted(BACKEND_SEARCH_QUERIES))})" + ), + ) + + discover_parser = subparsers.add_parser( + "discover", + help="List open target PRs", + parents=[logging_parent, github_repo_parent, search_parent], + ) + discover_parser.set_defaults(func=cmd_discover) + + plan_parser = subparsers.add_parser( + "plan", + help="List PR/backend pairs needing a run", + parents=[logging_parent, github_repo_parent, search_parent], + ) + plan_parser.add_argument( + "--state-file", + type=Path, + required=True, + help="Path to state.json", + ) + plan_parser.add_argument( + "--max-items", + type=int, + default=None, + help="Cap the number of planned work items", + ) + plan_parser.set_defaults(func=cmd_plan) + + record_parser = subparsers.add_parser( + "record", + help="Persist one check result", + parents=[logging_parent], + ) + record_parser.add_argument("--state-file", type=Path, required=True) + record_parser.add_argument("--pr-number", type=int, required=True) + record_parser.add_argument("--backend", choices=sorted(BACKEND_SEARCH_QUERIES), required=True) + record_parser.add_argument("--title", default="") + record_parser.add_argument("--head-sha", required=True) + record_parser.add_argument( + "--status", + choices=["gaps", "clean", "failed"], + required=True, + ) + record_parser.add_argument("--gap-count", type=int, default=0) + record_parser.add_argument("--lit-failure-count", type=int, default=0) + record_parser.add_argument("--output-dir", required=True) + record_parser.add_argument("--error", default=None) + record_parser.set_defaults(func=cmd_record) + + evaluate_parser = subparsers.add_parser( + "evaluate-output", + help="Summarize gap and LIT failure counts from a run output directory", + parents=[logging_parent], + ) + evaluate_parser.add_argument("--output-dir", type=Path, required=True) + evaluate_parser.set_defaults(func=cmd_evaluate_output) + + rebuild_parser = subparsers.add_parser( + "rebuild-state", + help="Rebuild state.json from run artifacts and saved reports", + parents=[logging_parent, github_repo_parent], + ) + rebuild_parser.add_argument( + "--state-file", + type=Path, + required=True, + help="Path to write rebuilt state.json", + ) + rebuild_parser.add_argument( + "--output-root", + type=Path, + required=True, + help="Directory containing per-run output folders (-)", + ) + rebuild_parser.add_argument( + "--report-dir", + type=Path, + default=None, + help=( + "Directory with latest.json and runs/*.json snapshots " + "(default: /reports)" + ), + ) + rebuild_parser.add_argument( + "--fetch-pr-metadata", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Resolve title/head_sha via gh for runs missing report metadata " + "(default: enabled)" + ), + ) + rebuild_parser.add_argument( + "--merge-existing-state", + action=argparse.BooleanOptionalAction, + default=True, + help="Reuse metadata from an existing state file when valid (default: enabled)", + ) + rebuild_parser.add_argument( + "--backup", + action=argparse.BooleanOptionalAction, + default=True, + help="Back up the existing state file to state.json.bak (default: enabled)", + ) + rebuild_parser.add_argument( + "--dry-run", + action="store_true", + help="Print rebuild stats without writing state.json", + ) + rebuild_parser.set_defaults(func=cmd_rebuild_state) + + report_parser = subparsers.add_parser( + "report", + help="Write latest.json and latest.md from state", + parents=[logging_parent, github_repo_parent], + ) + report_parser.add_argument("--state-file", type=Path, required=True) + report_parser.add_argument( + "--report-dir", + type=Path, + required=True, + help="Directory for latest.json, latest.md, and runs/ snapshots", + ) + report_parser.add_argument( + "--max-gap-lines", + type=int, + default=20, + help="Max uncovered lines to include per PR in the report (default: 20)", + ) + report_parser.add_argument( + "--no-run-snapshot", + action="store_true", + help="Skip writing report_dir/runs/.json", + ) + report_parser.set_defaults(func=cmd_report) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + configure_logging(args.log_level) + except PrCheckerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + try: + return args.func(args) + except PrCheckerError as exc: + log.error("%s", exc) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py new file mode 100644 index 00000000..5cd71f83 --- /dev/null +++ b/tests/test_pr_checker.py @@ -0,0 +1,595 @@ +"""Unit tests for periodic LLVM PR coverage checking helpers.""" + +from __future__ import annotations + +import csv +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import pandas as pd + +from pr_check.checker import ( + STATE_VERSION, + DiscoveredPr, + PrCheckerError, + _aggregate_search_results, + build_report_payload, + count_csv_data_rows, + count_lit_failures, + diff_report_entries, + entry_key, + evaluate_output_dir, + filter_report_payload, + load_gap_rows, + load_state, + plan_work, + record_result, + render_report_markdown, + save_state, + write_reports, +) + + +def _write_gap_csv(path: Path, rows: list[list[object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + csv.writer(handle).writerows(rows) + + +class EntryKeyTest(unittest.TestCase): + def test_formats_pr_and_backend(self) -> None: + self.assertEqual(entry_key(203468, "amdgpu"), "203468:amdgpu") + + +class StateFileTest(unittest.TestCase): + def test_load_missing_state_returns_empty_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state = load_state(Path(tmp) / "state.json") + self.assertEqual(state["version"], STATE_VERSION) + self.assertEqual(state["entries"], {}) + + def test_save_and_load_roundtrip(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "state.json" + state = {"version": STATE_VERSION, "entries": {}} + record_result( + state, + pr_number=42, + backend="spirv", + title="SPIR-V fix", + head_sha="abc123", + status="clean", + gap_count=0, + lit_failure_count=0, + output_dir="/tmp/out", + ) + save_state(path, state) + loaded = load_state(path) + + self.assertIn("42:spirv", loaded["entries"]) + self.assertEqual(loaded["entries"]["42:spirv"]["status"], "clean") + + def test_unsupported_version_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "state.json" + path.write_text(json.dumps({"version": 99, "entries": {}}), encoding="utf-8") + with self.assertRaises(PrCheckerError): + load_state(path) + + +class PlanWorkTest(unittest.TestCase): + def test_queues_new_and_head_changed_items_only(self) -> None: + discovered = [ + DiscoveredPr( + pr_number=1, + title="A", + head_sha="sha-new", + updated_at="2026-01-01", + backends=["amdgpu", "spirv"], + ), + DiscoveredPr( + pr_number=2, + title="B", + head_sha="same-sha", + updated_at="2026-01-01", + backends=["amdgpu"], + ), + ] + state = { + "version": STATE_VERSION, + "entries": { + "1:amdgpu": { + "pr_number": 1, + "backend": "amdgpu", + "title": "A", + "head_sha": "sha-old", + "status": "gaps", + "gap_count": 3, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": "/tmp/1-amdgpu", + "error": None, + }, + "2:amdgpu": { + "pr_number": 2, + "backend": "amdgpu", + "title": "B", + "head_sha": "same-sha", + "status": "clean", + "gap_count": 0, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": "/tmp/2-amdgpu", + "error": None, + }, + }, + } + + work = plan_work(discovered, state) + reasons = {(item.pr_number, item.backend): item.reason for item in work} + + self.assertEqual(reasons[(1, "amdgpu")], "head_changed") + self.assertEqual(reasons[(1, "spirv")], "new") + self.assertNotIn((2, "amdgpu"), reasons) + + +class AggregateSearchResultsTest(unittest.TestCase): + def test_merges_backends_for_same_pr(self) -> None: + frame = pd.DataFrame( + [ + {"number": 10, "title": "AMDGPU pass", "updatedAt": "2026-01-01", "backend": "amdgpu"}, + {"number": 10, "title": "", "updatedAt": "2026-01-03", "backend": "spirv"}, + {"number": 5, "title": "SPIR-V only", "updatedAt": "2026-01-02", "backend": "spirv"}, + ] + ) + aggregated = _aggregate_search_results(frame) + + self.assertEqual(aggregated["pr_number"].tolist(), [5, 10]) + row_10 = aggregated.loc[aggregated["pr_number"] == 10].iloc[0] + self.assertEqual(row_10["backends"], ["amdgpu", "spirv"]) + self.assertEqual(row_10["title"], "AMDGPU pass") + self.assertEqual(row_10["updated_at"], "2026-01-03") + + +class GapCsvHelpersTest(unittest.TestCase): + def test_count_and_load_gap_rows(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + gap_csv = Path(tmp) / "target_lines_uncovered.csv" + _write_gap_csv( + gap_csv, + [ + ["file", "line_no", "text"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "return x;"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "11", "return y;"], + ], + ) + + self.assertEqual(count_csv_data_rows(gap_csv), 2) + rows = load_gap_rows(gap_csv, max_rows=1) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["line_no"], "10") + + def test_count_lit_failures(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + lit_path = Path(tmp) / "lit_failures.json" + lit_path.write_text( + json.dumps({"tests": [{"code": "FAIL"}, {"code": "PASS"}]}), + encoding="utf-8", + ) + self.assertEqual(count_lit_failures(lit_path), 1) + + +class EvaluateOutputDirTest(unittest.TestCase): + def test_reports_gaps_status(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_gap_csv( + root / "commit_lines_report" / "target_lines_uncovered.csv", + [["file", "line_no", "text"], ["llvm/Foo.cpp", "1", "x = 1;"]], + ) + result = evaluate_output_dir(root) + + self.assertEqual(result["gap_count"], 1) + self.assertEqual(result["status"], "gaps") + + +class ReportGenerationTest(unittest.TestCase): + def test_build_render_and_write_reports(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + run_dir = root / "run-123-amdgpu" + _write_gap_csv( + run_dir / "commit_lines_report" / "target_lines_uncovered.csv", + [ + ["file", "line_no", "text"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "return x;"], + ], + ) + state = { + "version": STATE_VERSION, + "entries": { + "123:amdgpu": { + "pr_number": 123, + "backend": "amdgpu", + "title": "Fix foo", + "head_sha": "abc123def456", + "status": "gaps", + "gap_count": 1, + "lit_failure_count": 2, + "checked_at": "2026-07-29T18:00:00+00:00", + "output_dir": str(run_dir), + "error": None, + } + }, + } + + payload = build_report_payload(state) + self.assertEqual(payload["summary"]["with_gaps"], 1) + self.assertEqual(payload["entries_with_gaps"][0]["sample_gaps"][0]["line_no"], "10") + + markdown = render_report_markdown(payload) + self.assertIn("#123", markdown) + self.assertIn("LIT failure", markdown) + + report_dir = root / "reports" + written = write_reports(report_dir, payload, write_run_snapshot=True) + self.assertTrue(written["latest_json"].is_file()) + self.assertTrue(written["latest_md"].is_file()) + self.assertTrue(written["new_prs_md"].is_file()) + self.assertTrue(written["run_snapshot"].is_file()) + + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn("new checks", new_prs_text) + self.assertIn("#123", new_prs_text) + + +class ReportDeltaTest(unittest.TestCase): + def _sample_entry( + self, + *, + pr_number: int = 123, + backend: str = "amdgpu", + checked_at: str = "2026-07-29T18:00:00+00:00", + head_sha: str = "abc123def456", + status: str = "gaps", + gap_count: int = 1, + ) -> dict[str, object]: + return { + "key": entry_key(pr_number, backend), + "pr_number": pr_number, + "backend": backend, + "title": "Fix foo", + "head_sha": head_sha, + "status": status, + "gap_count": gap_count, + "lit_failure_count": 0, + "checked_at": checked_at, + "output_dir": "/tmp/out", + "error": None, + "pr_url": f"https://github.com/llvm/llvm-project/pull/{pr_number}", + "sample_gaps": [], + } + + def _sample_payload(self, entries: list[dict[str, object]]) -> dict[str, object]: + with_gaps = [entry for entry in entries if entry["status"] == "gaps"] + failed = [entry for entry in entries if entry["status"] == "failed"] + status_counts = {"gaps": 0, "clean": 0, "failed": 0} + for entry in entries: + status_counts[str(entry["status"])] += 1 + return { + "generated_at": "2026-08-05T08:00:00+00:00", + "github_repo": "llvm/llvm-project", + "summary": { + "total_entries": len(entries), + "with_gaps": status_counts["gaps"], + "clean": status_counts["clean"], + "failed": status_counts["failed"], + }, + "entries_with_gaps": with_gaps, + "failed_entries": failed, + "all_entries": entries, + } + + def test_diff_detects_new_and_changed_entries(self) -> None: + previous = self._sample_payload( + [ + self._sample_entry(pr_number=1, checked_at="2026-01-01T00:00:00+00:00"), + self._sample_entry( + pr_number=2, + backend="spirv", + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + ] + ) + current = self._sample_payload( + [ + self._sample_entry( + pr_number=1, + checked_at="2026-01-02T00:00:00+00:00", + head_sha="updated-sha", + ), + self._sample_entry( + pr_number=2, + backend="spirv", + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + self._sample_entry(pr_number=3, checked_at="2026-01-02T00:00:00+00:00"), + ] + ) + + changed = diff_report_entries(current, previous) + self.assertEqual(changed, {"1:amdgpu", "3:amdgpu"}) + + def test_diff_without_previous_treats_all_entries_as_new(self) -> None: + current = self._sample_payload([self._sample_entry()]) + self.assertEqual(diff_report_entries(current, None), {"123:amdgpu"}) + + def test_filter_report_payload_recomputes_summary(self) -> None: + payload = self._sample_payload( + [ + self._sample_entry(pr_number=1), + self._sample_entry( + pr_number=2, + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + ] + ) + filtered = filter_report_payload(payload, {"1:amdgpu"}) + self.assertEqual(filtered["summary"]["total_entries"], 1) + self.assertEqual(filtered["summary"]["with_gaps"], 1) + self.assertEqual(len(filtered["all_entries"]), 1) + + def test_write_reports_emits_delta_against_previous_latest_json(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report_dir = Path(tmp) / "reports" + previous = self._sample_payload([self._sample_entry(pr_number=1)]) + current = self._sample_payload( + [ + self._sample_entry(pr_number=1), + self._sample_entry( + pr_number=2, + status="clean", + gap_count=0, + checked_at="2026-08-05T09:00:00+00:00", + ), + ] + ) + + write_reports(report_dir, previous, write_run_snapshot=False) + written = write_reports(report_dir, current, write_run_snapshot=False) + + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn("New or updated since last report: 1 PR/backend pair(s)", new_prs_text) + self.assertIn("1 clean", new_prs_text) + self.assertNotIn("#1", new_prs_text) + + def test_write_reports_with_no_changes_writes_empty_delta(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report_dir = Path(tmp) / "reports" + payload = self._sample_payload([self._sample_entry()]) + + write_reports(report_dir, payload, write_run_snapshot=False) + written = write_reports(report_dir, payload, write_run_snapshot=False) + + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn( + "No PRs with coverage gaps or failed checks since the last report.", + new_prs_text, + ) + self.assertIn("0 PR/backend pair(s)", new_prs_text) + + def test_render_report_markdown_delta_mode(self) -> None: + payload = self._sample_payload([self._sample_entry()]) + markdown = render_report_markdown(payload, delta=True) + self.assertIn("# LLVM PR coverage gap report — new checks", markdown) + self.assertIn("New or updated since last report:", markdown) + + +class DiscoverPrsTest(unittest.TestCase): + def test_backend_search_terms_include_label_and_age_filter_for_amdgpu(self) -> None: + from pr_check.checker import _backend_search_terms + + terms = _backend_search_terms("amdgpu", max_age_days=30) + self.assertEqual(len(terms), 1) + self.assertIn('label:"backend:AMDGPU"', terms[0]) + self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') + + def test_backend_search_terms_include_label_and_age_filter_for_spirv(self) -> None: + from pr_check.checker import _backend_search_terms + + terms = _backend_search_terms("spirv", max_age_days=30) + self.assertEqual(len(terms), 1) + self.assertIn('label:"backend:SPIR-V"', terms[0]) + self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') + + def test_pr_touches_backend_path_matches_target_prefix(self) -> None: + from pr_check.checker import _pr_touches_backend_path + + pr_view = { + "files": [ + {"path": "clang/test/CodeGenHIP/foo.hip"}, + {"path": "llvm/lib/Target/AMDGPU/Foo.cpp"}, + ] + } + self.assertTrue(_pr_touches_backend_path(pr_view, "amdgpu")) + self.assertFalse(_pr_touches_backend_path(pr_view, "spirv")) + + def test_merge_search_items_deduplicates_by_pr_number(self) -> None: + from pr_check.checker import _merge_search_items + + merged = _merge_search_items( + [ + [{"number": 10, "title": "From path", "updatedAt": "2026-01-01"}], + [{"number": 10, "title": "", "updatedAt": "2026-01-03"}, {"number": 5, "title": "Label only", "updatedAt": "2026-01-02"}], + ] + ) + + self.assertEqual([item["number"] for item in merged], [5, 10]) + self.assertEqual(merged[1]["updatedAt"], "2026-01-03") + self.assertEqual(merged[1]["title"], "From path") + + def _pr_view_with_target_files(self) -> dict[str, object]: + return { + "headRefOid": "deadbeef", + "title": "Test PR", + "updatedAt": "2026-01-02", + "files": [ + {"path": "llvm/lib/Target/AMDGPU/Foo.cpp"}, + {"path": "llvm/lib/Target/SPIRV/Bar.cpp"}, + ], + } + + def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: + from pr_check.checker import discover_prs + + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): + with mock.patch( + "pr_check.checker._view_pr", + return_value=self._pr_view_with_target_files(), + ): + discovered = discover_prs(limit=1) + + self.assertEqual(len(discovered), 1) + self.assertEqual(discovered[0].pr_number, 99) + self.assertEqual(discovered[0].head_sha, "deadbeef") + self.assertEqual(discovered[0].backends, ["amdgpu", "spirv"]) + + def test_discover_prs_honors_backends_filter(self) -> None: + from pr_check.checker import discover_prs + + searched: list[str] = [] + + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + searched.append(backend) + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): + with mock.patch( + "pr_check.checker._view_pr", + return_value=self._pr_view_with_target_files(), + ): + discovered = discover_prs(limit=1, backends=["amdgpu"]) + + self.assertEqual(searched, ["amdgpu"]) + self.assertEqual(discovered[0].backends, ["amdgpu"]) + + def test_discover_prs_skips_prs_without_target_path_changes(self) -> None: + from pr_check.checker import discover_prs + + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): + with mock.patch( + "pr_check.checker._view_pr", + return_value={ + "headRefOid": "deadbeef", + "title": "Test PR", + "updatedAt": "2026-01-02", + "files": [{"path": "clang/test/CodeGenHIP/foo.hip"}], + }, + ): + discovered = discover_prs(limit=1, backends=["amdgpu"]) + + self.assertEqual(discovered, []) + + +class RebuildStateTest(unittest.TestCase): + def _write_completed_run( + self, + root: Path, + *, + pr_number: int, + backend: str, + gap_rows: list[list[object]] | None = None, + ) -> Path: + run_dir = root / f"{pr_number}-{backend}" + gap_csv = run_dir / "commit_lines_report" / "target_lines_uncovered.csv" + rows: list[list[object]] = [["file", "line_no", "text"]] + if gap_rows: + rows.extend(gap_rows) + _write_gap_csv(gap_csv, rows) + lit_json = run_dir / "baseline" / "lit_failures.json" + lit_json.parent.mkdir(parents=True, exist_ok=True) + lit_json.write_text(json.dumps({"tests": []}), encoding="utf-8") + return run_dir + + def test_rebuild_state_from_completed_run(self) -> None: + from pr_check.checker import rebuild_state_from_runs + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runs = root / "runs" + reports = root / "reports" + self._write_completed_run( + runs, + pr_number=42, + backend="amdgpu", + gap_rows=[["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "assert(x);"]], + ) + report_payload = { + "all_entries": [ + { + "key": "42:amdgpu", + "pr_number": 42, + "backend": "amdgpu", + "title": "AMDGPU fix", + "head_sha": "abc123def456", + "status": "gaps", + "gap_count": 1, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": str(runs / "42-amdgpu"), + "error": None, + } + ] + } + reports.mkdir() + (reports / "latest.json").write_text(json.dumps(report_payload), encoding="utf-8") + + state, stats = rebuild_state_from_runs( + output_root=runs, + report_dir=reports, + fetch_missing_pr_metadata=False, + ) + + self.assertEqual(stats["runs_evaluated"], 1) + self.assertEqual(stats["metadata_from_reports"], 1) + entry = state["entries"]["42:amdgpu"] + self.assertEqual(entry["title"], "AMDGPU fix") + self.assertEqual(entry["head_sha"], "abc123def456") + self.assertEqual(entry["status"], "gaps") + self.assertEqual(entry["gap_count"], 1) + + def test_rebuild_state_skips_incomplete_run(self) -> None: + from pr_check.checker import rebuild_state_from_runs + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runs = root / "runs" + incomplete = runs / "99-spirv" + incomplete.mkdir(parents=True) + + state, stats = rebuild_state_from_runs( + output_root=runs, + fetch_missing_pr_metadata=False, + ) + + self.assertEqual(state["entries"], {}) + self.assertEqual(stats["runs_skipped_incomplete"], 1) + + +if __name__ == "__main__": + unittest.main()