From 897effe670f5ec0cc8d1f9bc4da28036af13822c Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 12:55:58 -0500 Subject: [PATCH 01/26] Add pr_check module for periodic LLVM PR discovery and state tracking. Introduces discover/plan/record subcommands with a versioned state.json schema to track which AMDGPU and SPIR-V PRs need coverage-gap reruns. --- scripts/pr_check/__init__.py | 1 + scripts/pr_check/__main__.py | 4 + scripts/pr_check/checker.py | 467 +++++++++++++++++++++++++++++++++++ 3 files changed, 472 insertions(+) create mode 100644 scripts/pr_check/__init__.py create mode 100644 scripts/pr_check/__main__.py create mode 100644 scripts/pr_check/checker.py diff --git a/scripts/pr_check/__init__.py b/scripts/pr_check/__init__.py new file mode 100644 index 00000000..90fe763c --- /dev/null +++ b/scripts/pr_check/__init__.py @@ -0,0 +1 @@ +"""Periodic LLVM PR coverage gap checking helpers.""" diff --git a/scripts/pr_check/__main__.py b/scripts/pr_check/__main__.py new file mode 100644 index 00000000..75a273f1 --- /dev/null +++ b/scripts/pr_check/__main__.py @@ -0,0 +1,4 @@ +from .checker import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py new file mode 100644 index 00000000..eb68d800 --- /dev/null +++ b/scripts/pr_check/checker.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Discover, plan, and record periodic LLVM PR coverage-gap checks.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pandas as pd + +STATE_VERSION = 1 +DEFAULT_GITHUB_REPO = "llvm/llvm-project" +DEFAULT_SEARCH_LIMIT = 100 + +BACKEND_SEARCH_QUERIES: dict[str, str] = { + "amdgpu": "path:llvm/lib/Target/AMDGPU", + "spirv": "path:llvm/lib/Target/SPIRV", +} + +SEARCH_JSON_FIELDS = ["number", "title", "updatedAt"] +PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt"] + + +class PrCheckerError(Exception): + """Error from PR checker operations.""" + + +@dataclass(frozen=True) +class DiscoveredPr: + """An open PR that touches at least one tracked backend.""" + + pr_number: int + title: str + head_sha: str + updated_at: str + backends: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class WorkItem: + """A PR/backend pair that should be checked.""" + + pr_number: int + backend: str + title: str + head_sha: str + reason: str + + +@dataclass +class StateEntry: + """Persistent record for one PR/backend check.""" + + pr_number: int + backend: str + title: str + head_sha: str + status: str + gap_count: int + lit_failure_count: int + checked_at: str + output_dir: str + error: str | None = None + + +def entry_key(pr_number: int, backend: str) -> str: + return f"{pr_number}:{backend}" + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _require_gh() -> str: + from shutil import which + + gh_path = which("gh") + if gh_path is None: + raise PrCheckerError("required command not found: gh") + return gh_path + + +def _run_gh(args: list[str], *, timeout: int = 120) -> str: + gh_path = _require_gh() + try: + result = subprocess.run( + [gh_path, *args], + capture_output=True, + text=True, + encoding="utf-8", + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise PrCheckerError(f"gh command timed out after {timeout}s: {' '.join(args)}") from exc + except OSError as exc: + raise PrCheckerError(f"failed to run gh: {exc}") from exc + + if result.returncode != 0: + stderr = (result.stderr or "").strip() or "(no error message)" + raise PrCheckerError(f"gh {' '.join(args)} failed: {stderr}") + + return result.stdout + + +def _gh_json(args: list[str], *, timeout: int = 120) -> Any: + stdout = _run_gh([*args, "--json", ",".join(_json_fields_for_command(args))], timeout=timeout) + if not stdout.strip(): + return [] + try: + return json.loads(stdout) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"gh returned invalid JSON: {exc.msg}") from exc + + +def _json_fields_for_command(args: list[str]) -> list[str]: + if args and args[0] == "search": + return SEARCH_JSON_FIELDS + if args and args[0] == "pr" and len(args) >= 2 and args[1] == "view": + return PR_VIEW_JSON_FIELDS + raise PrCheckerError(f"unsupported gh command for JSON parsing: {' '.join(args)}") + + +def _search_backend_prs( + backend: str, + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, +) -> list[dict[str, Any]]: + query = BACKEND_SEARCH_QUERIES[backend] + payload = _gh_json( + [ + "search", + "prs", + query, + "--repo", + github_repo, + "--state", + "open", + "--limit", + str(limit), + ] + ) + if not isinstance(payload, list): + raise PrCheckerError(f"unexpected gh search payload for {backend}: {type(payload)!r}") + return payload + + +def _view_pr(pr_number: int, github_repo: str) -> dict[str, Any]: + payload = _gh_json(["pr", "view", str(pr_number), "--repo", github_repo]) + if not isinstance(payload, dict): + raise PrCheckerError(f"unexpected gh pr view payload for #{pr_number}: {type(payload)!r}") + return payload + + +def _search_results_dataframe( + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, +) -> pd.DataFrame: + """Run backend searches and return one row per (PR, backend) match.""" + frames: list[pd.DataFrame] = [] + for backend in BACKEND_SEARCH_QUERIES: + items = _search_backend_prs(backend, github_repo=github_repo, limit=limit) + if not items: + continue + frame = pd.DataFrame(items) + frame["backend"] = backend + frames.append(frame) + + if not frames: + return pd.DataFrame(columns=["number", "title", "updatedAt", "backend"]) + + return pd.concat(frames, ignore_index=True) + + +def _non_empty_first(values: pd.Series) -> str: + for value in values: + if pd.notna(value) and str(value): + return str(value) + return "" + + +def _aggregate_search_results(search_df: pd.DataFrame) -> pd.DataFrame: + """Collapse per-backend search hits to one row per PR.""" + if search_df.empty: + return pd.DataFrame(columns=["pr_number", "title", "updated_at", "backends"]) + + return ( + search_df.groupby("number", sort=True) + .agg( + title=("title", _non_empty_first), + updated_at=("updatedAt", "max"), + backends=("backend", lambda values: sorted(set(values))), + ) + .reset_index() + .rename(columns={"number": "pr_number"}) + ) + + +def discover_prs(*, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT) -> list[DiscoveredPr]: + """Find open PRs touching AMDGPU and/or SPIR-V target paths.""" + grouped = _aggregate_search_results( + _search_results_dataframe(github_repo=github_repo, limit=limit) + ) + if grouped.empty: + return [] + + discovered: list[DiscoveredPr] = [] + for row in grouped.itertuples(index=False): + pr_number = int(row.pr_number) + pr_view = _view_pr(pr_number, github_repo) + head_sha = pr_view.get("headRefOid") + if not head_sha: + raise PrCheckerError(f"could not resolve headRefOid for {github_repo}#{pr_number}") + + discovered.append( + DiscoveredPr( + pr_number=pr_number, + title=pr_view.get("title") or row.title or "", + head_sha=head_sha, + updated_at=pr_view.get("updatedAt") or row.updated_at or "", + backends=list(row.backends), + ) + ) + + return discovered + + +def load_state(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"version": STATE_VERSION, "entries": {}} + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"invalid state file {path}: {exc.msg}") from exc + + if not isinstance(payload, dict): + raise PrCheckerError(f"invalid state file {path}: expected object at top level") + if payload.get("version") != STATE_VERSION: + raise PrCheckerError( + f"unsupported state version in {path}: {payload.get('version')!r} (expected {STATE_VERSION})" + ) + if not isinstance(payload.get("entries"), dict): + raise PrCheckerError(f"invalid state file {path}: missing entries object") + + return payload + + +def save_state(path: Path, state: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _state_entry_from_dict(raw: dict[str, Any]) -> StateEntry: + return StateEntry( + pr_number=int(raw["pr_number"]), + backend=str(raw["backend"]), + title=str(raw.get("title", "")), + head_sha=str(raw["head_sha"]), + status=str(raw["status"]), + gap_count=int(raw.get("gap_count", 0)), + lit_failure_count=int(raw.get("lit_failure_count", 0)), + checked_at=str(raw.get("checked_at", "")), + output_dir=str(raw.get("output_dir", "")), + error=raw.get("error"), + ) + + +def plan_work(discovered: list[DiscoveredPr], state: dict[str, Any]) -> list[WorkItem]: + """Return PR/backend pairs that need a fresh coverage run.""" + entries: dict[str, Any] = state["entries"] + work: list[WorkItem] = [] + + for pr in discovered: + for backend in pr.backends: + key = entry_key(pr.pr_number, backend) + existing = entries.get(key) + if existing is None: + work.append( + WorkItem( + pr_number=pr.pr_number, + backend=backend, + title=pr.title, + head_sha=pr.head_sha, + reason="new", + ) + ) + continue + + previous = _state_entry_from_dict(existing) + if previous.head_sha != pr.head_sha: + work.append( + WorkItem( + pr_number=pr.pr_number, + backend=backend, + title=pr.title, + head_sha=pr.head_sha, + reason="head_changed", + ) + ) + + work.sort(key=lambda item: (item.pr_number, item.backend)) + return work + + +def record_result( + state: dict[str, Any], + *, + pr_number: int, + backend: str, + title: str, + head_sha: str, + status: str, + gap_count: int, + lit_failure_count: int, + output_dir: str, + error: str | None = None, +) -> None: + key = entry_key(pr_number, backend) + state["entries"][key] = { + "pr_number": pr_number, + "backend": backend, + "title": title, + "head_sha": head_sha, + "status": status, + "gap_count": gap_count, + "lit_failure_count": lit_failure_count, + "checked_at": utc_now_iso(), + "output_dir": output_dir, + "error": error, + } + + +def _discovered_to_json(discovered: list[DiscoveredPr]) -> list[dict[str, Any]]: + return [ + { + "pr_number": pr.pr_number, + "title": pr.title, + "head_sha": pr.head_sha, + "updated_at": pr.updated_at, + "backends": pr.backends, + } + for pr in discovered + ] + + +def _work_to_json(work: list[WorkItem]) -> list[dict[str, Any]]: + return [asdict(item) for item in work] + + +def cmd_discover(args: argparse.Namespace) -> int: + discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) + payload = _discovered_to_json(discovered) + print(json.dumps(payload, indent=2)) + return 0 + + +def cmd_plan(args: argparse.Namespace) -> int: + discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) + state = load_state(args.state_file) + work = plan_work(discovered, state) + if args.max_items is not None: + work = work[: args.max_items] + print(json.dumps(_work_to_json(work), indent=2)) + return 0 + + +def cmd_record(args: argparse.Namespace) -> int: + state = load_state(args.state_file) + record_result( + state, + pr_number=args.pr_number, + backend=args.backend, + title=args.title, + head_sha=args.head_sha, + status=args.status, + gap_count=args.gap_count, + lit_failure_count=args.lit_failure_count, + output_dir=args.output_dir, + error=args.error, + ) + save_state(args.state_file, state) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--github-repo", + default=DEFAULT_GITHUB_REPO, + help=f"GitHub repo hosting PRs (default: {DEFAULT_GITHUB_REPO})", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + discover_parser = subparsers.add_parser("discover", help="List open target PRs") + discover_parser.add_argument( + "--limit", + type=int, + default=DEFAULT_SEARCH_LIMIT, + help=f"Max PRs per backend search (default: {DEFAULT_SEARCH_LIMIT})", + ) + discover_parser.set_defaults(func=cmd_discover) + + plan_parser = subparsers.add_parser("plan", help="List PR/backend pairs needing a run") + plan_parser.add_argument( + "--state-file", + type=Path, + required=True, + help="Path to state.json", + ) + plan_parser.add_argument( + "--limit", + type=int, + default=DEFAULT_SEARCH_LIMIT, + help=f"Max PRs per backend search (default: {DEFAULT_SEARCH_LIMIT})", + ) + plan_parser.add_argument( + "--max-items", + type=int, + default=None, + help="Cap the number of planned work items", + ) + plan_parser.set_defaults(func=cmd_plan) + + record_parser = subparsers.add_parser("record", help="Persist one check result") + record_parser.add_argument("--state-file", type=Path, required=True) + record_parser.add_argument("--pr-number", type=int, required=True) + record_parser.add_argument("--backend", choices=sorted(BACKEND_SEARCH_QUERIES), required=True) + record_parser.add_argument("--title", default="") + record_parser.add_argument("--head-sha", required=True) + record_parser.add_argument( + "--status", + choices=["gaps", "clean", "failed"], + required=True, + ) + record_parser.add_argument("--gap-count", type=int, default=0) + record_parser.add_argument("--lit-failure-count", type=int, default=0) + record_parser.add_argument("--output-dir", required=True) + record_parser.add_argument("--error", default=None) + record_parser.set_defaults(func=cmd_record) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return args.func(args) + except PrCheckerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From bc63cf046d1ea20244611c6e4c0f85329dffe3d3 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 13:17:51 -0500 Subject: [PATCH 02/26] Add check-llvm-prs orchestrator for periodic PR coverage checks. Wraps build-image-pr.sh and pr-cov-gaps-detection.sh with discover/plan execution, backend-specific Docker tags, and evaluate-output recording. --- scripts/pr-check/check-llvm-prs.sh | 324 +++++++++++++++++++++++++++++ scripts/pr_check/checker.py | 62 ++++++ 2 files changed, 386 insertions(+) create mode 100755 scripts/pr-check/check-llvm-prs.sh diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh new file mode 100755 index 00000000..f589ffe6 --- /dev/null +++ b/scripts/pr-check/check-llvm-prs.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash +# Periodic orchestrator for LLVM AMDGPU/SPIR-V PR coverage-gap detection. +# +# Discovers open target PRs, plans work against state.json, and runs +# pr-cov-gaps-detection.sh for PR/backend pairs whose head SHA changed. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +LLVM_REPO="${LLVM_REPO:-}" +PR_CHECK_JOBS="${PR_CHECK_JOBS:-$(nproc)}" +PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" +PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" +PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" +PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" +GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" +IMAGE_NAME="${IMAGE_NAME:-fuzz-fill-test}" + +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" + +discover_only=0 +plan_only=0 + +usage() { + cat < Load environment overrides from a file + (default: ${SCRIPT_DIR}/config.env if present) + --llvm-repo Local llvm-project clone (required unless set in config) + --state-file State file (default: ${PR_CHECK_STATE_FILE}) + --output-root Per-run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) + --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) + --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) + --github-repo + GitHub repo hosting PRs (default: ${GITHUB_REPO}) + --help, -h Show this help + +Configuration: + Copy ${SCRIPT_DIR}/config.example.env to ${SCRIPT_DIR}/config.env and set + LLVM_REPO before running under cron. + +Examples: + $(basename "$0") --discover-only + $(basename "$0") --plan-only + $(basename "$0") --llvm-repo /path/to/llvm-project +EOF +} + +load_config() { + if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck source=/dev/null + source "$CONFIG_FILE" + fi +} + +run_pr_check() { + PYTHONPATH="${REPO_ROOT}/scripts" python3 -m pr_check "$@" +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +validate_positive_int() { + local name="$1" + local value="$2" + if [[ ! "$value" =~ ^[0-9]+$ ]] || [[ "$value" -eq 0 ]]; then + echo "error: ${name} must be a positive integer: ${value}" >&2 + exit 1 + fi +} + +validate_runtime_config() { + if [[ -z "$LLVM_REPO" ]]; then + echo "error: LLVM_REPO is required (set in config.env or pass --llvm-repo)" >&2 + exit 1 + fi + if [[ ! -d "$LLVM_REPO" ]]; then + echo "error: LLVM_REPO is not a directory: ${LLVM_REPO}" >&2 + exit 1 + fi + validate_positive_int "PR_CHECK_JOBS" "$PR_CHECK_JOBS" + validate_positive_int "PR_CHECK_MAX_PER_RUN" "$PR_CHECK_MAX_PER_RUN" + validate_positive_int "PR_CHECK_SEARCH_LIMIT" "$PR_CHECK_SEARCH_LIMIT" +} + +record_failed_run() { + local pr_number="$1" + local backend="$2" + local title="$3" + local head_sha="$4" + local output_dir="$5" + local error_message="$6" + + run_pr_check record \ + --state-file "$PR_CHECK_STATE_FILE" \ + --pr-number "$pr_number" \ + --backend "$backend" \ + --title "$title" \ + --head-sha "$head_sha" \ + --status failed \ + --gap-count 0 \ + --lit-failure-count 0 \ + --output-dir "$output_dir" \ + --error "$error_message" +} + +run_one_work_item() { + local pr_number="$1" + local backend="$2" + local title="$3" + local head_sha="$4" + + local output_dir="${PR_CHECK_OUTPUT_ROOT}/${pr_number}-${backend}" + local image_tag="llvm-pr-${pr_number}-${backend}" + local image_ref="${IMAGE_NAME}:${image_tag}" + + mkdir -p "$output_dir" + output_dir="$(realpath "$output_dir")" + + echo "=== PR #${pr_number} (${backend}) ===" + echo "Title: ${title}" + echo "Head: ${head_sha}" + echo "Output: ${output_dir}" + + set +e + "${REPO_ROOT}/scripts/docker/build-image-pr.sh" \ + --llvm-repo "$LLVM_REPO" \ + --pr-id "$pr_number" \ + --allowlist "$backend" \ + --github-repo "$GITHUB_REPO" \ + --tag "$image_tag" \ + -j "$PR_CHECK_JOBS" + build_status=$? + + if [[ "$build_status" -ne 0 ]]; then + set -e + record_failed_run "$pr_number" "$backend" "$title" "$head_sha" "$output_dir" "docker image build failed" + return "$build_status" + fi + + "${REPO_ROOT}/scripts/docker/pr-cov-gaps-detection.sh" \ + --image "$image_ref" \ + --output-dir "$output_dir" \ + --keep-image \ + -j "$PR_CHECK_JOBS" + detect_status=$? + set -e + + if [[ "$detect_status" -ne 0 ]]; then + record_failed_run "$pr_number" "$backend" "$title" "$head_sha" "$output_dir" "coverage-gap detection failed" + return "$detect_status" + fi + + local evaluation + evaluation="$(run_pr_check evaluate-output --output-dir "$output_dir")" + local gap_count lit_failure_count status + gap_count="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["gap_count"])' <<<"$evaluation")" + lit_failure_count="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["lit_failure_count"])' <<<"$evaluation")" + status="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])' <<<"$evaluation")" + + run_pr_check record \ + --state-file "$PR_CHECK_STATE_FILE" \ + --pr-number "$pr_number" \ + --backend "$backend" \ + --title "$title" \ + --head-sha "$head_sha" \ + --status "$status" \ + --gap-count "$gap_count" \ + --lit-failure-count "$lit_failure_count" \ + --output-dir "$output_dir" + + echo "Result: status=${status} gap_count=${gap_count} lit_failures=${lit_failure_count}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --discover-only) + discover_only=1 + shift + ;; + --plan-only) + plan_only=1 + shift + ;; + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --llvm-repo) + [[ $# -ge 2 ]] || { echo "error: --llvm-repo requires a value" >&2; exit 2; } + LLVM_REPO="$2" + shift 2 + ;; + --state-file) + [[ $# -ge 2 ]] || { echo "error: --state-file requires a value" >&2; exit 2; } + PR_CHECK_STATE_FILE="$2" + shift 2 + ;; + --output-root) + [[ $# -ge 2 ]] || { echo "error: --output-root requires a value" >&2; exit 2; } + PR_CHECK_OUTPUT_ROOT="$2" + shift 2 + ;; + --max-per-run) + [[ $# -ge 2 ]] || { echo "error: --max-per-run requires a value" >&2; exit 2; } + PR_CHECK_MAX_PER_RUN="$2" + shift 2 + ;; + --jobs) + [[ $# -ge 2 ]] || { echo "error: --jobs requires a value" >&2; exit 2; } + PR_CHECK_JOBS="$2" + shift 2 + ;; + --github-repo) + [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; } + GITHUB_REPO="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ $# -gt 0 ]]; then + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 +fi + +load_config +require_command python3 +require_command docker +require_command gh +require_command git + +mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" "$PR_CHECK_OUTPUT_ROOT" + +common_args=( + --github-repo "$GITHUB_REPO" +) + +if [[ "$discover_only" -eq 1 ]]; then + run_pr_check discover "${common_args[@]}" --limit "$PR_CHECK_SEARCH_LIMIT" + exit 0 +fi + +if [[ "$plan_only" -eq 1 ]]; then + run_pr_check plan \ + "${common_args[@]}" \ + --state-file "$PR_CHECK_STATE_FILE" \ + --limit "$PR_CHECK_SEARCH_LIMIT" \ + --max-items "$PR_CHECK_MAX_PER_RUN" + exit 0 +fi + +validate_runtime_config + +work_json="$(run_pr_check plan \ + "${common_args[@]}" \ + --state-file "$PR_CHECK_STATE_FILE" \ + --limit "$PR_CHECK_SEARCH_LIMIT" \ + --max-items "$PR_CHECK_MAX_PER_RUN")" + +work_count="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$work_json")" +if [[ "$work_count" -eq 0 ]]; then + echo "No PR/backend pairs need checking." + exit 0 +fi + +echo "Planned ${work_count} PR/backend check(s)." + +failures=0 +while IFS= read -r item; do + pr_number="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["pr_number"])' "$item")" + backend="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["backend"])' "$item")" + title="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["title"])' "$item")" + head_sha="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["head_sha"])' "$item")" + reason="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["reason"])' "$item")" + + echo + echo ">>> Running check (${reason}): #${pr_number} ${backend}" + if ! run_one_work_item "$pr_number" "$backend" "$title" "$head_sha"; then + failures=$((failures + 1)) + fi +done < <(python3 -c 'import json,sys; print("\n".join(json.dumps(x) for x in json.load(sys.stdin)))' <<<"$work_json") + +if [[ "$failures" -gt 0 ]]; then + echo + echo "${failures} check(s) failed (see state file for details)." + exit 1 +fi + +echo +echo "All planned checks completed successfully." diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index eb68d800..2c82b7a9 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import csv import json import subprocess import sys @@ -28,6 +29,7 @@ SEARCH_JSON_FIELDS = ["number", "title", "updatedAt"] PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt"] +LIT_FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"}) class PrCheckerError(Exception): @@ -359,6 +361,53 @@ def _work_to_json(work: list[WorkItem]) -> list[dict[str, Any]]: return [asdict(item) for item in work] +def count_csv_data_rows(path: Path) -> int: + """Return the number of data rows in a CSV file (excluding the header).""" + if not path.is_file(): + return 0 + + with path.open(encoding="utf-8", newline="") as handle: + reader = csv.reader(handle) + next(reader, None) + return sum(1 for _ in reader) + + +def count_lit_failures(path: Path) -> int: + """Count LIT tests that failed during baseline collection.""" + if not path.is_file(): + return 0 + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return 0 + + tests = payload.get("tests", []) + if not isinstance(tests, list): + return 0 + + return sum( + 1 + for test in tests + if isinstance(test, dict) and test.get("code") in LIT_FAILURE_CODES + ) + + +def evaluate_output_dir(output_dir: Path) -> dict[str, Any]: + """Summarize a completed pr-cov-gaps-detection output directory.""" + gap_csv = output_dir / "commit_lines_report" / "target_lines_uncovered.csv" + lit_failures_json = output_dir / "baseline" / "lit_failures.json" + gap_count = count_csv_data_rows(gap_csv) + lit_failure_count = count_lit_failures(lit_failures_json) + return { + "gap_count": gap_count, + "lit_failure_count": lit_failure_count, + "status": "gaps" if gap_count > 0 else "clean", + "gap_report": str(gap_csv), + "lit_failures_json": str(lit_failures_json), + } + + def cmd_discover(args: argparse.Namespace) -> int: discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) payload = _discovered_to_json(discovered) @@ -394,6 +443,12 @@ def cmd_record(args: argparse.Namespace) -> int: return 0 +def cmd_evaluate_output(args: argparse.Namespace) -> int: + payload = evaluate_output_dir(args.output_dir) + print(json.dumps(payload, indent=2)) + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -450,6 +505,13 @@ def build_parser() -> argparse.ArgumentParser: record_parser.add_argument("--error", default=None) record_parser.set_defaults(func=cmd_record) + evaluate_parser = subparsers.add_parser( + "evaluate-output", + help="Summarize gap and LIT failure counts from a run output directory", + ) + evaluate_parser.add_argument("--output-dir", type=Path, required=True) + evaluate_parser.set_defaults(func=cmd_evaluate_output) + return parser From 51c2fe8bc0ca476230a810e76eea52ebcefe7adb Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 14:26:41 -0500 Subject: [PATCH 03/26] Add coverage gap report generation for periodic PR checks. Introduces a report subcommand that writes latest.json and latest.md from state using pandas for gap CSV loading and aggregation, and wires report generation into check-llvm-prs after each run. --- scripts/pr-check/check-llvm-prs.sh | 31 ++- scripts/pr_check/checker.py | 295 ++++++++++++++++++++++++++++- 2 files changed, 318 insertions(+), 8 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index f589ffe6..e7c38fc2 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -15,6 +15,7 @@ PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" +PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" IMAGE_NAME="${IMAGE_NAME:-fuzz-fill-test}" @@ -22,6 +23,7 @@ CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" discover_only=0 plan_only=0 +report_only=0 usage() { cat < Load environment overrides from a file (default: ${SCRIPT_DIR}/config.env if present) --llvm-repo Local llvm-project clone (required unless set in config) --state-file State file (default: ${PR_CHECK_STATE_FILE}) --output-root Per-run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) + --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) --github-repo @@ -66,6 +70,14 @@ run_pr_check() { PYTHONPATH="${REPO_ROOT}/scripts" python3 -m pr_check "$@" } +generate_report() { + echo "=== coverage gap report ===" + run_pr_check report \ + --github-repo "$GITHUB_REPO" \ + --state-file "$PR_CHECK_STATE_FILE" \ + --report-dir "$PR_CHECK_REPORT_DIR" +} + require_command() { if ! command -v "$1" >/dev/null 2>&1; then echo "error: required command not found: $1" >&2 @@ -195,6 +207,10 @@ while [[ $# -gt 0 ]]; do plan_only=1 shift ;; + --report-only) + report_only=1 + shift + ;; --config) [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } CONFIG_FILE="$2" @@ -215,6 +231,11 @@ while [[ $# -gt 0 ]]; do PR_CHECK_OUTPUT_ROOT="$2" shift 2 ;; + --report-dir) + [[ $# -ge 2 ]] || { echo "error: --report-dir requires a value" >&2; exit 2; } + PR_CHECK_REPORT_DIR="$2" + shift 2 + ;; --max-per-run) [[ $# -ge 2 ]] || { echo "error: --max-per-run requires a value" >&2; exit 2; } PR_CHECK_MAX_PER_RUN="$2" @@ -263,12 +284,17 @@ require_command docker require_command gh require_command git -mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" "$PR_CHECK_OUTPUT_ROOT" +mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" "$PR_CHECK_OUTPUT_ROOT" "$PR_CHECK_REPORT_DIR" common_args=( --github-repo "$GITHUB_REPO" ) +if [[ "$report_only" -eq 1 ]]; then + generate_report + exit 0 +fi + if [[ "$discover_only" -eq 1 ]]; then run_pr_check discover "${common_args[@]}" --limit "$PR_CHECK_SEARCH_LIMIT" exit 0 @@ -294,6 +320,7 @@ work_json="$(run_pr_check plan \ work_count="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$work_json")" if [[ "$work_count" -eq 0 ]]; then echo "No PR/backend pairs need checking." + generate_report exit 0 fi @@ -317,8 +344,10 @@ done < <(python3 -c 'import json,sys; print("\n".join(json.dumps(x) for x in jso if [[ "$failures" -gt 0 ]]; then echo echo "${failures} check(s) failed (see state file for details)." + generate_report exit 1 fi echo echo "All planned checks completed successfully." +generate_report diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index 2c82b7a9..54173fde 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -7,7 +7,6 @@ from __future__ import annotations import argparse -import csv import json import subprocess import sys @@ -361,15 +360,38 @@ def _work_to_json(work: list[WorkItem]) -> list[dict[str, Any]]: return [asdict(item) for item in work] +def _read_gap_csv(gap_csv: Path, *, max_rows: int | None = None) -> pd.DataFrame: + """Load a target_lines_uncovered.csv file as a normalized DataFrame.""" + if not gap_csv.is_file(): + return pd.DataFrame(columns=["file", "line_no", "text"]) + + frame = pd.read_csv(gap_csv, nrows=max_rows) + if frame.empty: + return pd.DataFrame(columns=["file", "line_no", "text"]) + + for column in ("file", "line_no", "text"): + if column not in frame.columns: + frame[column] = "" + + normalized = frame[["file", "line_no", "text"]].fillna("").astype(str) + return normalized.apply(lambda series: series.str.strip()) + + def count_csv_data_rows(path: Path) -> int: """Return the number of data rows in a CSV file (excluding the header).""" - if not path.is_file(): - return 0 + return len(_read_gap_csv(path)) - with path.open(encoding="utf-8", newline="") as handle: - reader = csv.reader(handle) - next(reader, None) - return sum(1 for _ in reader) + +def load_gap_rows(gap_csv: Path, *, max_rows: int = 20) -> list[dict[str, str]]: + """Read uncovered line rows from a target_lines_uncovered.csv file.""" + if max_rows <= 0: + return [] + + frame = _read_gap_csv(gap_csv, max_rows=max_rows) + if frame.empty: + return [] + + return frame.to_dict(orient="records") def count_lit_failures(path: Path) -> int: @@ -408,6 +430,223 @@ def evaluate_output_dir(output_dir: Path) -> dict[str, Any]: } +def pr_url(github_repo: str, pr_number: int) -> str: + return f"https://github.com/{github_repo}/pull/{pr_number}" + + +def _state_entries_dataframe(state: dict[str, Any]) -> pd.DataFrame: + """Convert state entries into a flat DataFrame for report aggregation.""" + rows: list[dict[str, Any]] = [] + for key, raw in state.get("entries", {}).items(): + if not isinstance(raw, dict): + continue + + entry = _state_entry_from_dict(raw) + rows.append( + { + "key": key, + "pr_number": entry.pr_number, + "backend": entry.backend, + "title": entry.title, + "head_sha": entry.head_sha, + "status": entry.status, + "gap_count": entry.gap_count, + "lit_failure_count": entry.lit_failure_count, + "checked_at": entry.checked_at, + "output_dir": entry.output_dir, + "error": entry.error, + } + ) + + columns = [ + "key", + "pr_number", + "backend", + "title", + "head_sha", + "status", + "gap_count", + "lit_failure_count", + "checked_at", + "output_dir", + "error", + ] + if not rows: + return pd.DataFrame(columns=columns) + + return pd.DataFrame(rows) + + +def _attach_sample_gaps(frame: pd.DataFrame, *, max_gap_lines: int) -> pd.DataFrame: + """Add a sample_gaps list column for entries that reported coverage gaps.""" + + def sample_gaps_for_row(row: pd.Series) -> list[dict[str, str]]: + if row["status"] != "gaps" or int(row["gap_count"]) <= 0: + return [] + gap_csv = Path(row["output_dir"]) / "commit_lines_report" / "target_lines_uncovered.csv" + return load_gap_rows(gap_csv, max_rows=max_gap_lines) + + enriched = frame.copy() + enriched["sample_gaps"] = enriched.apply(sample_gaps_for_row, axis=1) + return enriched + + +def build_report_payload( + state: dict[str, Any], + *, + github_repo: str = DEFAULT_GITHUB_REPO, + max_gap_lines: int = 20, +) -> dict[str, Any]: + """Aggregate state entries into a report payload.""" + frame = _state_entries_dataframe(state) + if frame.empty: + return { + "generated_at": utc_now_iso(), + "github_repo": github_repo, + "summary": { + "total_entries": 0, + "with_gaps": 0, + "clean": 0, + "failed": 0, + }, + "entries_with_gaps": [], + "failed_entries": [], + "all_entries": [], + } + + frame = _attach_sample_gaps(frame, max_gap_lines=max_gap_lines) + frame["pr_url"] = frame["pr_number"].map(lambda number: pr_url(github_repo, int(number))) + + status_counts = frame["status"].value_counts() + with_gaps_frame = frame[frame["status"] == "gaps"].sort_values( + ["gap_count", "pr_number", "backend"], + ascending=[False, True, True], + ) + failed_frame = frame[frame["status"] == "failed"].sort_values(["pr_number", "backend"]) + all_entries_frame = frame.sort_values(["pr_number", "backend"]) + + return { + "generated_at": utc_now_iso(), + "github_repo": github_repo, + "summary": { + "total_entries": int(len(frame)), + "with_gaps": int(status_counts.get("gaps", 0)), + "clean": int(status_counts.get("clean", 0)), + "failed": int(status_counts.get("failed", 0)), + }, + "entries_with_gaps": with_gaps_frame.to_dict(orient="records"), + "failed_entries": failed_frame.to_dict(orient="records"), + "all_entries": all_entries_frame.to_dict(orient="records"), + } + + +def _markdown_escape_cell(value: str) -> str: + return value.replace("|", "\\|").replace("\n", " ") + + +def render_report_markdown(report: dict[str, Any]) -> str: + """Render a human-readable Markdown summary.""" + lines = [ + "# LLVM PR coverage gap report", + "", + f"Generated: {report['generated_at']}", + f"Repository: {report['github_repo']}", + ] + summary = report["summary"] + lines.extend( + [ + ( + "Entries: " + f"{summary['total_entries']} total, " + f"{summary['with_gaps']} with gaps, " + f"{summary['clean']} clean, " + f"{summary['failed']} failed" + ), + "", + ] + ) + + entries_with_gaps = report.get("entries_with_gaps", []) + if entries_with_gaps: + lines.append("## PRs with coverage gaps") + lines.append("") + for entry in entries_with_gaps: + lines.append( + f"### #{entry['pr_number']} ({entry['backend']}) — " + f"{entry['gap_count']} uncovered line(s)" + ) + lines.append("") + lines.append(f"- **Title:** {entry['title']}") + lines.append(f"- **PR:** {entry['pr_url']}") + lines.append(f"- **Head:** `{entry['head_sha'][:12]}`") + lines.append(f"- **Checked:** {entry['checked_at']}") + if entry["lit_failure_count"] > 0: + lines.append( + f"- **Warning:** {entry['lit_failure_count']} LIT failure(s) during baseline" + ) + lines.append("") + sample_gaps = entry.get("sample_gaps", []) + if sample_gaps: + lines.extend(["| File | Line | Text |", "|------|------|------|"]) + for row in sample_gaps: + text = _markdown_escape_cell(row.get("text", "")) + file_path = _markdown_escape_cell(row.get("file", "")) + lines.append(f"| `{file_path}` | {row.get('line_no', '')} | `{text}` |") + remaining = entry["gap_count"] - len(sample_gaps) + if remaining > 0: + lines.extend( + [ + "", + ( + f"*…and {remaining} more line(s). See " + f"`{entry['output_dir']}/commit_lines_report/target_lines_uncovered.csv`*" + ), + ] + ) + lines.append("") + + failed_entries = report.get("failed_entries", []) + if failed_entries: + lines.extend(["## Failed checks", ""]) + for entry in failed_entries: + lines.append( + f"- #{entry['pr_number']} ({entry['backend']}): {entry.get('error') or 'unknown error'}" + ) + lines.append("") + + if not entries_with_gaps and not failed_entries: + lines.append("No PRs with coverage gaps or failed checks in the current state.") + lines.append("") + + return "\n".join(lines) + + +def write_reports( + report_dir: Path, + payload: dict[str, Any], + *, + write_run_snapshot: bool = True, +) -> dict[str, Path]: + """Write latest.json, latest.md, and an optional timestamped snapshot.""" + report_dir.mkdir(parents=True, exist_ok=True) + + latest_json = report_dir / "latest.json" + latest_md = report_dir / "latest.md" + latest_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + latest_md.write_text(render_report_markdown(payload) + "\n", encoding="utf-8") + + written = {"latest_json": latest_json, "latest_md": latest_md} + if write_run_snapshot: + runs_dir = report_dir / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + snapshot_name = payload["generated_at"].replace(":", "").replace("+00:00", "Z") + snapshot_path = runs_dir / f"{snapshot_name}.json" + snapshot_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + written["run_snapshot"] = snapshot_path + + return written + + def cmd_discover(args: argparse.Namespace) -> int: discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) payload = _discovered_to_json(discovered) @@ -449,6 +688,27 @@ def cmd_evaluate_output(args: argparse.Namespace) -> int: return 0 +def cmd_report(args: argparse.Namespace) -> int: + state = load_state(args.state_file) + payload = build_report_payload( + state, + github_repo=args.github_repo, + max_gap_lines=args.max_gap_lines, + ) + written = write_reports( + args.report_dir, + payload, + write_run_snapshot=not args.no_run_snapshot, + ) + print( + json.dumps( + {key: str(path) for key, path in written.items()}, + indent=2, + ) + ) + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -512,6 +772,27 @@ def build_parser() -> argparse.ArgumentParser: evaluate_parser.add_argument("--output-dir", type=Path, required=True) evaluate_parser.set_defaults(func=cmd_evaluate_output) + report_parser = subparsers.add_parser("report", help="Write latest.json and latest.md from state") + report_parser.add_argument("--state-file", type=Path, required=True) + report_parser.add_argument( + "--report-dir", + type=Path, + required=True, + help="Directory for latest.json, latest.md, and runs/ snapshots", + ) + report_parser.add_argument( + "--max-gap-lines", + type=int, + default=20, + help="Max uncovered lines to include per PR in the report (default: 20)", + ) + report_parser.add_argument( + "--no-run-snapshot", + action="store_true", + help="Skip writing report_dir/runs/.json", + ) + report_parser.set_defaults(func=cmd_report) + return parser From 47530719f5188caa0e5e987b204f30df7341f95b Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 14:41:03 -0500 Subject: [PATCH 04/26] Document periodic LLVM PR coverage checking setup. Adds config.example.env, scripts/pr-check/README.md with cron/systemd examples, and a README cross-link for the check-llvm-prs workflow. --- README.md | 14 ++++ scripts/pr-check/README.md | 122 ++++++++++++++++++++++++++++ scripts/pr-check/config.example.env | 26 ++++++ 3 files changed, 162 insertions(+) create mode 100644 scripts/pr-check/README.md create mode 100644 scripts/pr-check/config.example.env diff --git a/README.md b/README.md index c075fee1..fa8d701f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Replace `HEAD` with a hash, branch, or `main~3` as needed. - [Build](#build) - [Build from an LLVM pull request](#build-from-an-llvm-pull-request) - [Workflow 2: PR coverage gap detection](#workflow-2-pr-coverage-gap-detection) + - [Periodic PR checking](#periodic-pr-checking) - [Run integration tests](#run-integration-tests) - [Run a container](#run-a-container) - [Tests](#tests) @@ -404,6 +405,19 @@ If the image `fuzz-fill-test:llvm-pr-` already exists, omit `--build-image` t Main output: `/commit_lines_report/target_lines_uncovered.csv`. See [Workflow 2](#workflow-2-uncovered-lines-in-a-commit) for report semantics. +### Periodic PR checking + +To automatically check open AMDGPU and SPIR-V PRs on a schedule, use [`scripts/pr-check/check-llvm-prs.sh`](scripts/pr-check/check-llvm-prs.sh). It discovers PRs via `gh`, re-runs gap detection when a PR head SHA changes, and writes local reports under `data/pr-check/reports/`. + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Set LLVM_REPO in config.env, then: +./scripts/pr-check/check-llvm-prs.sh --discover-only +./scripts/pr-check/check-llvm-prs.sh +``` + +See [`scripts/pr-check/README.md`](scripts/pr-check/README.md) for cron/systemd setup, configuration options, and output layout. + ### Run integration tests ```bash diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md new file mode 100644 index 00000000..8495276d --- /dev/null +++ b/scripts/pr-check/README.md @@ -0,0 +1,122 @@ +# Periodic LLVM PR coverage gap checking + +Automatically discover open AMDGPU and SPIR-V PRs on `llvm/llvm-project`, run [Workflow 2](../docker/pr-cov-gaps-detection.sh) coverage-gap detection when a PR's head SHA changes, and write local reports under `data/pr-check/reports/`. + +## Prerequisites + +- [Docker](https://docs.docker.com/) with BuildKit +- [GitHub CLI](https://cli.github.com/) (`gh auth login`) +- `git` +- A local `llvm-project` clone (used as a reference for faster PR fetches; PR content is fetched from GitHub) +- Python 3.10+ with fuzz-fill dependencies installed (`pip install -e .`) + +## One-time setup + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Edit config.env and set LLVM_REPO +``` + +`config.env` is local-only (not committed). Paths in the example are relative to the fuzz-fill repo root. + +## Usage + +Run from the fuzz-fill repo root: + +```bash +# List open PRs touching AMDGPU or SPIR-V target paths +./scripts/pr-check/check-llvm-prs.sh --discover-only + +# Show planned work (PR/backend pairs needing a fresh check) +./scripts/pr-check/check-llvm-prs.sh --plan-only + +# Regenerate latest.json / latest.md from state without running checks +./scripts/pr-check/check-llvm-prs.sh --report-only + +# Discover, plan, run up to PR_CHECK_MAX_PER_RUN checks, and write reports +./scripts/pr-check/check-llvm-prs.sh +``` + +Override config path: + +```bash +./scripts/pr-check/check-llvm-prs.sh --config /path/to/my-config.env +``` + +## What each run does + +1. **Discover** — `gh search prs` for open PRs touching `llvm/lib/Target/AMDGPU` and/or `llvm/lib/Target/SPIRV` +2. **Plan** — compare against `state.json`; queue entries that are new or have a changed head SHA +3. **Check** — for each planned `(PR, backend)` pair (up to `PR_CHECK_MAX_PER_RUN`): + - Build Docker image `fuzz-fill-test:llvm-pr--` via [`build-image-pr.sh`](../docker/build-image-pr.sh) + - Run [`pr-cov-gaps-detection.sh`](../docker/pr-cov-gaps-detection.sh) + - Record `gaps`, `clean`, or `failed` in state +4. **Report** — write `data/pr-check/reports/latest.json` and `latest.md` + +## Output layout + +| Path | Description | +|------|-------------| +| `data/pr-check/state.json` | Persistent check history keyed by `":"` | +| `data/pr-check/runs/-/` | Per-check artifacts (`baseline/`, `commit_lines_report/`, …) | +| `data/pr-check/reports/latest.json` | Machine-readable summary of all checked PRs | +| `data/pr-check/reports/latest.md` | Human-readable gap report with PR links | +| `data/pr-check/reports/runs/.json` | Snapshot from each report generation | + +A PR has **coverage gaps** when `target_lines_uncovered.csv` is non-empty (added lines not covered by the regression suite). + +## Cron example + +```cron +# Every 6 hours: run at most one PR/backend check, then refresh reports +0 */6 * * * cd /path/to/fuzz-fill && ./scripts/pr-check/check-llvm-prs.sh >> /var/log/fuzz-fill-pr-check.log 2>&1 +``` + +Keep `PR_CHECK_MAX_PER_RUN=1` unless you have enough CPU/time for multiple LLVM Docker builds per invocation. + +## systemd timer example + +`/etc/systemd/system/fuzz-fill-pr-check.service`: + +```ini +[Unit] +Description=fuzz-fill periodic LLVM PR coverage gap check + +[Service] +Type=oneshot +WorkingDirectory=/path/to/fuzz-fill +ExecStart=/path/to/fuzz-fill/scripts/pr-check/check-llvm-prs.sh +User=your-user +``` + +`/etc/systemd/system/fuzz-fill-pr-check.timer`: + +```ini +[Unit] +Description=Run fuzz-fill PR coverage checks every 6 hours + +[Timer] +OnCalendar=*-*-* 00,06,12,18:00:00 +Persistent=true + +[Install] +WantedBy=timers.target +``` + +Enable with: + +```bash +sudo systemctl enable --now fuzz-fill-pr-check.timer +``` + +## Low-level CLI + +The orchestrator wraps [`scripts/pr_check/checker.py`](../pr_check/checker.py): + +```bash +PYTHONPATH=scripts python3 -m pr_check discover +PYTHONPATH=scripts python3 -m pr_check plan --state-file data/pr-check/state.json +PYTHONPATH=scripts python3 -m pr_check report \ + --state-file data/pr-check/state.json \ + --report-dir data/pr-check/reports +``` diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env new file mode 100644 index 00000000..38d9bb85 --- /dev/null +++ b/scripts/pr-check/config.example.env @@ -0,0 +1,26 @@ +# Copy this file to scripts/pr-check/config.env and adjust paths. +# Run check-llvm-prs.sh from the fuzz-fill repo root (or use absolute paths below). + +# Local llvm-project clone used as a git reference for PR image builds. +# PR content is still fetched fresh from GitHub; the clone can be stale. +LLVM_REPO=/path/to/llvm-project + +# Parallel jobs for Docker LLVM builds and llvm-lit baseline runs. +PR_CHECK_JOBS=$(nproc) + +# Max PR/backend checks per cron invocation (each first-time build compiles LLVM). +PR_CHECK_MAX_PER_RUN=1 + +# Max open PRs returned per backend search (AMDGPU and SPIR-V). +PR_CHECK_SEARCH_LIMIT=100 + +# Host paths for state, per-run artifacts, and aggregated reports. +PR_CHECK_OUTPUT_ROOT=data/pr-check/runs +PR_CHECK_STATE_FILE=data/pr-check/state.json +PR_CHECK_REPORT_DIR=data/pr-check/reports + +# Optional: override the GitHub repo hosting PRs (default: llvm/llvm-project). +# GITHUB_REPO=llvm/llvm-project + +# Optional: personal access token for higher GitHub API rate limits. +# GITHUB_TOKEN= From dedfdc46db171f873ac46cbb006c5a1cd66eb990 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 14:42:11 -0500 Subject: [PATCH 05/26] Add unit tests and gitignore entry for periodic PR checking. Covers pr_check state/plan/report helpers with mocked discovery, and ignores local scripts/pr-check/config.env. --- .gitignore | 1 + tests/test_pr_checker.py | 270 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 tests/test_pr_checker.py diff --git a/.gitignore b/.gitignore index 42eab3b1..d28fa9f8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ data/* logs/* coverage_output/* sbom/* +scripts/pr-check/config.env requirements.txt **/__pycache__/* integration-tests/Output/* diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py new file mode 100644 index 00000000..95924441 --- /dev/null +++ b/tests/test_pr_checker.py @@ -0,0 +1,270 @@ +"""Unit tests for periodic LLVM PR coverage checking helpers.""" + +from __future__ import annotations + +import csv +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import pandas as pd + +SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from pr_check.checker import ( # noqa: E402 + STATE_VERSION, + DiscoveredPr, + PrCheckerError, + _aggregate_search_results, + build_report_payload, + count_csv_data_rows, + count_lit_failures, + entry_key, + evaluate_output_dir, + load_gap_rows, + load_state, + plan_work, + record_result, + render_report_markdown, + save_state, + write_reports, +) + + +def _write_gap_csv(path: Path, rows: list[list[object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + csv.writer(handle).writerows(rows) + + +class EntryKeyTest(unittest.TestCase): + def test_formats_pr_and_backend(self) -> None: + self.assertEqual(entry_key(203468, "amdgpu"), "203468:amdgpu") + + +class StateFileTest(unittest.TestCase): + def test_load_missing_state_returns_empty_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + state = load_state(Path(tmp) / "state.json") + self.assertEqual(state["version"], STATE_VERSION) + self.assertEqual(state["entries"], {}) + + def test_save_and_load_roundtrip(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "state.json" + state = {"version": STATE_VERSION, "entries": {}} + record_result( + state, + pr_number=42, + backend="spirv", + title="SPIR-V fix", + head_sha="abc123", + status="clean", + gap_count=0, + lit_failure_count=0, + output_dir="/tmp/out", + ) + save_state(path, state) + loaded = load_state(path) + + self.assertIn("42:spirv", loaded["entries"]) + self.assertEqual(loaded["entries"]["42:spirv"]["status"], "clean") + + def test_unsupported_version_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "state.json" + path.write_text(json.dumps({"version": 99, "entries": {}}), encoding="utf-8") + with self.assertRaises(PrCheckerError): + load_state(path) + + +class PlanWorkTest(unittest.TestCase): + def test_queues_new_and_head_changed_items_only(self) -> None: + discovered = [ + DiscoveredPr( + pr_number=1, + title="A", + head_sha="sha-new", + updated_at="2026-01-01", + backends=["amdgpu", "spirv"], + ), + DiscoveredPr( + pr_number=2, + title="B", + head_sha="same-sha", + updated_at="2026-01-01", + backends=["amdgpu"], + ), + ] + state = { + "version": STATE_VERSION, + "entries": { + "1:amdgpu": { + "pr_number": 1, + "backend": "amdgpu", + "title": "A", + "head_sha": "sha-old", + "status": "gaps", + "gap_count": 3, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": "/tmp/1-amdgpu", + "error": None, + }, + "2:amdgpu": { + "pr_number": 2, + "backend": "amdgpu", + "title": "B", + "head_sha": "same-sha", + "status": "clean", + "gap_count": 0, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": "/tmp/2-amdgpu", + "error": None, + }, + }, + } + + work = plan_work(discovered, state) + reasons = {(item.pr_number, item.backend): item.reason for item in work} + + self.assertEqual(reasons[(1, "amdgpu")], "head_changed") + self.assertEqual(reasons[(1, "spirv")], "new") + self.assertNotIn((2, "amdgpu"), reasons) + + +class AggregateSearchResultsTest(unittest.TestCase): + def test_merges_backends_for_same_pr(self) -> None: + frame = pd.DataFrame( + [ + {"number": 10, "title": "AMDGPU pass", "updatedAt": "2026-01-01", "backend": "amdgpu"}, + {"number": 10, "title": "", "updatedAt": "2026-01-03", "backend": "spirv"}, + {"number": 5, "title": "SPIR-V only", "updatedAt": "2026-01-02", "backend": "spirv"}, + ] + ) + aggregated = _aggregate_search_results(frame) + + self.assertEqual(aggregated["pr_number"].tolist(), [5, 10]) + row_10 = aggregated.loc[aggregated["pr_number"] == 10].iloc[0] + self.assertEqual(row_10["backends"], ["amdgpu", "spirv"]) + self.assertEqual(row_10["title"], "AMDGPU pass") + self.assertEqual(row_10["updated_at"], "2026-01-03") + + +class GapCsvHelpersTest(unittest.TestCase): + def test_count_and_load_gap_rows(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + gap_csv = Path(tmp) / "target_lines_uncovered.csv" + _write_gap_csv( + gap_csv, + [ + ["file", "line_no", "text"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "return x;"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "11", "return y;"], + ], + ) + + self.assertEqual(count_csv_data_rows(gap_csv), 2) + rows = load_gap_rows(gap_csv, max_rows=1) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["line_no"], "10") + + def test_count_lit_failures(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + lit_path = Path(tmp) / "lit_failures.json" + lit_path.write_text( + json.dumps({"tests": [{"code": "FAIL"}, {"code": "PASS"}]}), + encoding="utf-8", + ) + self.assertEqual(count_lit_failures(lit_path), 1) + + +class EvaluateOutputDirTest(unittest.TestCase): + def test_reports_gaps_status(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_gap_csv( + root / "commit_lines_report" / "target_lines_uncovered.csv", + [["file", "line_no", "text"], ["llvm/Foo.cpp", "1", "x = 1;"]], + ) + result = evaluate_output_dir(root) + + self.assertEqual(result["gap_count"], 1) + self.assertEqual(result["status"], "gaps") + + +class ReportGenerationTest(unittest.TestCase): + def test_build_render_and_write_reports(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + run_dir = root / "run-123-amdgpu" + _write_gap_csv( + run_dir / "commit_lines_report" / "target_lines_uncovered.csv", + [ + ["file", "line_no", "text"], + ["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "return x;"], + ], + ) + state = { + "version": STATE_VERSION, + "entries": { + "123:amdgpu": { + "pr_number": 123, + "backend": "amdgpu", + "title": "Fix foo", + "head_sha": "abc123def456", + "status": "gaps", + "gap_count": 1, + "lit_failure_count": 2, + "checked_at": "2026-07-29T18:00:00+00:00", + "output_dir": str(run_dir), + "error": None, + } + }, + } + + payload = build_report_payload(state) + self.assertEqual(payload["summary"]["with_gaps"], 1) + self.assertEqual(payload["entries_with_gaps"][0]["sample_gaps"][0]["line_no"], "10") + + markdown = render_report_markdown(payload) + self.assertIn("#123", markdown) + self.assertIn("LIT failure", markdown) + + report_dir = root / "reports" + written = write_reports(report_dir, payload, write_run_snapshot=True) + self.assertTrue(written["latest_json"].is_file()) + self.assertTrue(written["latest_md"].is_file()) + self.assertTrue(written["run_snapshot"].is_file()) + + +class DiscoverPrsTest(unittest.TestCase): + def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: + from pr_check.checker import discover_prs + + search_payload = [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + with mock.patch("pr_check.checker._search_backend_prs", return_value=search_payload): + with mock.patch( + "pr_check.checker._view_pr", + return_value={ + "headRefOid": "deadbeef", + "title": "Test PR", + "updatedAt": "2026-01-02", + }, + ): + discovered = discover_prs(limit=1) + + self.assertEqual(len(discovered), 1) + self.assertEqual(discovered[0].pr_number, 99) + self.assertEqual(discovered[0].head_sha, "deadbeef") + self.assertEqual(discovered[0].backends, ["amdgpu", "spirv"]) + + +if __name__ == "__main__": + unittest.main() From 2da3625d57bdeacedeee2fd0b6425ad1cd1560b8 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 29 Jul 2026 14:45:50 -0500 Subject: [PATCH 06/26] Fix --github-repo placement on pr_check subcommands. Move the flag onto discover, plan, and report so check-llvm-prs.sh can pass it after the subcommand name without argparse errors. --- scripts/pr_check/checker.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index 54173fde..fb8edf2b 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -711,14 +711,20 @@ def cmd_report(args: argparse.Namespace) -> int: def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( + subparsers = parser.add_subparsers(dest="command", required=True) + + github_repo_parent = argparse.ArgumentParser(add_help=False) + github_repo_parent.add_argument( "--github-repo", default=DEFAULT_GITHUB_REPO, help=f"GitHub repo hosting PRs (default: {DEFAULT_GITHUB_REPO})", ) - subparsers = parser.add_subparsers(dest="command", required=True) - discover_parser = subparsers.add_parser("discover", help="List open target PRs") + discover_parser = subparsers.add_parser( + "discover", + help="List open target PRs", + parents=[github_repo_parent], + ) discover_parser.add_argument( "--limit", type=int, @@ -727,7 +733,11 @@ def build_parser() -> argparse.ArgumentParser: ) discover_parser.set_defaults(func=cmd_discover) - plan_parser = subparsers.add_parser("plan", help="List PR/backend pairs needing a run") + plan_parser = subparsers.add_parser( + "plan", + help="List PR/backend pairs needing a run", + parents=[github_repo_parent], + ) plan_parser.add_argument( "--state-file", type=Path, @@ -772,7 +782,11 @@ def build_parser() -> argparse.ArgumentParser: evaluate_parser.add_argument("--output-dir", type=Path, required=True) evaluate_parser.set_defaults(func=cmd_evaluate_output) - report_parser = subparsers.add_parser("report", help="Write latest.json and latest.md from state") + report_parser = subparsers.add_parser( + "report", + help="Write latest.json and latest.md from state", + parents=[github_repo_parent], + ) report_parser.add_argument("--state-file", type=Path, required=True) report_parser.add_argument( "--report-dir", From e2e2840929da311f5c8fe93902b781679465f03d Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 04:50:05 -0500 Subject: [PATCH 07/26] Fix gh search path --- scripts/pr_check/checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index fb8edf2b..2c628095 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -22,8 +22,8 @@ DEFAULT_SEARCH_LIMIT = 100 BACKEND_SEARCH_QUERIES: dict[str, str] = { - "amdgpu": "path:llvm/lib/Target/AMDGPU", - "spirv": "path:llvm/lib/Target/SPIRV", + "amdgpu": 'path:"llvm/lib/Target/AMDGPU"', + "spirv": 'path:"llvm/lib/Target/SPIRV"', } SEARCH_JSON_FIELDS = ["number", "title", "updatedAt"] From de8e17c1c1d7f91849cd00dc24c0f4ab88f00e0d Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 05:07:59 -0500 Subject: [PATCH 08/26] Add logging to pr check flow --- scripts/pr-check/check-llvm-prs.sh | 22 +++- scripts/pr-check/config.example.env | 3 + scripts/pr_check/checker.py | 174 ++++++++++++++++++++++++++-- 3 files changed, 190 insertions(+), 9 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index e7c38fc2..2dd072e5 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -16,6 +16,7 @@ PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +PR_CHECK_LOG_LEVEL="${PR_CHECK_LOG_LEVEL:-info}" GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" IMAGE_NAME="${IMAGE_NAME:-fuzz-fill-test}" @@ -44,6 +45,9 @@ Options: --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) + --log-level Python log level: debug, info, warning, error + (default: ${PR_CHECK_LOG_LEVEL}) + -v, --verbose Shorthand for --log-level debug --github-repo GitHub repo hosting PRs (default: ${GITHUB_REPO}) --help, -h Show this help @@ -67,7 +71,9 @@ load_config() { } run_pr_check() { - PYTHONPATH="${REPO_ROOT}/scripts" python3 -m pr_check "$@" + PYTHONPATH="${REPO_ROOT}/scripts" python3 -m pr_check \ + "$@" \ + --log-level "$PR_CHECK_LOG_LEVEL" } generate_report() { @@ -246,6 +252,15 @@ while [[ $# -gt 0 ]]; do PR_CHECK_JOBS="$2" shift 2 ;; + --log-level) + [[ $# -ge 2 ]] || { echo "error: --log-level requires a value" >&2; exit 2; } + PR_CHECK_LOG_LEVEL="$2" + shift 2 + ;; + -v|--verbose) + PR_CHECK_LOG_LEVEL="debug" + shift + ;; --github-repo) [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; } GITHUB_REPO="$2" @@ -296,11 +311,13 @@ if [[ "$report_only" -eq 1 ]]; then fi if [[ "$discover_only" -eq 1 ]]; then + echo "=== discover open PRs (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 run_pr_check discover "${common_args[@]}" --limit "$PR_CHECK_SEARCH_LIMIT" exit 0 fi if [[ "$plan_only" -eq 1 ]]; then + echo "=== plan PR checks (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 run_pr_check plan \ "${common_args[@]}" \ --state-file "$PR_CHECK_STATE_FILE" \ @@ -311,6 +328,9 @@ fi validate_runtime_config +echo "=== automated PR check run (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 +echo "Discovering and planning work (max ${PR_CHECK_MAX_PER_RUN} check(s) this run)..." >&2 + work_json="$(run_pr_check plan \ "${common_args[@]}" \ --state-file "$PR_CHECK_STATE_FILE" \ diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env index 38d9bb85..a897603f 100644 --- a/scripts/pr-check/config.example.env +++ b/scripts/pr-check/config.example.env @@ -19,6 +19,9 @@ PR_CHECK_OUTPUT_ROOT=data/pr-check/runs PR_CHECK_STATE_FILE=data/pr-check/state.json PR_CHECK_REPORT_DIR=data/pr-check/reports +# Optional: log level for pr_check progress messages on stderr (debug, info, warning, error). +PR_CHECK_LOG_LEVEL=info + # Optional: override the GitHub repo hosting PRs (default: llvm/llvm-project). # GITHUB_REPO=llvm/llvm-project diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index 2c628095..fd491f9a 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -8,6 +8,7 @@ import argparse import json +import logging import subprocess import sys from dataclasses import asdict, dataclass, field @@ -30,6 +31,9 @@ PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt"] LIT_FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"}) +LOG_FORMAT = "%(levelname)-8s %(message)s" +log = logging.getLogger("pr_check") + class PrCheckerError(Exception): """Error from PR checker operations.""" @@ -81,6 +85,22 @@ def utc_now_iso() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat() +def configure_logging(level: str = "info") -> None: + """Send log records to stderr so stdout stays free for JSON payloads.""" + numeric_level = getattr(logging, level.upper(), None) + if not isinstance(numeric_level, int): + raise PrCheckerError(f"invalid log level: {level}") + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + + root = logging.getLogger("pr_check") + root.handlers.clear() + root.setLevel(numeric_level) + root.addHandler(handler) + root.propagate = False + + def _require_gh() -> str: from shutil import which @@ -92,6 +112,7 @@ def _require_gh() -> str: def _run_gh(args: list[str], *, timeout: int = 120) -> str: gh_path = _require_gh() + log.debug("running gh %s", " ".join(args)) try: result = subprocess.run( [gh_path, *args], @@ -138,6 +159,13 @@ def _search_backend_prs( limit: int = DEFAULT_SEARCH_LIMIT, ) -> list[dict[str, Any]]: query = BACKEND_SEARCH_QUERIES[backend] + log.info( + "Searching %s PRs on %s (query=%r, limit=%d)", + backend, + github_repo, + query, + limit, + ) payload = _gh_json( [ "search", @@ -153,6 +181,7 @@ def _search_backend_prs( ) if not isinstance(payload, list): raise PrCheckerError(f"unexpected gh search payload for {backend}: {type(payload)!r}") + log.info("Found %d open %s PR(s)", len(payload), backend) return payload @@ -210,15 +239,32 @@ def _aggregate_search_results(search_df: pd.DataFrame) -> pd.DataFrame: def discover_prs(*, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT) -> list[DiscoveredPr]: """Find open PRs touching AMDGPU and/or SPIR-V target paths.""" - grouped = _aggregate_search_results( - _search_results_dataframe(github_repo=github_repo, limit=limit) + log.info( + "Discovering open PRs on %s (limit=%d per backend: %s)", + github_repo, + limit, + ", ".join(sorted(BACKEND_SEARCH_QUERIES)), ) + search_df = _search_results_dataframe(github_repo=github_repo, limit=limit) + grouped = _aggregate_search_results(search_df) if grouped.empty: + log.info("No matching open PRs found") return [] + total = len(grouped) + log.info( + "Search returned %d unique PR(s); resolving head SHAs via gh pr view", + total, + ) + discovered: list[DiscoveredPr] = [] - for row in grouped.itertuples(index=False): + for index, row in enumerate(grouped.itertuples(index=False), start=1): pr_number = int(row.pr_number) + if index == 1 or index == total or index % 10 == 0: + log.info("Resolving head SHA %d/%d: #%d", index, total, pr_number) + else: + log.debug("Resolving head SHA %d/%d: #%d", index, total, pr_number) + pr_view = _view_pr(pr_number, github_repo) head_sha = pr_view.get("headRefOid") if not head_sha: @@ -234,11 +280,13 @@ def discover_prs(*, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT ) ) + log.info("Discovery complete: %d PR(s) with head SHAs", len(discovered)) return discovered def load_state(path: Path) -> dict[str, Any]: if not path.exists(): + log.info("State file not found, starting fresh: %s", path) return {"version": STATE_VERSION, "entries": {}} try: @@ -255,12 +303,14 @@ def load_state(path: Path) -> dict[str, Any]: if not isinstance(payload.get("entries"), dict): raise PrCheckerError(f"invalid state file {path}: missing entries object") + log.info("Loaded state from %s (%d entries)", path, len(payload["entries"])) return payload def save_state(path: Path, state: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + log.info("Saved state to %s (%d entries)", path, len(state.get("entries", {}))) def _state_entry_from_dict(raw: dict[str, Any]) -> StateEntry: @@ -282,12 +332,24 @@ def plan_work(discovered: list[DiscoveredPr], state: dict[str, Any]) -> list[Wor """Return PR/backend pairs that need a fresh coverage run.""" entries: dict[str, Any] = state["entries"] work: list[WorkItem] = [] + pair_count = 0 + skipped_up_to_date = 0 + new_count = 0 + head_changed_count = 0 + + log.info( + "Planning work from %d discovered PR(s) against %d state entries", + len(discovered), + len(entries), + ) for pr in discovered: for backend in pr.backends: + pair_count += 1 key = entry_key(pr.pr_number, backend) existing = entries.get(key) if existing is None: + new_count += 1 work.append( WorkItem( pr_number=pr.pr_number, @@ -297,10 +359,12 @@ def plan_work(discovered: list[DiscoveredPr], state: dict[str, Any]) -> list[Wor reason="new", ) ) + log.debug("Queued #%d (%s): new", pr.pr_number, backend) continue previous = _state_entry_from_dict(existing) if previous.head_sha != pr.head_sha: + head_changed_count += 1 work.append( WorkItem( pr_number=pr.pr_number, @@ -310,8 +374,39 @@ def plan_work(discovered: list[DiscoveredPr], state: dict[str, Any]) -> list[Wor reason="head_changed", ) ) + log.debug( + "Queued #%d (%s): head changed %s -> %s", + pr.pr_number, + backend, + previous.head_sha[:12], + pr.head_sha[:12], + ) + else: + skipped_up_to_date += 1 + log.debug( + "Skipping #%d (%s): already checked at head %s", + pr.pr_number, + backend, + pr.head_sha[:12], + ) work.sort(key=lambda item: (item.pr_number, item.backend)) + log.info( + "Plan summary: %d PR/backend pair(s) scanned, %d queued (%d new, %d head_changed), %d up-to-date", + pair_count, + len(work), + new_count, + head_changed_count, + skipped_up_to_date, + ) + for item in work: + log.info( + " -> #%d (%s) [%s] head=%s", + item.pr_number, + item.backend, + item.reason, + item.head_sha[:12], + ) return work @@ -329,6 +424,14 @@ def record_result( error: str | None = None, ) -> None: key = entry_key(pr_number, backend) + log.info( + "Recording result for #%d (%s): status=%s gap_count=%d lit_failures=%d", + pr_number, + backend, + status, + gap_count, + lit_failure_count, + ) state["entries"][key] = { "pr_number": pr_number, "backend": backend, @@ -644,12 +747,17 @@ def write_reports( snapshot_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") written["run_snapshot"] = snapshot_path + log.info("Wrote report files under %s", report_dir) + for name, path in written.items(): + log.info(" %s: %s", name, path) + return written def cmd_discover(args: argparse.Namespace) -> int: discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) payload = _discovered_to_json(discovered) + log.info("Writing %d discovered PR(s) to stdout as JSON", len(payload)) print(json.dumps(payload, indent=2)) return 0 @@ -659,7 +767,14 @@ def cmd_plan(args: argparse.Namespace) -> int: state = load_state(args.state_file) work = plan_work(discovered, state) if args.max_items is not None: + if len(work) > args.max_items: + log.info( + "Capping planned work from %d item(s) to %d (--max-items)", + len(work), + args.max_items, + ) work = work[: args.max_items] + log.info("Writing %d planned work item(s) to stdout as JSON", len(work)) print(json.dumps(_work_to_json(work), indent=2)) return 0 @@ -683,18 +798,34 @@ def cmd_record(args: argparse.Namespace) -> int: def cmd_evaluate_output(args: argparse.Namespace) -> int: + log.info("Evaluating output directory: %s", args.output_dir) payload = evaluate_output_dir(args.output_dir) + log.info( + "Evaluation: status=%s gap_count=%d lit_failures=%d", + payload["status"], + payload["gap_count"], + payload["lit_failure_count"], + ) print(json.dumps(payload, indent=2)) return 0 def cmd_report(args: argparse.Namespace) -> int: + log.info("Generating coverage gap report") state = load_state(args.state_file) payload = build_report_payload( state, github_repo=args.github_repo, max_gap_lines=args.max_gap_lines, ) + summary = payload["summary"] + log.info( + "Report summary: %d total, %d with gaps, %d clean, %d failed", + summary["total_entries"], + summary["with_gaps"], + summary["clean"], + summary["failed"], + ) written = write_reports( args.report_dir, payload, @@ -713,6 +844,22 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) + logging_parent = argparse.ArgumentParser(add_help=False) + logging_parent.add_argument( + "--log-level", + default="info", + choices=["debug", "info", "warning", "error"], + help="Log level for progress messages on stderr (default: info)", + ) + logging_parent.add_argument( + "-v", + "--verbose", + action="store_const", + const="debug", + dest="log_level", + help="Shorthand for --log-level debug", + ) + github_repo_parent = argparse.ArgumentParser(add_help=False) github_repo_parent.add_argument( "--github-repo", @@ -723,7 +870,7 @@ def build_parser() -> argparse.ArgumentParser: discover_parser = subparsers.add_parser( "discover", help="List open target PRs", - parents=[github_repo_parent], + parents=[logging_parent, github_repo_parent], ) discover_parser.add_argument( "--limit", @@ -736,7 +883,7 @@ def build_parser() -> argparse.ArgumentParser: plan_parser = subparsers.add_parser( "plan", help="List PR/backend pairs needing a run", - parents=[github_repo_parent], + parents=[logging_parent, github_repo_parent], ) plan_parser.add_argument( "--state-file", @@ -758,7 +905,11 @@ def build_parser() -> argparse.ArgumentParser: ) plan_parser.set_defaults(func=cmd_plan) - record_parser = subparsers.add_parser("record", help="Persist one check result") + record_parser = subparsers.add_parser( + "record", + help="Persist one check result", + parents=[logging_parent], + ) record_parser.add_argument("--state-file", type=Path, required=True) record_parser.add_argument("--pr-number", type=int, required=True) record_parser.add_argument("--backend", choices=sorted(BACKEND_SEARCH_QUERIES), required=True) @@ -778,6 +929,7 @@ def build_parser() -> argparse.ArgumentParser: evaluate_parser = subparsers.add_parser( "evaluate-output", help="Summarize gap and LIT failure counts from a run output directory", + parents=[logging_parent], ) evaluate_parser.add_argument("--output-dir", type=Path, required=True) evaluate_parser.set_defaults(func=cmd_evaluate_output) @@ -785,7 +937,7 @@ def build_parser() -> argparse.ArgumentParser: report_parser = subparsers.add_parser( "report", help="Write latest.json and latest.md from state", - parents=[github_repo_parent], + parents=[logging_parent, github_repo_parent], ) report_parser.add_argument("--state-file", type=Path, required=True) report_parser.add_argument( @@ -814,11 +966,17 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) try: - return args.func(args) + configure_logging(args.log_level) except PrCheckerError as exc: print(f"error: {exc}", file=sys.stderr) return 1 + try: + return args.func(args) + except PrCheckerError as exc: + log.error("%s", exc) + return 1 + if __name__ == "__main__": raise SystemExit(main()) From 7058aa241237ab9192cdda7d49f8c9a8cd80ea68 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 05:21:21 -0500 Subject: [PATCH 09/26] Move SPIR-V CMake flag This is required by current upstream LLVM CMake --- scripts/build-llvm-sancov.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build-llvm-sancov.sh b/scripts/build-llvm-sancov.sh index 15bf88c6..99fc347d 100755 --- a/scripts/build-llvm-sancov.sh +++ b/scripts/build-llvm-sancov.sh @@ -158,7 +158,8 @@ LLVM_CMAKE_BASE=( -G Ninja -DCMAKE_C_COMPILER="$C_COMPILER" -DCMAKE_CXX_COMPILER="$CXX_COMPILER" - -DLLVM_TARGETS_TO_BUILD="X86;AMDGPU;SPIRV" + -DLLVM_TARGETS_TO_BUILD="X86;AMDGPU" + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV" -DLLVM_ENABLE_PROJECTS="" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_SPLIT_DWARF=ON From 0de579682332c077a7813bb5ba890c2892439d13 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 05:29:37 -0500 Subject: [PATCH 10/26] Fix image retention bug --- scripts/docker/pr-cov-gaps-detection.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/docker/pr-cov-gaps-detection.sh b/scripts/docker/pr-cov-gaps-detection.sh index 35dc9c2b..edbcf5f9 100755 --- a/scripts/docker/pr-cov-gaps-detection.sh +++ b/scripts/docker/pr-cov-gaps-detection.sh @@ -38,8 +38,8 @@ Required: Options: --build-image Build PR image via build-image-pr.sh before detection - --keep-image Keep the PR image after detection (default: remove it - when --build-image was used) + --keep-image Keep the PR image after detection when used with + --build-image (no effect when the image is pre-built) --llvm-repo Local llvm-project clone (required with --build-image) --backend-tests amdgpu or spirv (required with --build-image) --github-repo @@ -215,8 +215,8 @@ if [[ -z "$output_dir" ]]; then fi if [[ "$build_image" -eq 0 ]]; then - if [[ -n "$llvm_repo" || -n "$backend_tests" || -n "$github_repo" || "$keep_image" -eq 1 ]]; then - echo "error: --llvm-repo, --backend-tests, --github-repo, and --keep-image require --build-image" >&2 + if [[ -n "$llvm_repo" || -n "$backend_tests" || -n "$github_repo" ]]; then + echo "error: --llvm-repo, --backend-tests, and --github-repo require --build-image" >&2 exit 1 fi else From cfe116f90e22ec75b94cec0def0070618cab313b Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 07:37:27 -0500 Subject: [PATCH 11/26] Limit PR discovery to recently opened PRs and require sancov -union Filter discovery with GitHub created:> instead of updated:>, so cron runs target fresh PRs rather than stale branches that were only touched recently. Use gh api search/issues for the date qualifier and expose --max-age-days / PR_CHECK_MAX_AGE_DAYS (default 30). --- scripts/pr-check/check-llvm-prs.sh | 9 ++ scripts/pr-check/config.example.env | 3 + scripts/pr_check/checker.py | 153 +++++++++++++++++++++------- src/coverage/sancov.py | 29 ++++-- tests/test_pr_checker.py | 7 ++ 5 files changed, 152 insertions(+), 49 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index 2dd072e5..6db41526 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -13,6 +13,7 @@ LLVM_REPO="${LLVM_REPO:-}" PR_CHECK_JOBS="${PR_CHECK_JOBS:-$(nproc)}" PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" +PR_CHECK_MAX_AGE_DAYS="${PR_CHECK_MAX_AGE_DAYS:-30}" PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" @@ -44,6 +45,7 @@ Options: --output-root Per-run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) + --max-age-days Only PRs opened in the last N days (default: ${PR_CHECK_MAX_AGE_DAYS}) --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) --log-level Python log level: debug, info, warning, error (default: ${PR_CHECK_LOG_LEVEL}) @@ -112,6 +114,7 @@ validate_runtime_config() { validate_positive_int "PR_CHECK_JOBS" "$PR_CHECK_JOBS" validate_positive_int "PR_CHECK_MAX_PER_RUN" "$PR_CHECK_MAX_PER_RUN" validate_positive_int "PR_CHECK_SEARCH_LIMIT" "$PR_CHECK_SEARCH_LIMIT" + validate_positive_int "PR_CHECK_MAX_AGE_DAYS" "$PR_CHECK_MAX_AGE_DAYS" } record_failed_run() { @@ -247,6 +250,11 @@ while [[ $# -gt 0 ]]; do PR_CHECK_MAX_PER_RUN="$2" shift 2 ;; + --max-age-days) + [[ $# -ge 2 ]] || { echo "error: --max-age-days requires a value" >&2; exit 2; } + PR_CHECK_MAX_AGE_DAYS="$2" + shift 2 + ;; --jobs) [[ $# -ge 2 ]] || { echo "error: --jobs requires a value" >&2; exit 2; } PR_CHECK_JOBS="$2" @@ -303,6 +311,7 @@ mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" "$PR_CHECK_OUTPUT_ROOT" "$PR_CHECK_ common_args=( --github-repo "$GITHUB_REPO" + --max-age-days "$PR_CHECK_MAX_AGE_DAYS" ) if [[ "$report_only" -eq 1 ]]; then diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env index a897603f..a6870e43 100644 --- a/scripts/pr-check/config.example.env +++ b/scripts/pr-check/config.example.env @@ -14,6 +14,9 @@ PR_CHECK_MAX_PER_RUN=1 # Max open PRs returned per backend search (AMDGPU and SPIR-V). PR_CHECK_SEARCH_LIMIT=100 +# Only include PRs opened within this many days (GitHub created:> filter). +PR_CHECK_MAX_AGE_DAYS=30 + # Host paths for state, per-run artifacts, and aggregated reports. PR_CHECK_OUTPUT_ROOT=data/pr-check/runs PR_CHECK_STATE_FILE=data/pr-check/state.json diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index fd491f9a..6a6f34aa 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -12,15 +12,17 @@ import subprocess import sys from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any +from urllib.parse import urlencode import pandas as pd STATE_VERSION = 1 DEFAULT_GITHUB_REPO = "llvm/llvm-project" DEFAULT_SEARCH_LIMIT = 100 +DEFAULT_MAX_PR_AGE_DAYS = 30 BACKEND_SEARCH_QUERIES: dict[str, str] = { "amdgpu": 'path:"llvm/lib/Target/AMDGPU"', @@ -101,6 +103,24 @@ def configure_logging(level: str = "info") -> None: root.propagate = False +def _validate_max_age_days(max_age_days: int) -> None: + if max_age_days <= 0: + raise PrCheckerError(f"max_age_days must be a positive integer: {max_age_days}") + + +def _search_created_since(max_age_days: int) -> str: + """Return a YYYY-MM-DD date for GitHub ``created:>`` search qualifiers.""" + _validate_max_age_days(max_age_days) + cutoff = datetime.now(timezone.utc).date() - timedelta(days=max_age_days) + return cutoff.isoformat() + + +def _backend_search_query(backend: str, *, max_age_days: int) -> str: + """Build a gh search query for one backend, limited to recently opened PRs.""" + created_since = _search_created_since(max_age_days) + return f'{BACKEND_SEARCH_QUERIES[backend]} created:>{created_since}' + + def _require_gh() -> str: from shutil import which @@ -144,6 +164,32 @@ def _gh_json(args: list[str], *, timeout: int = 120) -> Any: raise PrCheckerError(f"gh returned invalid JSON: {exc.msg}") from exc +def _gh_api_json(endpoint: str) -> Any: + stdout = _run_gh(["api", endpoint]) + if not stdout.strip(): + return {} + try: + return json.loads(stdout) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"gh api returned invalid JSON: {exc.msg}") from exc + + +def _normalize_search_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Map GitHub issue-search items to the fields used elsewhere in this module.""" + normalized: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + normalized.append( + { + "number": item.get("number"), + "title": item.get("title") or "", + "updatedAt": item.get("updated_at") or "", + } + ) + return normalized + + def _json_fields_for_command(args: list[str]) -> list[str]: if args and args[0] == "search": return SEARCH_JSON_FIELDS @@ -157,32 +203,32 @@ def _search_backend_prs( *, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, ) -> list[dict[str, Any]]: - query = BACKEND_SEARCH_QUERIES[backend] + query = _backend_search_query(backend, max_age_days=max_age_days) + search_query = f"repo:{github_repo} is:pr is:open {query}" log.info( - "Searching %s PRs on %s (query=%r, limit=%d)", + "Searching %s PRs on %s (query=%r, limit=%d, max_age_days=%d)", backend, github_repo, - query, + search_query, limit, + max_age_days, ) - payload = _gh_json( - [ - "search", - "prs", - query, - "--repo", - github_repo, - "--state", - "open", - "--limit", - str(limit), - ] + payload = _gh_api_json( + "search/issues?" + + urlencode( + { + "q": search_query, + "per_page": str(min(limit, 100)), + } + ) ) - if not isinstance(payload, list): - raise PrCheckerError(f"unexpected gh search payload for {backend}: {type(payload)!r}") - log.info("Found %d open %s PR(s)", len(payload), backend) - return payload + if not isinstance(payload, dict): + raise PrCheckerError(f"unexpected gh api search payload for {backend}: {type(payload)!r}") + items = _normalize_search_items(payload.get("items", [])) + log.info("Found %d open %s PR(s)", len(items), backend) + return items[:limit] def _view_pr(pr_number: int, github_repo: str) -> dict[str, Any]: @@ -196,11 +242,17 @@ def _search_results_dataframe( *, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, ) -> pd.DataFrame: """Run backend searches and return one row per (PR, backend) match.""" frames: list[pd.DataFrame] = [] for backend in BACKEND_SEARCH_QUERIES: - items = _search_backend_prs(backend, github_repo=github_repo, limit=limit) + items = _search_backend_prs( + backend, + github_repo=github_repo, + limit=limit, + max_age_days=max_age_days, + ) if not items: continue frame = pd.DataFrame(items) @@ -237,15 +289,25 @@ def _aggregate_search_results(search_df: pd.DataFrame) -> pd.DataFrame: ) -def discover_prs(*, github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT) -> list[DiscoveredPr]: +def discover_prs( + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, +) -> list[DiscoveredPr]: """Find open PRs touching AMDGPU and/or SPIR-V target paths.""" log.info( - "Discovering open PRs on %s (limit=%d per backend: %s)", + "Discovering open PRs on %s (limit=%d per backend, max_age_days=%d, backends=%s)", github_repo, limit, + max_age_days, ", ".join(sorted(BACKEND_SEARCH_QUERIES)), ) - search_df = _search_results_dataframe(github_repo=github_repo, limit=limit) + search_df = _search_results_dataframe( + github_repo=github_repo, + limit=limit, + max_age_days=max_age_days, + ) grouped = _aggregate_search_results(search_df) if grouped.empty: log.info("No matching open PRs found") @@ -755,7 +817,11 @@ def write_reports( def cmd_discover(args: argparse.Namespace) -> int: - discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) + discovered = discover_prs( + github_repo=args.github_repo, + limit=args.limit, + max_age_days=args.max_age_days, + ) payload = _discovered_to_json(discovered) log.info("Writing %d discovered PR(s) to stdout as JSON", len(payload)) print(json.dumps(payload, indent=2)) @@ -763,7 +829,11 @@ def cmd_discover(args: argparse.Namespace) -> int: def cmd_plan(args: argparse.Namespace) -> int: - discovered = discover_prs(github_repo=args.github_repo, limit=args.limit) + discovered = discover_prs( + github_repo=args.github_repo, + limit=args.limit, + max_age_days=args.max_age_days, + ) state = load_state(args.state_file) work = plan_work(discovered, state) if args.max_items is not None: @@ -867,23 +937,34 @@ def build_parser() -> argparse.ArgumentParser: help=f"GitHub repo hosting PRs (default: {DEFAULT_GITHUB_REPO})", ) - discover_parser = subparsers.add_parser( - "discover", - help="List open target PRs", - parents=[logging_parent, github_repo_parent], - ) - discover_parser.add_argument( + search_parent = argparse.ArgumentParser(add_help=False) + search_parent.add_argument( "--limit", type=int, default=DEFAULT_SEARCH_LIMIT, help=f"Max PRs per backend search (default: {DEFAULT_SEARCH_LIMIT})", ) + search_parent.add_argument( + "--max-age-days", + type=int, + default=DEFAULT_MAX_PR_AGE_DAYS, + help=( + "Only include PRs opened within this many days " + f"(default: {DEFAULT_MAX_PR_AGE_DAYS})" + ), + ) + + discover_parser = subparsers.add_parser( + "discover", + help="List open target PRs", + parents=[logging_parent, github_repo_parent, search_parent], + ) discover_parser.set_defaults(func=cmd_discover) plan_parser = subparsers.add_parser( "plan", help="List PR/backend pairs needing a run", - parents=[logging_parent, github_repo_parent], + parents=[logging_parent, github_repo_parent, search_parent], ) plan_parser.add_argument( "--state-file", @@ -891,12 +972,6 @@ def build_parser() -> argparse.ArgumentParser: required=True, help="Path to state.json", ) - plan_parser.add_argument( - "--limit", - type=int, - default=DEFAULT_SEARCH_LIMIT, - help=f"Max PRs per backend search (default: {DEFAULT_SEARCH_LIMIT})", - ) plan_parser.add_argument( "--max-items", type=int, diff --git a/src/coverage/sancov.py b/src/coverage/sancov.py index 37fbe7d5..26118419 100644 --- a/src/coverage/sancov.py +++ b/src/coverage/sancov.py @@ -15,6 +15,7 @@ logger = get_logger("coverage.sancov") + class Sancov: @staticmethod @@ -352,16 +353,16 @@ def merge(self) -> None: with log_timing(logger, f"sancov merge ({self.suffix})"): merged_out = self.get_merged_sancov_path() - raw_files = list(self.raw_sancov_dir.glob(f"{self.suffix}.*.sancov")) + raw_files = sorted(self.raw_sancov_dir.glob(f"{self.suffix}.*.sancov")) if not raw_files: raise FileNotFoundError(f"No sancov files found for suffix: {self.suffix}") - """Merge raw ``.sancov`` files with repeated ``sancov -union`` (batched).""" if len(raw_files) == 1: shutil.copy(raw_files[0], merged_out) return - sancov = self.sancov_bin + logger.info("Merging %d raw sancov file(s) via sancov -union", len(raw_files)) + batch = self.union_batch layer = list(raw_files) with tempfile.TemporaryDirectory(dir=merged_out.parent) as tmp: @@ -375,17 +376,25 @@ def merge(self) -> None: nxt.append(chunk[0]) continue out_f = tmp_path / f"u_{round_idx}_{len(nxt)}.sancov" - run_subprocess( - logger, - [str(sancov), "-union"] - + [str(p) for p in chunk] - + ["--output", str(out_f)], - check=True, - ) + self._union_raw_sancov_chunk(chunk, out_f) nxt.append(out_f) layer = nxt round_idx += 1 shutil.copy(layer[0], merged_out) + + def _union_raw_sancov_chunk(self, inputs: list[Path], output: Path) -> None: + result = run_subprocess( + logger, + [str(self.sancov_bin), "-union", *[str(path) for path in inputs], "--output", str(output)], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + stderr = (result.stderr or result.stdout or "").strip() + raise RuntimeError( + f"sancov -union failed for {len(inputs)} file(s): {stderr or result.returncode}" + ) def symbolize( self, diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index 95924441..a0aa96eb 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -245,6 +245,13 @@ def test_build_render_and_write_reports(self) -> None: class DiscoverPrsTest(unittest.TestCase): + def test_backend_search_query_includes_opened_within_filter(self) -> None: + from pr_check.checker import _backend_search_query + + query = _backend_search_query("amdgpu", max_age_days=30) + self.assertIn('path:"llvm/lib/Target/AMDGPU"', query) + self.assertRegex(query, r'created:>\d{4}-\d{2}-\d{2}') + def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: from pr_check.checker import discover_prs From fa1038542159c59a83c38cef242bfba03d9ada67 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 07:47:05 -0500 Subject: [PATCH 12/26] Change max search range for PRs to the last 14 days --- scripts/pr-check/check-llvm-prs.sh | 2 +- scripts/pr-check/config.example.env | 2 +- scripts/pr_check/checker.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index 6db41526..8f361757 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -13,7 +13,7 @@ LLVM_REPO="${LLVM_REPO:-}" PR_CHECK_JOBS="${PR_CHECK_JOBS:-$(nproc)}" PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" -PR_CHECK_MAX_AGE_DAYS="${PR_CHECK_MAX_AGE_DAYS:-30}" +PR_CHECK_MAX_AGE_DAYS="${PR_CHECK_MAX_AGE_DAYS:-14}" PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env index a6870e43..5fac7f65 100644 --- a/scripts/pr-check/config.example.env +++ b/scripts/pr-check/config.example.env @@ -15,7 +15,7 @@ PR_CHECK_MAX_PER_RUN=1 PR_CHECK_SEARCH_LIMIT=100 # Only include PRs opened within this many days (GitHub created:> filter). -PR_CHECK_MAX_AGE_DAYS=30 +PR_CHECK_MAX_AGE_DAYS=14 # Host paths for state, per-run artifacts, and aggregated reports. PR_CHECK_OUTPUT_ROOT=data/pr-check/runs diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index 6a6f34aa..337191a2 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -22,7 +22,7 @@ STATE_VERSION = 1 DEFAULT_GITHUB_REPO = "llvm/llvm-project" DEFAULT_SEARCH_LIMIT = 100 -DEFAULT_MAX_PR_AGE_DAYS = 30 +DEFAULT_MAX_PR_AGE_DAYS = 14 BACKEND_SEARCH_QUERIES: dict[str, str] = { "amdgpu": 'path:"llvm/lib/Target/AMDGPU"', From 83536fc5d84e36aa4ac9db5116218cc0fa98fee3 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 07:59:47 -0500 Subject: [PATCH 13/26] Add --backends filter to PR checker for amdgpu-only runs --- scripts/pr-check/check-llvm-prs.sh | 10 ++++++ scripts/pr_check/checker.py | 50 ++++++++++++++++++++++++++++-- tests/test_pr_checker.py | 29 +++++++++++++++-- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index 8f361757..1defea62 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -14,6 +14,7 @@ PR_CHECK_JOBS="${PR_CHECK_JOBS:-$(nproc)}" PR_CHECK_MAX_PER_RUN="${PR_CHECK_MAX_PER_RUN:-1}" PR_CHECK_SEARCH_LIMIT="${PR_CHECK_SEARCH_LIMIT:-100}" PR_CHECK_MAX_AGE_DAYS="${PR_CHECK_MAX_AGE_DAYS:-14}" +PR_CHECK_BACKENDS="${PR_CHECK_BACKENDS:-}" PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" @@ -46,6 +47,7 @@ Options: --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) --max-age-days Only PRs opened in the last N days (default: ${PR_CHECK_MAX_AGE_DAYS}) + --backends Comma-separated backends to check (default: amdgpu,spirv) --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) --log-level Python log level: debug, info, warning, error (default: ${PR_CHECK_LOG_LEVEL}) @@ -255,6 +257,11 @@ while [[ $# -gt 0 ]]; do PR_CHECK_MAX_AGE_DAYS="$2" shift 2 ;; + --backends) + [[ $# -ge 2 ]] || { echo "error: --backends requires a value" >&2; exit 2; } + PR_CHECK_BACKENDS="$2" + shift 2 + ;; --jobs) [[ $# -ge 2 ]] || { echo "error: --jobs requires a value" >&2; exit 2; } PR_CHECK_JOBS="$2" @@ -313,6 +320,9 @@ common_args=( --github-repo "$GITHUB_REPO" --max-age-days "$PR_CHECK_MAX_AGE_DAYS" ) +if [[ -n "$PR_CHECK_BACKENDS" ]]; then + common_args+=(--backends "$PR_CHECK_BACKENDS") +fi if [[ "$report_only" -eq 1 ]]; then generate_report diff --git a/scripts/pr_check/checker.py b/scripts/pr_check/checker.py index 337191a2..455f945e 100644 --- a/scripts/pr_check/checker.py +++ b/scripts/pr_check/checker.py @@ -108,6 +108,38 @@ def _validate_max_age_days(max_age_days: int) -> None: raise PrCheckerError(f"max_age_days must be a positive integer: {max_age_days}") +def _parse_backends_arg(raw: str | None) -> list[str] | None: + """Parse a comma-separated backend list from CLI input.""" + if raw is None: + return None + names = [part.strip() for part in raw.split(",") if part.strip()] + if not names: + raise PrCheckerError("backends must list at least one backend when provided") + return _validate_backends(names) + + +def _validate_backends(backends: list[str]) -> list[str]: + unknown = sorted(set(backends) - set(BACKEND_SEARCH_QUERIES)) + if unknown: + raise PrCheckerError( + f"unknown backend(s): {', '.join(unknown)} " + f"(known: {', '.join(sorted(BACKEND_SEARCH_QUERIES))})" + ) + seen: set[str] = set() + resolved: list[str] = [] + for backend in backends: + if backend not in seen: + seen.add(backend) + resolved.append(backend) + return resolved + + +def _backends_to_search(backends: list[str] | None) -> list[str]: + if backends is None: + return list(BACKEND_SEARCH_QUERIES) + return _validate_backends(backends) + + def _search_created_since(max_age_days: int) -> str: """Return a YYYY-MM-DD date for GitHub ``created:>`` search qualifiers.""" _validate_max_age_days(max_age_days) @@ -243,10 +275,11 @@ def _search_results_dataframe( github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT, max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, + backends: list[str] | None = None, ) -> pd.DataFrame: """Run backend searches and return one row per (PR, backend) match.""" frames: list[pd.DataFrame] = [] - for backend in BACKEND_SEARCH_QUERIES: + for backend in _backends_to_search(backends): items = _search_backend_prs( backend, github_repo=github_repo, @@ -294,19 +327,22 @@ def discover_prs( github_repo: str = DEFAULT_GITHUB_REPO, limit: int = DEFAULT_SEARCH_LIMIT, max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, + backends: list[str] | None = None, ) -> list[DiscoveredPr]: """Find open PRs touching AMDGPU and/or SPIR-V target paths.""" + search_backends = _backends_to_search(backends) log.info( "Discovering open PRs on %s (limit=%d per backend, max_age_days=%d, backends=%s)", github_repo, limit, max_age_days, - ", ".join(sorted(BACKEND_SEARCH_QUERIES)), + ", ".join(search_backends), ) search_df = _search_results_dataframe( github_repo=github_repo, limit=limit, max_age_days=max_age_days, + backends=backends, ) grouped = _aggregate_search_results(search_df) if grouped.empty: @@ -821,6 +857,7 @@ def cmd_discover(args: argparse.Namespace) -> int: github_repo=args.github_repo, limit=args.limit, max_age_days=args.max_age_days, + backends=_parse_backends_arg(args.backends), ) payload = _discovered_to_json(discovered) log.info("Writing %d discovered PR(s) to stdout as JSON", len(payload)) @@ -833,6 +870,7 @@ def cmd_plan(args: argparse.Namespace) -> int: github_repo=args.github_repo, limit=args.limit, max_age_days=args.max_age_days, + backends=_parse_backends_arg(args.backends), ) state = load_state(args.state_file) work = plan_work(discovered, state) @@ -953,6 +991,14 @@ def build_parser() -> argparse.ArgumentParser: f"(default: {DEFAULT_MAX_PR_AGE_DAYS})" ), ) + search_parent.add_argument( + "--backends", + default=None, + help=( + "Comma-separated backend names to search " + f"(default: {', '.join(sorted(BACKEND_SEARCH_QUERIES))})" + ), + ) discover_parser = subparsers.add_parser( "discover", diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index a0aa96eb..6175e7dd 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -255,8 +255,10 @@ def test_backend_search_query_includes_opened_within_filter(self) -> None: def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: from pr_check.checker import discover_prs - search_payload = [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] - with mock.patch("pr_check.checker._search_backend_prs", return_value=search_payload): + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): with mock.patch( "pr_check.checker._view_pr", return_value={ @@ -272,6 +274,29 @@ def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: self.assertEqual(discovered[0].head_sha, "deadbeef") self.assertEqual(discovered[0].backends, ["amdgpu", "spirv"]) + def test_discover_prs_honors_backends_filter(self) -> None: + from pr_check.checker import discover_prs + + searched: list[str] = [] + + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + searched.append(backend) + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): + with mock.patch( + "pr_check.checker._view_pr", + return_value={ + "headRefOid": "deadbeef", + "title": "Test PR", + "updatedAt": "2026-01-02", + }, + ): + discovered = discover_prs(limit=1, backends=["amdgpu"]) + + self.assertEqual(searched, ["amdgpu"]) + self.assertEqual(discovered[0].backends, ["amdgpu"]) + if __name__ == "__main__": unittest.main() From e0ac48473bb832f0c7d6545f3d9af6244d7d4011 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 08:03:59 -0500 Subject: [PATCH 14/26] Add --drain-queue to process all pending PR checks in one run --- scripts/pr-check/check-llvm-prs.sh | 108 ++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index 1defea62..97f9bb26 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -27,6 +27,7 @@ CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" discover_only=0 plan_only=0 report_only=0 +drain_queue=0 usage() { cat < Per-run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) --report-dir Report output directory (default: ${PR_CHECK_REPORT_DIR}) --max-per-run Max PR/backend checks per invocation (default: ${PR_CHECK_MAX_PER_RUN}) + --drain-queue Process all pending checks (one at a time until queue empty) --max-age-days Only PRs opened in the last N days (default: ${PR_CHECK_MAX_AGE_DAYS}) --backends Comma-separated backends to check (default: amdgpu,spirv) --jobs Parallel jobs for Docker build and LIT (default: ${PR_CHECK_JOBS}) @@ -208,6 +210,43 @@ run_one_work_item() { echo "Result: status=${status} gap_count=${gap_count} lit_failures=${lit_failure_count}" } +plan_work_json() { + local max_items="$1" + local plan_args=( + "${common_args[@]}" + --state-file "$PR_CHECK_STATE_FILE" + --limit "$PR_CHECK_SEARCH_LIMIT" + ) + if [[ -n "$max_items" ]]; then + plan_args+=(--max-items "$max_items") + fi + run_pr_check plan "${plan_args[@]}" +} + +work_count_from_json() { + local work_json="$1" + python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$work_json" +} + +run_work_items() { + local work_json="$1" + + while IFS= read -r item; do + local pr_number backend title head_sha reason + pr_number="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["pr_number"])' "$item")" + backend="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["backend"])' "$item")" + title="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["title"])' "$item")" + head_sha="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["head_sha"])' "$item")" + reason="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["reason"])' "$item")" + + echo + echo ">>> Running check (${reason}): #${pr_number} ${backend}" + if ! run_one_work_item "$pr_number" "$backend" "$title" "$head_sha"; then + failures=$((failures + 1)) + fi + done < <(python3 -c 'import json,sys; print("\n".join(json.dumps(x) for x in json.load(sys.stdin)))' <<<"$work_json") +} + while [[ $# -gt 0 ]]; do case "$1" in --discover-only) @@ -252,6 +291,10 @@ while [[ $# -gt 0 ]]; do PR_CHECK_MAX_PER_RUN="$2" shift 2 ;; + --drain-queue) + drain_queue=1 + shift + ;; --max-age-days) [[ $# -ge 2 ]] || { echo "error: --max-age-days requires a value" >&2; exit 2; } PR_CHECK_MAX_AGE_DAYS="$2" @@ -348,38 +391,51 @@ fi validate_runtime_config echo "=== automated PR check run (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 -echo "Discovering and planning work (max ${PR_CHECK_MAX_PER_RUN} check(s) this run)..." >&2 -work_json="$(run_pr_check plan \ - "${common_args[@]}" \ - --state-file "$PR_CHECK_STATE_FILE" \ - --limit "$PR_CHECK_SEARCH_LIMIT" \ - --max-items "$PR_CHECK_MAX_PER_RUN")" +failures=0 +total_checks=0 + +if [[ "$drain_queue" -eq 1 ]]; then + echo "Draining queue (one check at a time until empty)..." >&2 + while true; do + work_json="$(plan_work_json 1)" + work_count="$(work_count_from_json "$work_json")" + if [[ "$work_count" -eq 0 ]]; then + break + fi + + total_checks=$((total_checks + work_count)) + echo "Planned check ${total_checks} (draining queue)..." >&2 + run_work_items "$work_json" + done + + if [[ "$total_checks" -eq 0 ]]; then + echo "No PR/backend pairs need checking." + else + echo + echo "Completed ${total_checks} check(s) while draining queue." + fi +else + echo "Discovering and planning work (max ${PR_CHECK_MAX_PER_RUN} check(s) this run)..." >&2 + + work_json="$(plan_work_json "$PR_CHECK_MAX_PER_RUN")" + work_count="$(work_count_from_json "$work_json")" + if [[ "$work_count" -eq 0 ]]; then + echo "No PR/backend pairs need checking." + generate_report + exit 0 + fi -work_count="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$work_json")" -if [[ "$work_count" -eq 0 ]]; then - echo "No PR/backend pairs need checking." + echo "Planned ${work_count} PR/backend check(s)." + total_checks=$work_count + run_work_items "$work_json" +fi + +if [[ "$total_checks" -eq 0 ]]; then generate_report exit 0 fi -echo "Planned ${work_count} PR/backend check(s)." - -failures=0 -while IFS= read -r item; do - pr_number="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["pr_number"])' "$item")" - backend="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["backend"])' "$item")" - title="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["title"])' "$item")" - head_sha="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["head_sha"])' "$item")" - reason="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["reason"])' "$item")" - - echo - echo ">>> Running check (${reason}): #${pr_number} ${backend}" - if ! run_one_work_item "$pr_number" "$backend" "$title" "$head_sha"; then - failures=$((failures + 1)) - fi -done < <(python3 -c 'import json,sys; print("\n".join(json.dumps(x) for x in json.load(sys.stdin)))' <<<"$work_json") - if [[ "$failures" -gt 0 ]]; then echo echo "${failures} check(s) failed (see state file for details)." From 37233d334a68c008c8680e2fa3d26b160d91941e Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 08:13:03 -0500 Subject: [PATCH 15/26] Add run-daily-amdgpu.sh cron wrapper with flock and logging --- scripts/pr-check/run-daily-amdgpu.sh | 93 ++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 scripts/pr-check/run-daily-amdgpu.sh diff --git a/scripts/pr-check/run-daily-amdgpu.sh b/scripts/pr-check/run-daily-amdgpu.sh new file mode 100755 index 00000000..d67a4aa4 --- /dev/null +++ b/scripts/pr-check/run-daily-amdgpu.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Daily cron entry point for AMDGPU-only LLVM PR coverage-gap checking. +# +# Uses flock to avoid overlapping runs, loads local config.env, and drains +# all pending AMDGPU PR checks opened within the last 14 days. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" +LOCK_FILE="${REPO_ROOT}/data/pr-check/.amdgpu-daily.lock" +ORCHESTRATOR="${SCRIPT_DIR}/check-llvm-prs.sh" + +log() { + printf '[%s] %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" >&2 +} + +usage() { + cat <&2 + usage >&2 + exit 2 +fi + +# Cron runs with a minimal PATH; include common install locations. +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${PATH:-}" + +if [[ ! -x "$ORCHESTRATOR" ]]; then + log "error: orchestrator not found or not executable: ${ORCHESTRATOR}" + exit 1 +fi + +if [[ ! -f "$CONFIG_FILE" ]]; then + log "error: config file not found: ${CONFIG_FILE}" + log "Copy ${SCRIPT_DIR}/config.example.env to ${CONFIG_FILE} and set LLVM_REPO." + exit 1 +fi + +# shellcheck source=/dev/null +source "$CONFIG_FILE" + +if [[ -z "${LLVM_REPO:-}" ]]; then + log "error: LLVM_REPO is not set in ${CONFIG_FILE}" + exit 1 +fi + +if [[ "${LLVM_REPO}" != /* ]]; then + LLVM_REPO="${REPO_ROOT}/${LLVM_REPO}" + export LLVM_REPO + log "Resolved relative LLVM_REPO to ${LLVM_REPO}" +fi + +mkdir -p "$(dirname "$LOCK_FILE")" + +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log "Another daily AMDGPU PR check is already running; exiting." + exit 0 +fi + +log "Starting daily AMDGPU PR check (repo=${REPO_ROOT})" +cd "$REPO_ROOT" + +"${ORCHESTRATOR}" \ + --config "$CONFIG_FILE" \ + --backends amdgpu \ + --max-age-days "${PR_CHECK_MAX_AGE_DAYS:-14}" \ + --drain-queue + +log "Daily AMDGPU PR check finished." From 6d86b9c8e21dc666c8e5025d18bf2f8eb0db01da Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 08:19:33 -0500 Subject: [PATCH 16/26] Add example crontab for daily AMDGPU PR checks --- scripts/pr-check/cron/amdgpu-daily.example | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 scripts/pr-check/cron/amdgpu-daily.example diff --git a/scripts/pr-check/cron/amdgpu-daily.example b/scripts/pr-check/cron/amdgpu-daily.example new file mode 100644 index 00000000..b32107ba --- /dev/null +++ b/scripts/pr-check/cron/amdgpu-daily.example @@ -0,0 +1,28 @@ +# Example crontab entry for daily AMDGPU PR coverage-gap checking. +# +# Install: +# 1. Copy scripts/pr-check/config.example.env to scripts/pr-check/config.env +# 2. Set LLVM_REPO to an absolute path in config.env +# 3. Ensure gh auth login and Docker work for the cron user +# 4. Create the log file (or pick a path you can write to): +# sudo touch /var/log/fuzz-fill-amdgpu-pr-check.log +# sudo chown "$USER":"$USER" /var/log/fuzz-fill-amdgpu-pr-check.log +# 5. Edit paths below, then install one line into your crontab: +# crontab -e +# +# Optional: email on failure (if mail is configured on the host): +# MAILTO=you@example.com + +# Daily at 02:00 UTC — drain all pending AMDGPU PRs opened in the last 14 days. +0 2 * * * /home/USER/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh >> /var/log/fuzz-fill-amdgpu-pr-check.log 2>&1 + +# Notes: +# - Use absolute paths for the script and log file. +# - First run may take many hours (backfill within the 14-day window). +# - Subsequent runs only check new PRs or PRs with changed head SHAs. +# - Results: data/pr-check/reports/latest.md under the fuzz-fill repo. +# - If a prior run is still going, flock skips the new invocation (exit 0). +# - Dry run without builds: +# /home/USER/local/dev/fuzz-fill/scripts/pr-check/check-llvm-prs.sh \ +# --config /home/USER/local/dev/fuzz-fill/scripts/pr-check/config.env \ +# --backends amdgpu --plan-only From f5ae0c60176926170169b95b94add8be0d9ff5da Mon Sep 17 00:00:00 2001 From: agorzyns Date: Fri, 31 Jul 2026 10:33:54 -0500 Subject: [PATCH 17/26] Document daily AMDGPU cron setup and new PR check options --- README.md | 4 +- scripts/pr-check/README.md | 75 ++++++++++++++++++++++++++++- scripts/pr-check/config.example.env | 6 +++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa8d701f..c48798de 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,8 @@ Main output: `/commit_lines_report/target_lines_uncovered.csv`. See To automatically check open AMDGPU and SPIR-V PRs on a schedule, use [`scripts/pr-check/check-llvm-prs.sh`](scripts/pr-check/check-llvm-prs.sh). It discovers PRs via `gh`, re-runs gap detection when a PR head SHA changes, and writes local reports under `data/pr-check/reports/`. +For a **daily AMDGPU-only** cron job, use [`scripts/pr-check/run-daily-amdgpu.sh`](scripts/pr-check/run-daily-amdgpu.sh) (see [`scripts/pr-check/README.md`](scripts/pr-check/README.md)). + ```bash cp scripts/pr-check/config.example.env scripts/pr-check/config.env # Set LLVM_REPO in config.env, then: @@ -416,7 +418,7 @@ cp scripts/pr-check/config.example.env scripts/pr-check/config.env ./scripts/pr-check/check-llvm-prs.sh ``` -See [`scripts/pr-check/README.md`](scripts/pr-check/README.md) for cron/systemd setup, configuration options, and output layout. +See [`scripts/pr-check/README.md`](scripts/pr-check/README.md) for cron setup, configuration options, and output layout. ### Run integration tests diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index 8495276d..f8afe220 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -35,6 +35,10 @@ Run from the fuzz-fill repo root: # Discover, plan, run up to PR_CHECK_MAX_PER_RUN checks, and write reports ./scripts/pr-check/check-llvm-prs.sh + +# AMDGPU only, or drain all pending checks in one run +./scripts/pr-check/check-llvm-prs.sh --backends amdgpu +./scripts/pr-check/check-llvm-prs.sh --backends amdgpu --drain-queue ``` Override config path: @@ -43,6 +47,12 @@ Override config path: ./scripts/pr-check/check-llvm-prs.sh --config /path/to/my-config.env ``` +Daily cron entry point (AMDGPU only, drains the full queue): + +```bash +./scripts/pr-check/run-daily-amdgpu.sh +``` + ## What each run does 1. **Discover** — `gh search prs` for open PRs touching `llvm/lib/Target/AMDGPU` and/or `llvm/lib/Target/SPIRV` @@ -53,6 +63,15 @@ Override config path: - Record `gaps`, `clean`, or `failed` in state 4. **Report** — write `data/pr-check/reports/latest.json` and `latest.md` +### CLI options + +| Flag / env var | Default | Purpose | +|----------------|---------|---------| +| `--backends` / `PR_CHECK_BACKENDS` | amdgpu, spirv | Limit discovery to specific backends | +| `--max-age-days` / `PR_CHECK_MAX_AGE_DAYS` | 14 | Only PRs opened within the last N days | +| `--max-per-run` / `PR_CHECK_MAX_PER_RUN` | 1 | Max checks per invocation (ignored with `--drain-queue`) | +| `--drain-queue` | off | Process all pending checks one at a time until empty | + ## Output layout | Path | Description | @@ -65,7 +84,61 @@ Override config path: A PR has **coverage gaps** when `target_lines_uncovered.csv` is non-empty (added lines not covered by the regression suite). -## Cron example +## Daily AMDGPU cron + +For a once-daily job that checks all pending AMDGPU PRs opened within the last 14 days: + +### One-time setup + +```bash +cp scripts/pr-check/config.example.env scripts/pr-check/config.env +# Set LLVM_REPO to an absolute path (required for cron) +# Ensure gh auth login and Docker work for the cron user +pip install -e . +``` + +`config.env` is local-only (not committed). Use absolute paths in cron configs. + +### Test before scheduling + +```bash +# See what would run (no LLVM builds) +./scripts/pr-check/check-llvm-prs.sh \ + --config scripts/pr-check/config.env \ + --backends amdgpu \ + --plan-only + +# Full daily run (builds LLVM Docker images — can take hours on first backfill) +./scripts/pr-check/run-daily-amdgpu.sh +``` + +### Install cron + +See [`cron/amdgpu-daily.example`](cron/amdgpu-daily.example) for a ready-to-edit crontab snippet. Typical install: + +```bash +# Create log file (adjust path as needed) +sudo touch /var/log/fuzz-fill-amdgpu-pr-check.log +sudo chown "$USER":"$USER" /var/log/fuzz-fill-amdgpu-pr-check.log + +crontab -e +# Add the line from cron/amdgpu-daily.example (with your paths) +``` + +The wrapper [`run-daily-amdgpu.sh`](run-daily-amdgpu.sh) uses `flock` to skip if a prior run is still going. + +### Expected behavior + +| Run | What happens | +|-----|--------------| +| **First daily run** | Discovers all open AMDGPU PRs from the last 14 days; checks every PR not yet in `state.json` (or with a changed head SHA). May take many hours. | +| **Subsequent days** | Same 14-day discovery window, but most PRs are skipped (unchanged SHA). Only new or updated PRs are checked. | + +Results land in `data/pr-check/reports/latest.md`. + +## Lightweight polling cron + +For frequent runs that process one PR at a time (both backends): ```cron # Every 6 hours: run at most one PR/backend check, then refresh reports diff --git a/scripts/pr-check/config.example.env b/scripts/pr-check/config.example.env index 5fac7f65..de1721ef 100644 --- a/scripts/pr-check/config.example.env +++ b/scripts/pr-check/config.example.env @@ -9,14 +9,20 @@ LLVM_REPO=/path/to/llvm-project PR_CHECK_JOBS=$(nproc) # Max PR/backend checks per cron invocation (each first-time build compiles LLVM). +# Ignored when using run-daily-amdgpu.sh (it passes --drain-queue). PR_CHECK_MAX_PER_RUN=1 # Max open PRs returned per backend search (AMDGPU and SPIR-V). PR_CHECK_SEARCH_LIMIT=100 # Only include PRs opened within this many days (GitHub created:> filter). +# Used by the daily AMDGPU cron wrapper and --max-age-days on check-llvm-prs.sh. PR_CHECK_MAX_AGE_DAYS=14 +# Optional: limit discovery/planning to specific backends (comma-separated). +# Example for AMDGPU-only: PR_CHECK_BACKENDS=amdgpu +# PR_CHECK_BACKENDS= + # Host paths for state, per-run artifacts, and aggregated reports. PR_CHECK_OUTPUT_ROOT=data/pr-check/runs PR_CHECK_STATE_FILE=data/pr-check/state.json From 01c91019dc237544133978dfc3f948257b8a87d1 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Mon, 3 Aug 2026 01:26:01 -0500 Subject: [PATCH 18/26] Move log to fuzz-fill directory --- scripts/pr-check/README.md | 8 ++------ scripts/pr-check/cron/amdgpu-daily.example | 12 ++++++------ scripts/pr-check/run-daily-amdgpu.sh | 9 ++++++++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index f8afe220..8477f18d 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -117,15 +117,11 @@ pip install -e . See [`cron/amdgpu-daily.example`](cron/amdgpu-daily.example) for a ready-to-edit crontab snippet. Typical install: ```bash -# Create log file (adjust path as needed) -sudo touch /var/log/fuzz-fill-amdgpu-pr-check.log -sudo chown "$USER":"$USER" /var/log/fuzz-fill-amdgpu-pr-check.log - crontab -e # Add the line from cron/amdgpu-daily.example (with your paths) ``` -The wrapper [`run-daily-amdgpu.sh`](run-daily-amdgpu.sh) uses `flock` to skip if a prior run is still going. +The wrapper [`run-daily-amdgpu.sh`](run-daily-amdgpu.sh) appends logs to `logs/pr-check/amdgpu-daily.log` (gitignored, no sudo required) and uses `flock` to skip if a prior run is still going. ### Expected behavior @@ -142,7 +138,7 @@ For frequent runs that process one PR at a time (both backends): ```cron # Every 6 hours: run at most one PR/backend check, then refresh reports -0 */6 * * * cd /path/to/fuzz-fill && ./scripts/pr-check/check-llvm-prs.sh >> /var/log/fuzz-fill-pr-check.log 2>&1 +0 */6 * * * cd /path/to/fuzz-fill && ./scripts/pr-check/check-llvm-prs.sh >> logs/pr-check/polling.log 2>&1 ``` Keep `PR_CHECK_MAX_PER_RUN=1` unless you have enough CPU/time for multiple LLVM Docker builds per invocation. diff --git a/scripts/pr-check/cron/amdgpu-daily.example b/scripts/pr-check/cron/amdgpu-daily.example index b32107ba..84c44242 100644 --- a/scripts/pr-check/cron/amdgpu-daily.example +++ b/scripts/pr-check/cron/amdgpu-daily.example @@ -4,20 +4,20 @@ # 1. Copy scripts/pr-check/config.example.env to scripts/pr-check/config.env # 2. Set LLVM_REPO to an absolute path in config.env # 3. Ensure gh auth login and Docker work for the cron user -# 4. Create the log file (or pick a path you can write to): -# sudo touch /var/log/fuzz-fill-amdgpu-pr-check.log -# sudo chown "$USER":"$USER" /var/log/fuzz-fill-amdgpu-pr-check.log -# 5. Edit paths below, then install one line into your crontab: +# 4. Edit the script path below, then install into your user crontab: # crontab -e # +# Logs append to logs/pr-check/amdgpu-daily.log under the fuzz-fill repo +# (already gitignored — no sudo or /var/log needed). +# # Optional: email on failure (if mail is configured on the host): # MAILTO=you@example.com # Daily at 02:00 UTC — drain all pending AMDGPU PRs opened in the last 14 days. -0 2 * * * /home/USER/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh >> /var/log/fuzz-fill-amdgpu-pr-check.log 2>&1 +0 2 * * * /home/USER/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh # Notes: -# - Use absolute paths for the script and log file. +# - Use an absolute path to run-daily-amdgpu.sh. # - First run may take many hours (backfill within the 14-day window). # - Subsequent runs only check new PRs or PRs with changed head SHAs. # - Results: data/pr-check/reports/latest.md under the fuzz-fill repo. diff --git a/scripts/pr-check/run-daily-amdgpu.sh b/scripts/pr-check/run-daily-amdgpu.sh index d67a4aa4..4408db94 100755 --- a/scripts/pr-check/run-daily-amdgpu.sh +++ b/scripts/pr-check/run-daily-amdgpu.sh @@ -10,6 +10,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" LOCK_FILE="${REPO_ROOT}/data/pr-check/.amdgpu-daily.lock" +LOG_FILE="${PR_CHECK_LOG_FILE:-${REPO_ROOT}/logs/pr-check/amdgpu-daily.log}" ORCHESTRATOR="${SCRIPT_DIR}/check-llvm-prs.sh" log() { @@ -27,7 +28,10 @@ Configuration: to an absolute path before scheduling this script. Environment: - PR_CHECK_CONFIG Override config file path (default: ${CONFIG_FILE}) + PR_CHECK_CONFIG Override config file path (default: ${CONFIG_FILE}) + PR_CHECK_LOG_FILE Override log file path (default: ${LOG_FILE}) + +Logs append to logs/pr-check/amdgpu-daily.log under the repo (gitignored). For a dry run without builds, use: ${ORCHESTRATOR} --config ${CONFIG_FILE} --backends amdgpu --plan-only @@ -48,6 +52,9 @@ fi # Cron runs with a minimal PATH; include common install locations. export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HOME}/.local/bin:${PATH:-}" +mkdir -p "$(dirname "$LOG_FILE")" +exec > >(tee -a "$LOG_FILE") 2>&1 + if [[ ! -x "$ORCHESTRATOR" ]]; then log "error: orchestrator not found or not executable: ${ORCHESTRATOR}" exit 1 From d4bede70132786e5752f9c7beb6b3ff227733f17 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Mon, 3 Aug 2026 01:38:50 -0500 Subject: [PATCH 19/26] Make cron instructions more precise --- scripts/pr-check/cron/amdgpu-daily.example | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/pr-check/cron/amdgpu-daily.example b/scripts/pr-check/cron/amdgpu-daily.example index 84c44242..251d543e 100644 --- a/scripts/pr-check/cron/amdgpu-daily.example +++ b/scripts/pr-check/cron/amdgpu-daily.example @@ -4,9 +4,21 @@ # 1. Copy scripts/pr-check/config.example.env to scripts/pr-check/config.env # 2. Set LLVM_REPO to an absolute path in config.env # 3. Ensure gh auth login and Docker work for the cron user -# 4. Edit the script path below, then install into your user crontab: -# crontab -e +# 4. Install the schedule into *your* crontab (runs as your user, not root): +# a. Open your crontab file in an editor: +# crontab -e +# (First time: pick an editor, e.g. nano. Opens ~/.crontab or a temp file.) +# b. Copy the cron line from step 5 below (the line starting with "0 2"). +# c. Replace /home/USER with your home directory, e.g. for user agorzyns: +# /home/agorzyns/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh +# Tip: run `realpath scripts/pr-check/run-daily-amdgpu.sh` from the repo root. +# d. Paste the edited line as a new line at the end of the file. +# e. Save and exit the editor (in nano: Ctrl+O, Enter, Ctrl+X). +# f. Confirm it was installed: +# crontab -l +# You should see your new line listed. # +# 5. Cron line to paste (edit the path first — see 4c): # Logs append to logs/pr-check/amdgpu-daily.log under the fuzz-fill repo # (already gitignored — no sudo or /var/log needed). # @@ -16,6 +28,8 @@ # Daily at 02:00 UTC — drain all pending AMDGPU PRs opened in the last 14 days. 0 2 * * * /home/USER/local/dev/fuzz-fill/scripts/pr-check/run-daily-amdgpu.sh +# Cron schedule format: minute hour day-of-month month day-of-week command +# "0 2 * * *" = every day at 02:00 (server local time; check `date` if unsure). # Notes: # - Use an absolute path to run-daily-amdgpu.sh. # - First run may take many hours (backfill within the 14-day window). From a50b206e436de561dcbd2061375d926e8ad603da Mon Sep 17 00:00:00 2001 From: agorzyns Date: Mon, 3 Aug 2026 01:41:42 -0500 Subject: [PATCH 20/26] Move pr_check module to src/ --- pyproject.toml | 1 + scripts/pr-check/README.md | 8 ++++---- scripts/pr-check/check-llvm-prs.sh | 2 +- {scripts => src}/pr_check/__init__.py | 0 {scripts => src}/pr_check/__main__.py | 0 {scripts => src}/pr_check/checker.py | 0 tests/test_pr_checker.py | 7 +------ 7 files changed, 7 insertions(+), 11 deletions(-) rename {scripts => src}/pr_check/__init__.py (100%) rename {scripts => src}/pr_check/__main__.py (100%) rename {scripts => src}/pr_check/checker.py (100%) diff --git a/pyproject.toml b/pyproject.toml index b1c43365..9be86551 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ reduce = "reduce.__main__:main" llvm-test-suite-coverage = "coverage.__main__:main" added-lines = "added_lines.__main__:main" +pr-check = "pr_check.__main__:main" [tool.setuptools] package-dir = { "" = "src" } diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index 8477f18d..7118d1ed 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -180,12 +180,12 @@ sudo systemctl enable --now fuzz-fill-pr-check.timer ## Low-level CLI -The orchestrator wraps [`scripts/pr_check/checker.py`](../pr_check/checker.py): +The orchestrator wraps [`src/pr_check/checker.py`](../../src/pr_check/checker.py): ```bash -PYTHONPATH=scripts python3 -m pr_check discover -PYTHONPATH=scripts python3 -m pr_check plan --state-file data/pr-check/state.json -PYTHONPATH=scripts python3 -m pr_check report \ +PYTHONPATH=src python3 -m pr_check discover +PYTHONPATH=src python3 -m pr_check plan --state-file data/pr-check/state.json +PYTHONPATH=src python3 -m pr_check report \ --state-file data/pr-check/state.json \ --report-dir data/pr-check/reports ``` diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index 97f9bb26..b2c26c35 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -77,7 +77,7 @@ load_config() { } run_pr_check() { - PYTHONPATH="${REPO_ROOT}/scripts" python3 -m pr_check \ + PYTHONPATH="${REPO_ROOT}/src" python3 -m pr_check \ "$@" \ --log-level "$PR_CHECK_LOG_LEVEL" } diff --git a/scripts/pr_check/__init__.py b/src/pr_check/__init__.py similarity index 100% rename from scripts/pr_check/__init__.py rename to src/pr_check/__init__.py diff --git a/scripts/pr_check/__main__.py b/src/pr_check/__main__.py similarity index 100% rename from scripts/pr_check/__main__.py rename to src/pr_check/__main__.py diff --git a/scripts/pr_check/checker.py b/src/pr_check/checker.py similarity index 100% rename from scripts/pr_check/checker.py rename to src/pr_check/checker.py diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index 6175e7dd..2e3908c6 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -4,7 +4,6 @@ import csv import json -import sys import tempfile import unittest from pathlib import Path @@ -12,11 +11,7 @@ import pandas as pd -SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts" -if str(SCRIPTS_DIR) not in sys.path: - sys.path.insert(0, str(SCRIPTS_DIR)) - -from pr_check.checker import ( # noqa: E402 +from pr_check.checker import ( STATE_VERSION, DiscoveredPr, PrCheckerError, From 00cdaf2f011023ac949872ad3c5b20f0ea476c16 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Mon, 3 Aug 2026 04:08:59 -0500 Subject: [PATCH 21/26] Fix gh filter to capture all AMDGPU PRs --- src/pr_check/checker.py | 146 ++++++++++++++++++++++++++++++--------- tests/test_pr_checker.py | 74 ++++++++++++++++---- 2 files changed, 176 insertions(+), 44 deletions(-) diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py index 455f945e..facac093 100644 --- a/src/pr_check/checker.py +++ b/src/pr_check/checker.py @@ -24,13 +24,22 @@ DEFAULT_SEARCH_LIMIT = 100 DEFAULT_MAX_PR_AGE_DAYS = 14 -BACKEND_SEARCH_QUERIES: dict[str, str] = { - "amdgpu": 'path:"llvm/lib/Target/AMDGPU"', - "spirv": 'path:"llvm/lib/Target/SPIRV"', +BACKEND_SEARCH_QUERIES: dict[str, list[str]] = { + "amdgpu": [ + 'label:"backend:AMDGPU"', + ], + "spirv": [ + 'path:"llvm/lib/Target/SPIRV"', + ], +} + +BACKEND_TARGET_PATH_PREFIXES: dict[str, str] = { + "amdgpu": "llvm/lib/Target/AMDGPU/", + "spirv": "llvm/lib/Target/SPIRV/", } SEARCH_JSON_FIELDS = ["number", "title", "updatedAt"] -PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt"] +PR_VIEW_JSON_FIELDS = ["number", "title", "headRefOid", "updatedAt", "files"] LIT_FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"}) LOG_FORMAT = "%(levelname)-8s %(message)s" @@ -147,10 +156,50 @@ def _search_created_since(max_age_days: int) -> str: return cutoff.isoformat() -def _backend_search_query(backend: str, *, max_age_days: int) -> str: - """Build a gh search query for one backend, limited to recently opened PRs.""" +def _backend_search_terms(backend: str, *, max_age_days: int) -> list[str]: + """Build gh search terms for one backend, limited to recently opened PRs.""" created_since = _search_created_since(max_age_days) - return f'{BACKEND_SEARCH_QUERIES[backend]} created:>{created_since}' + age_filter = f"created:>{created_since}" + return [f"{term} {age_filter}" for term in BACKEND_SEARCH_QUERIES[backend]] + + +def _pr_changed_paths(pr_view: dict[str, Any]) -> list[str]: + files = pr_view.get("files") + if not isinstance(files, list): + return [] + paths: list[str] = [] + for entry in files: + if not isinstance(entry, dict): + continue + path = entry.get("path") + if isinstance(path, str) and path: + paths.append(path) + return paths + + +def _pr_touches_backend_path(pr_view: dict[str, Any], backend: str) -> bool: + prefix = BACKEND_TARGET_PATH_PREFIXES[backend] + return any(path.startswith(prefix) for path in _pr_changed_paths(pr_view)) + + +def _merge_search_items(items_list: list[list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Merge GitHub search hits, keeping one row per PR number.""" + merged: dict[int, dict[str, Any]] = {} + for items in items_list: + for item in items: + number = item.get("number") + if number is None: + continue + pr_number = int(number) + existing = merged.get(pr_number) + if existing is None: + merged[pr_number] = dict(item) + continue + if item.get("title") and not existing.get("title"): + existing["title"] = item["title"] + if (item.get("updatedAt") or "") > (existing.get("updatedAt") or ""): + existing["updatedAt"] = item["updatedAt"] + return [merged[pr_number] for pr_number in sorted(merged)] def _require_gh() -> str: @@ -230,23 +279,11 @@ def _json_fields_for_command(args: list[str]) -> list[str]: raise PrCheckerError(f"unsupported gh command for JSON parsing: {' '.join(args)}") -def _search_backend_prs( - backend: str, +def _search_issues( + search_query: str, *, - github_repo: str = DEFAULT_GITHUB_REPO, - limit: int = DEFAULT_SEARCH_LIMIT, - max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, + limit: int, ) -> list[dict[str, Any]]: - query = _backend_search_query(backend, max_age_days=max_age_days) - search_query = f"repo:{github_repo} is:pr is:open {query}" - log.info( - "Searching %s PRs on %s (query=%r, limit=%d, max_age_days=%d)", - backend, - github_repo, - search_query, - limit, - max_age_days, - ) payload = _gh_api_json( "search/issues?" + urlencode( @@ -257,10 +294,36 @@ def _search_backend_prs( ) ) if not isinstance(payload, dict): - raise PrCheckerError(f"unexpected gh api search payload for {backend}: {type(payload)!r}") - items = _normalize_search_items(payload.get("items", [])) - log.info("Found %d open %s PR(s)", len(items), backend) - return items[:limit] + raise PrCheckerError(f"unexpected gh api search payload: {type(payload)!r}") + return _normalize_search_items(payload.get("items", [])) + + +def _search_backend_prs( + backend: str, + *, + github_repo: str = DEFAULT_GITHUB_REPO, + limit: int = DEFAULT_SEARCH_LIMIT, + max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, +) -> list[dict[str, Any]]: + search_terms = _backend_search_terms(backend, max_age_days=max_age_days) + per_term_results: list[list[dict[str, Any]]] = [] + for term in search_terms: + search_query = f"repo:{github_repo} is:pr is:open {term}" + log.info( + "Searching %s PRs on %s (query=%r, limit=%d, max_age_days=%d)", + backend, + github_repo, + search_query, + limit, + max_age_days, + ) + items = _search_issues(search_query, limit=limit) + log.info("Found %d open %s PR(s) for search term", len(items), backend) + per_term_results.append(items) + + items = _merge_search_items(per_term_results)[:limit] + log.info("Found %d unique open %s PR(s) after merging search terms", len(items), backend) + return items def _view_pr(pr_number: int, github_repo: str) -> dict[str, Any]: @@ -329,7 +392,7 @@ def discover_prs( max_age_days: int = DEFAULT_MAX_PR_AGE_DAYS, backends: list[str] | None = None, ) -> list[DiscoveredPr]: - """Find open PRs touching AMDGPU and/or SPIR-V target paths.""" + """Find open PRs with changed files under AMDGPU/SPIR-V target directories.""" search_backends = _backends_to_search(backends) log.info( "Discovering open PRs on %s (limit=%d per backend, max_age_days=%d, backends=%s)", @@ -351,34 +414,53 @@ def discover_prs( total = len(grouped) log.info( - "Search returned %d unique PR(s); resolving head SHAs via gh pr view", + "Search returned %d unique PR(s); resolving PR metadata and filtering by target paths", total, ) discovered: list[DiscoveredPr] = [] + skipped_without_target_paths = 0 for index, row in enumerate(grouped.itertuples(index=False), start=1): pr_number = int(row.pr_number) if index == 1 or index == total or index % 10 == 0: - log.info("Resolving head SHA %d/%d: #%d", index, total, pr_number) + log.info("Inspecting PR %d/%d: #%d", index, total, pr_number) else: - log.debug("Resolving head SHA %d/%d: #%d", index, total, pr_number) + log.debug("Inspecting PR %d/%d: #%d", index, total, pr_number) pr_view = _view_pr(pr_number, github_repo) head_sha = pr_view.get("headRefOid") if not head_sha: raise PrCheckerError(f"could not resolve headRefOid for {github_repo}#{pr_number}") + matched_backends = [ + backend + for backend in row.backends + if _pr_touches_backend_path(pr_view, backend) + ] + if not matched_backends: + skipped_without_target_paths += 1 + log.debug( + "Skipping #%d: no changed files under tracked target path(s) for %s", + pr_number, + ", ".join(row.backends), + ) + continue + discovered.append( DiscoveredPr( pr_number=pr_number, title=pr_view.get("title") or row.title or "", head_sha=head_sha, updated_at=pr_view.get("updatedAt") or row.updated_at or "", - backends=list(row.backends), + backends=matched_backends, ) ) - log.info("Discovery complete: %d PR(s) with head SHAs", len(discovered)) + log.info( + "Discovery complete: %d PR(s) with target-path changes (%d skipped without target-path changes)", + len(discovered), + skipped_without_target_paths, + ) return discovered diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index 2e3908c6..b6083d2b 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -240,12 +240,50 @@ def test_build_render_and_write_reports(self) -> None: class DiscoverPrsTest(unittest.TestCase): - def test_backend_search_query_includes_opened_within_filter(self) -> None: - from pr_check.checker import _backend_search_query + def test_backend_search_terms_include_label_and_age_filter_for_amdgpu(self) -> None: + from pr_check.checker import _backend_search_terms - query = _backend_search_query("amdgpu", max_age_days=30) - self.assertIn('path:"llvm/lib/Target/AMDGPU"', query) - self.assertRegex(query, r'created:>\d{4}-\d{2}-\d{2}') + terms = _backend_search_terms("amdgpu", max_age_days=30) + self.assertEqual(len(terms), 1) + self.assertIn('label:"backend:AMDGPU"', terms[0]) + self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') + + def test_pr_touches_backend_path_matches_target_prefix(self) -> None: + from pr_check.checker import _pr_touches_backend_path + + pr_view = { + "files": [ + {"path": "clang/test/CodeGenHIP/foo.hip"}, + {"path": "llvm/lib/Target/AMDGPU/Foo.cpp"}, + ] + } + self.assertTrue(_pr_touches_backend_path(pr_view, "amdgpu")) + self.assertFalse(_pr_touches_backend_path(pr_view, "spirv")) + + def test_merge_search_items_deduplicates_by_pr_number(self) -> None: + from pr_check.checker import _merge_search_items + + merged = _merge_search_items( + [ + [{"number": 10, "title": "From path", "updatedAt": "2026-01-01"}], + [{"number": 10, "title": "", "updatedAt": "2026-01-03"}, {"number": 5, "title": "Label only", "updatedAt": "2026-01-02"}], + ] + ) + + self.assertEqual([item["number"] for item in merged], [5, 10]) + self.assertEqual(merged[1]["updatedAt"], "2026-01-03") + self.assertEqual(merged[1]["title"], "From path") + + def _pr_view_with_target_files(self) -> dict[str, object]: + return { + "headRefOid": "deadbeef", + "title": "Test PR", + "updatedAt": "2026-01-02", + "files": [ + {"path": "llvm/lib/Target/AMDGPU/Foo.cpp"}, + {"path": "llvm/lib/Target/SPIRV/Bar.cpp"}, + ], + } def test_discover_prs_resolves_head_sha_from_pr_view(self) -> None: from pr_check.checker import discover_prs @@ -256,11 +294,7 @@ def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): with mock.patch( "pr_check.checker._view_pr", - return_value={ - "headRefOid": "deadbeef", - "title": "Test PR", - "updatedAt": "2026-01-02", - }, + return_value=self._pr_view_with_target_files(), ): discovered = discover_prs(limit=1) @@ -278,6 +312,22 @@ def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: searched.append(backend) return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): + with mock.patch( + "pr_check.checker._view_pr", + return_value=self._pr_view_with_target_files(), + ): + discovered = discover_prs(limit=1, backends=["amdgpu"]) + + self.assertEqual(searched, ["amdgpu"]) + self.assertEqual(discovered[0].backends, ["amdgpu"]) + + def test_discover_prs_skips_prs_without_target_path_changes(self) -> None: + from pr_check.checker import discover_prs + + def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: + return [{"number": 99, "title": "Test PR", "updatedAt": "2026-01-01"}] + with mock.patch("pr_check.checker._search_backend_prs", side_effect=fake_search): with mock.patch( "pr_check.checker._view_pr", @@ -285,12 +335,12 @@ def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: "headRefOid": "deadbeef", "title": "Test PR", "updatedAt": "2026-01-02", + "files": [{"path": "clang/test/CodeGenHIP/foo.hip"}], }, ): discovered = discover_prs(limit=1, backends=["amdgpu"]) - self.assertEqual(searched, ["amdgpu"]) - self.assertEqual(discovered[0].backends, ["amdgpu"]) + self.assertEqual(discovered, []) if __name__ == "__main__": From fe8f53643b8adfdc4d66555ac92e467198d6abd2 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Tue, 4 Aug 2026 02:58:06 -0500 Subject: [PATCH 22/26] Extract pr check report writing into separate script Add guard to run report writing even if pr check fails while processing the pr queue --- scripts/pr-check/README.md | 17 ++++++- scripts/pr-check/check-llvm-prs.sh | 10 ++++ scripts/pr-check/get-latest.sh | 80 ++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100755 scripts/pr-check/get-latest.sh diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index 7118d1ed..9c5fec3f 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -30,7 +30,10 @@ Run from the fuzz-fill repo root: # Show planned work (PR/backend pairs needing a fresh check) ./scripts/pr-check/check-llvm-prs.sh --plan-only -# Regenerate latest.json / latest.md from state without running checks +# Refresh latest.json / latest.md from state (no checks, no LLVM_REPO needed) +./scripts/pr-check/get-latest.sh + +# Same as get-latest.sh, via the main orchestrator flag ./scripts/pr-check/check-llvm-prs.sh --report-only # Discover, plan, run up to PR_CHECK_MAX_PER_RUN checks, and write reports @@ -84,6 +87,18 @@ Daily cron entry point (AMDGPU only, drains the full queue): A PR has **coverage gaps** when `target_lines_uncovered.csv` is non-empty (added lines not covered by the regression suite). +## Refreshing reports + +Reports are built from `state.json`, not by scanning `runs/`. After a failed or interrupted check run, or anytime you want the summary synced to current state: + +```bash +./scripts/pr-check/get-latest.sh +``` + +Use this when `latest.md` looks stale (e.g. fewer entries than `state.json`, or checks finished but the orchestrator exited early). The markdown body lists only PRs **with gaps**; clean PRs appear in the summary line and in `latest.json`. + +The main orchestrator also refreshes reports at normal exit. An EXIT trap regenerates the report if at least one check ran but the process did not finish cleanly (Ctrl+C, OOM, or unexpected early exit during `--drain-queue`). + ## Daily AMDGPU cron For a once-daily job that checks all pending AMDGPU PRs opened within the last 14 days: diff --git a/scripts/pr-check/check-llvm-prs.sh b/scripts/pr-check/check-llvm-prs.sh index b2c26c35..55ac06c6 100755 --- a/scripts/pr-check/check-llvm-prs.sh +++ b/scripts/pr-check/check-llvm-prs.sh @@ -82,7 +82,10 @@ run_pr_check() { --log-level "$PR_CHECK_LOG_LEVEL" } +report_generated=0 + generate_report() { + report_generated=1 echo "=== coverage gap report ===" run_pr_check report \ --github-repo "$GITHUB_REPO" \ @@ -90,6 +93,12 @@ generate_report() { --report-dir "$PR_CHECK_REPORT_DIR" } +cleanup_report() { + if [[ "$report_generated" -eq 0 && "$total_checks" -gt 0 ]]; then + generate_report || true + fi +} + require_command() { if ! command -v "$1" >/dev/null 2>&1; then echo "error: required command not found: $1" >&2 @@ -394,6 +403,7 @@ echo "=== automated PR check run (log-level=${PR_CHECK_LOG_LEVEL}) ===" >&2 failures=0 total_checks=0 +trap cleanup_report EXIT if [[ "$drain_queue" -eq 1 ]]; then echo "Draining queue (one check at a time until empty)..." >&2 diff --git a/scripts/pr-check/get-latest.sh b/scripts/pr-check/get-latest.sh new file mode 100755 index 00000000..e5a90370 --- /dev/null +++ b/scripts/pr-check/get-latest.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Regenerate latest.json / latest.md from data/pr-check/state.json. +# +# Does not run PR checks or require LLVM_REPO — only refreshes the report +# summary from whatever is already recorded in state. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" +ORCHESTRATOR="${SCRIPT_DIR}/check-llvm-prs.sh" +REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" + +usage() { + cat < Load path overrides from a config file + (default: ${CONFIG_FILE} if present) + --help, -h Show this help + +Environment: + PR_CHECK_CONFIG Override config file path (default: ${CONFIG_FILE}) + PR_CHECK_REPORT_DIR Report output directory (default: data/pr-check/reports) + +Examples: + $(basename "$0") + $(basename "$0") --config /path/to/config.env +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ ! -x "$ORCHESTRATOR" ]]; then + echo "error: orchestrator not found or not executable: ${ORCHESTRATOR}" >&2 + exit 1 +fi + +orchestrator_args=(--report-only) +if [[ -f "$CONFIG_FILE" ]]; then + orchestrator_args+=(--config "$CONFIG_FILE") + # shellcheck source=/dev/null + source "$CONFIG_FILE" + REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +fi + +cd "$REPO_ROOT" +"${ORCHESTRATOR}" "${orchestrator_args[@]}" + +echo +echo "Report updated:" +echo " ${REPORT_DIR}/latest.md" +echo " ${REPORT_DIR}/latest.json" From a1cea56a5dca902c2d02a7a20df0903d2d0ef47f Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 5 Aug 2026 02:17:01 -0500 Subject: [PATCH 23/26] Add report option to show diff since last invocation of PR checking --- scripts/pr-check/README.md | 5 +- scripts/pr-check/get-latest.sh | 1 + src/pr_check/checker.py | 163 +++++++++++++++++++++++++++++---- tests/test_pr_checker.py | 155 +++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 19 deletions(-) diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index 9c5fec3f..e5bf64aa 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -64,7 +64,7 @@ Daily cron entry point (AMDGPU only, drains the full queue): - Build Docker image `fuzz-fill-test:llvm-pr--` via [`build-image-pr.sh`](../docker/build-image-pr.sh) - Run [`pr-cov-gaps-detection.sh`](../docker/pr-cov-gaps-detection.sh) - Record `gaps`, `clean`, or `failed` in state -4. **Report** — write `data/pr-check/reports/latest.json` and `latest.md` +4. **Report** — write `data/pr-check/reports/latest.json`, `latest.md`, and `new-prs.md` ### CLI options @@ -83,6 +83,7 @@ Daily cron entry point (AMDGPU only, drains the full queue): | `data/pr-check/runs/-/` | Per-check artifacts (`baseline/`, `commit_lines_report/`, …) | | `data/pr-check/reports/latest.json` | Machine-readable summary of all checked PRs | | `data/pr-check/reports/latest.md` | Human-readable gap report with PR links | +| `data/pr-check/reports/new-prs.md` | PRs new or updated since the previous report (diff vs prior `latest.json`) | | `data/pr-check/reports/runs/.json` | Snapshot from each report generation | A PR has **coverage gaps** when `target_lines_uncovered.csv` is non-empty (added lines not covered by the regression suite). @@ -95,7 +96,7 @@ Reports are built from `state.json`, not by scanning `runs/`. After a failed or ./scripts/pr-check/get-latest.sh ``` -Use this when `latest.md` looks stale (e.g. fewer entries than `state.json`, or checks finished but the orchestrator exited early). The markdown body lists only PRs **with gaps**; clean PRs appear in the summary line and in `latest.json`. +Use this when `latest.md` looks stale (e.g. fewer entries than `state.json`, or checks finished but the orchestrator exited early). The markdown body lists only PRs **with gaps**; clean PRs appear in the summary line and in `latest.json`. The same refresh also updates `new-prs.md` by diffing the current state against the previous `latest.json` — useful after an interrupted run where checks completed but report generation did not. The main orchestrator also refreshes reports at normal exit. An EXIT trap regenerates the report if at least one check ran but the process did not finish cleanly (Ctrl+C, OOM, or unexpected early exit during `--drain-queue`). diff --git a/scripts/pr-check/get-latest.sh b/scripts/pr-check/get-latest.sh index e5a90370..63cd1303 100755 --- a/scripts/pr-check/get-latest.sh +++ b/scripts/pr-check/get-latest.sh @@ -78,3 +78,4 @@ echo echo "Report updated:" echo " ${REPORT_DIR}/latest.md" echo " ${REPORT_DIR}/latest.json" +echo " ${REPORT_DIR}/new-prs.md" diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py index facac093..033e3f52 100644 --- a/src/pr_check/checker.py +++ b/src/pr_check/checker.py @@ -774,6 +774,105 @@ def sample_gaps_for_row(row: pd.Series) -> list[dict[str, str]]: return enriched +def _entry_key_from_record(entry: dict[str, Any]) -> str: + return entry_key(int(entry["pr_number"]), str(entry["backend"])) + + +def _index_report_entries(payload: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + """Map ``":"`` keys to entries from a report payload.""" + if payload is None: + return {} + indexed: dict[str, dict[str, Any]] = {} + for entry in payload.get("all_entries", []): + if isinstance(entry, dict): + indexed[_entry_key_from_record(entry)] = entry + return indexed + + +def diff_report_entries( + current: dict[str, Any], + previous: dict[str, Any] | None, +) -> set[str]: + """Return keys for entries that are new or updated since the previous report.""" + current_index = _index_report_entries(current) + if previous is None: + return set(current_index) + + previous_index = _index_report_entries(previous) + changed: set[str] = set() + for key, entry in current_index.items(): + prior = previous_index.get(key) + if prior is None: + changed.add(key) + continue + if entry.get("checked_at") != prior.get("checked_at"): + changed.add(key) + continue + if entry.get("head_sha") != prior.get("head_sha"): + changed.add(key) + return changed + + +def filter_report_payload(payload: dict[str, Any], keys: set[str]) -> dict[str, Any]: + """Return a copy of a report payload limited to the given entry keys.""" + if not keys: + return { + "generated_at": payload["generated_at"], + "github_repo": payload["github_repo"], + "summary": { + "total_entries": 0, + "with_gaps": 0, + "clean": 0, + "failed": 0, + }, + "entries_with_gaps": [], + "failed_entries": [], + "all_entries": [], + } + + def keep(entry: dict[str, Any]) -> bool: + return _entry_key_from_record(entry) in keys + + entries_with_gaps = [ + entry for entry in payload.get("entries_with_gaps", []) if keep(entry) + ] + failed_entries = [entry for entry in payload.get("failed_entries", []) if keep(entry)] + all_entries = [entry for entry in payload.get("all_entries", []) if keep(entry)] + + status_counts = {"gaps": 0, "clean": 0, "failed": 0} + for entry in all_entries: + status = entry.get("status") + if status in status_counts: + status_counts[status] += 1 + + return { + "generated_at": payload["generated_at"], + "github_repo": payload["github_repo"], + "summary": { + "total_entries": len(all_entries), + "with_gaps": status_counts["gaps"], + "clean": status_counts["clean"], + "failed": status_counts["failed"], + }, + "entries_with_gaps": entries_with_gaps, + "failed_entries": failed_entries, + "all_entries": all_entries, + } + + +def load_previous_report(latest_json: Path) -> dict[str, Any] | None: + """Load the previous latest.json report snapshot, if it exists.""" + if not latest_json.is_file(): + return None + try: + payload = json.loads(latest_json.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise PrCheckerError(f"invalid report file {latest_json}: {exc.msg}") from exc + if not isinstance(payload, dict): + raise PrCheckerError(f"invalid report file {latest_json}: expected object at top level") + return payload + + def build_report_payload( state: dict[str, Any], *, @@ -827,27 +926,37 @@ def _markdown_escape_cell(value: str) -> str: return value.replace("|", "\\|").replace("\n", " ") -def render_report_markdown(report: dict[str, Any]) -> str: +def render_report_markdown(report: dict[str, Any], *, delta: bool = False) -> str: """Render a human-readable Markdown summary.""" + title = ( + "# LLVM PR coverage gap report — new checks" + if delta + else "# LLVM PR coverage gap report" + ) lines = [ - "# LLVM PR coverage gap report", + title, "", f"Generated: {report['generated_at']}", f"Repository: {report['github_repo']}", ] summary = report["summary"] - lines.extend( - [ - ( - "Entries: " - f"{summary['total_entries']} total, " - f"{summary['with_gaps']} with gaps, " - f"{summary['clean']} clean, " - f"{summary['failed']} failed" - ), - "", - ] - ) + if delta: + summary_line = ( + "New or updated since last report: " + f"{summary['total_entries']} PR/backend pair(s) " + f"({summary['with_gaps']} with gaps, " + f"{summary['clean']} clean, " + f"{summary['failed']} failed)" + ) + else: + summary_line = ( + "Entries: " + f"{summary['total_entries']} total, " + f"{summary['with_gaps']} with gaps, " + f"{summary['clean']} clean, " + f"{summary['failed']} failed" + ) + lines.extend([summary_line, ""]) entries_with_gaps = report.get("entries_with_gaps", []) if entries_with_gaps: @@ -898,7 +1007,12 @@ def render_report_markdown(report: dict[str, Any]) -> str: lines.append("") if not entries_with_gaps and not failed_entries: - lines.append("No PRs with coverage gaps or failed checks in the current state.") + if delta: + lines.append( + "No PRs with coverage gaps or failed checks since the last report." + ) + else: + lines.append("No PRs with coverage gaps or failed checks in the current state.") lines.append("") return "\n".join(lines) @@ -910,15 +1024,30 @@ def write_reports( *, write_run_snapshot: bool = True, ) -> dict[str, Path]: - """Write latest.json, latest.md, and an optional timestamped snapshot.""" + """Write latest.json, latest.md, new-prs.md, and an optional timestamped snapshot.""" report_dir.mkdir(parents=True, exist_ok=True) latest_json = report_dir / "latest.json" latest_md = report_dir / "latest.md" + new_prs_md = report_dir / "new-prs.md" + + previous_payload = load_previous_report(latest_json) + changed_keys = diff_report_entries(payload, previous_payload) + delta_payload = filter_report_payload(payload, changed_keys) + log.info( + "Report delta: %d new or updated PR/backend pair(s) since last report", + len(changed_keys), + ) + latest_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") latest_md.write_text(render_report_markdown(payload) + "\n", encoding="utf-8") + new_prs_md.write_text(render_report_markdown(delta_payload, delta=True) + "\n", encoding="utf-8") - written = {"latest_json": latest_json, "latest_md": latest_md} + written = { + "latest_json": latest_json, + "latest_md": latest_md, + "new_prs_md": new_prs_md, + } if write_run_snapshot: runs_dir = report_dir / "runs" runs_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index b6083d2b..62e12017 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -19,8 +19,10 @@ build_report_payload, count_csv_data_rows, count_lit_failures, + diff_report_entries, entry_key, evaluate_output_dir, + filter_report_payload, load_gap_rows, load_state, plan_work, @@ -236,8 +238,161 @@ def test_build_render_and_write_reports(self) -> None: written = write_reports(report_dir, payload, write_run_snapshot=True) self.assertTrue(written["latest_json"].is_file()) self.assertTrue(written["latest_md"].is_file()) + self.assertTrue(written["new_prs_md"].is_file()) self.assertTrue(written["run_snapshot"].is_file()) + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn("new checks", new_prs_text) + self.assertIn("#123", new_prs_text) + + +class ReportDeltaTest(unittest.TestCase): + def _sample_entry( + self, + *, + pr_number: int = 123, + backend: str = "amdgpu", + checked_at: str = "2026-07-29T18:00:00+00:00", + head_sha: str = "abc123def456", + status: str = "gaps", + gap_count: int = 1, + ) -> dict[str, object]: + return { + "key": entry_key(pr_number, backend), + "pr_number": pr_number, + "backend": backend, + "title": "Fix foo", + "head_sha": head_sha, + "status": status, + "gap_count": gap_count, + "lit_failure_count": 0, + "checked_at": checked_at, + "output_dir": "/tmp/out", + "error": None, + "pr_url": f"https://github.com/llvm/llvm-project/pull/{pr_number}", + "sample_gaps": [], + } + + def _sample_payload(self, entries: list[dict[str, object]]) -> dict[str, object]: + with_gaps = [entry for entry in entries if entry["status"] == "gaps"] + failed = [entry for entry in entries if entry["status"] == "failed"] + status_counts = {"gaps": 0, "clean": 0, "failed": 0} + for entry in entries: + status_counts[str(entry["status"])] += 1 + return { + "generated_at": "2026-08-05T08:00:00+00:00", + "github_repo": "llvm/llvm-project", + "summary": { + "total_entries": len(entries), + "with_gaps": status_counts["gaps"], + "clean": status_counts["clean"], + "failed": status_counts["failed"], + }, + "entries_with_gaps": with_gaps, + "failed_entries": failed, + "all_entries": entries, + } + + def test_diff_detects_new_and_changed_entries(self) -> None: + previous = self._sample_payload( + [ + self._sample_entry(pr_number=1, checked_at="2026-01-01T00:00:00+00:00"), + self._sample_entry( + pr_number=2, + backend="spirv", + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + ] + ) + current = self._sample_payload( + [ + self._sample_entry( + pr_number=1, + checked_at="2026-01-02T00:00:00+00:00", + head_sha="updated-sha", + ), + self._sample_entry( + pr_number=2, + backend="spirv", + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + self._sample_entry(pr_number=3, checked_at="2026-01-02T00:00:00+00:00"), + ] + ) + + changed = diff_report_entries(current, previous) + self.assertEqual(changed, {"1:amdgpu", "3:amdgpu"}) + + def test_diff_without_previous_treats_all_entries_as_new(self) -> None: + current = self._sample_payload([self._sample_entry()]) + self.assertEqual(diff_report_entries(current, None), {"123:amdgpu"}) + + def test_filter_report_payload_recomputes_summary(self) -> None: + payload = self._sample_payload( + [ + self._sample_entry(pr_number=1), + self._sample_entry( + pr_number=2, + status="clean", + gap_count=0, + checked_at="2026-01-01T00:00:00+00:00", + ), + ] + ) + filtered = filter_report_payload(payload, {"1:amdgpu"}) + self.assertEqual(filtered["summary"]["total_entries"], 1) + self.assertEqual(filtered["summary"]["with_gaps"], 1) + self.assertEqual(len(filtered["all_entries"]), 1) + + def test_write_reports_emits_delta_against_previous_latest_json(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report_dir = Path(tmp) / "reports" + previous = self._sample_payload([self._sample_entry(pr_number=1)]) + current = self._sample_payload( + [ + self._sample_entry(pr_number=1), + self._sample_entry( + pr_number=2, + status="clean", + gap_count=0, + checked_at="2026-08-05T09:00:00+00:00", + ), + ] + ) + + write_reports(report_dir, previous, write_run_snapshot=False) + written = write_reports(report_dir, current, write_run_snapshot=False) + + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn("New or updated since last report: 1 PR/backend pair(s)", new_prs_text) + self.assertIn("1 clean", new_prs_text) + self.assertNotIn("#1", new_prs_text) + + def test_write_reports_with_no_changes_writes_empty_delta(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report_dir = Path(tmp) / "reports" + payload = self._sample_payload([self._sample_entry()]) + + write_reports(report_dir, payload, write_run_snapshot=False) + written = write_reports(report_dir, payload, write_run_snapshot=False) + + new_prs_text = written["new_prs_md"].read_text(encoding="utf-8") + self.assertIn( + "No PRs with coverage gaps or failed checks since the last report.", + new_prs_text, + ) + self.assertIn("0 PR/backend pair(s)", new_prs_text) + + def test_render_report_markdown_delta_mode(self) -> None: + payload = self._sample_payload([self._sample_entry()]) + markdown = render_report_markdown(payload, delta=True) + self.assertIn("# LLVM PR coverage gap report — new checks", markdown) + self.assertIn("New or updated since last report:", markdown) + class DiscoverPrsTest(unittest.TestCase): def test_backend_search_terms_include_label_and_age_filter_for_amdgpu(self) -> None: From 62ff708b2aec28f90228eba466318cae507f9ce6 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 5 Aug 2026 05:44:28 -0500 Subject: [PATCH 24/26] Use backend label for SPIRV PR searches --- scripts/pr-check/README.md | 2 +- src/pr_check/checker.py | 2 +- tests/test_pr_checker.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index e5bf64aa..9ca7de88 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -58,7 +58,7 @@ Daily cron entry point (AMDGPU only, drains the full queue): ## What each run does -1. **Discover** — `gh search prs` for open PRs touching `llvm/lib/Target/AMDGPU` and/or `llvm/lib/Target/SPIRV` +1. **Discover** — `gh search prs` for open PRs with label `backend:AMDGPU` and/or `backend:SPIRV`, then keep only those with changed files under `llvm/lib/Target/AMDGPU/` or `llvm/lib/Target/SPIRV/` 2. **Plan** — compare against `state.json`; queue entries that are new or have a changed head SHA 3. **Check** — for each planned `(PR, backend)` pair (up to `PR_CHECK_MAX_PER_RUN`): - Build Docker image `fuzz-fill-test:llvm-pr--` via [`build-image-pr.sh`](../docker/build-image-pr.sh) diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py index 033e3f52..462462af 100644 --- a/src/pr_check/checker.py +++ b/src/pr_check/checker.py @@ -29,7 +29,7 @@ 'label:"backend:AMDGPU"', ], "spirv": [ - 'path:"llvm/lib/Target/SPIRV"', + 'label:"backend:SPIRV"', ], } diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index 62e12017..21fd910d 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -403,6 +403,14 @@ def test_backend_search_terms_include_label_and_age_filter_for_amdgpu(self) -> N self.assertIn('label:"backend:AMDGPU"', terms[0]) self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') + def test_backend_search_terms_include_label_and_age_filter_for_spirv(self) -> None: + from pr_check.checker import _backend_search_terms + + terms = _backend_search_terms("spirv", max_age_days=30) + self.assertEqual(len(terms), 1) + self.assertIn('label:"backend:SPIRV"', terms[0]) + self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') + def test_pr_touches_backend_path_matches_target_prefix(self) -> None: from pr_check.checker import _pr_touches_backend_path From fb1a4f1326914b8fd7a9ea5af6af71ca5afd0438 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Wed, 5 Aug 2026 05:46:41 -0500 Subject: [PATCH 25/26] Fix SPIRV backend label typo --- src/pr_check/checker.py | 2 +- tests/test_pr_checker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py index 462462af..720b0499 100644 --- a/src/pr_check/checker.py +++ b/src/pr_check/checker.py @@ -29,7 +29,7 @@ 'label:"backend:AMDGPU"', ], "spirv": [ - 'label:"backend:SPIRV"', + 'label:"backend:SPIR-V"', ], } diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index 21fd910d..a4cc3f1f 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -408,7 +408,7 @@ def test_backend_search_terms_include_label_and_age_filter_for_spirv(self) -> No terms = _backend_search_terms("spirv", max_age_days=30) self.assertEqual(len(terms), 1) - self.assertIn('label:"backend:SPIRV"', terms[0]) + self.assertIn('label:"backend:SPIR-V"', terms[0]) self.assertRegex(terms[0], r'created:>\d{4}-\d{2}-\d{2}') def test_pr_touches_backend_path_matches_target_prefix(self) -> None: From eb05cf02e6d28b55cf6a8b532a064241a6e356a5 Mon Sep 17 00:00:00 2001 From: agorzyns Date: Thu, 6 Aug 2026 02:30:36 -0500 Subject: [PATCH 26/26] Rebuild state.json from artifacts when run exits badly --- scripts/pr-check/README.md | 22 ++ scripts/pr-check/rebuild-state.sh | 118 ++++++++++ src/pr_check/checker.py | 348 ++++++++++++++++++++++++++++++ tests/test_pr_checker.py | 85 ++++++++ 4 files changed, 573 insertions(+) create mode 100755 scripts/pr-check/rebuild-state.sh diff --git a/scripts/pr-check/README.md b/scripts/pr-check/README.md index 9ca7de88..8290fb08 100644 --- a/scripts/pr-check/README.md +++ b/scripts/pr-check/README.md @@ -98,6 +98,28 @@ Reports are built from `state.json`, not by scanning `runs/`. After a failed or Use this when `latest.md` looks stale (e.g. fewer entries than `state.json`, or checks finished but the orchestrator exited early). The markdown body lists only PRs **with gaps**; clean PRs appear in the summary line and in `latest.json`. The same refresh also updates `new-prs.md` by diffing the current state against the previous `latest.json` — useful after an interrupted run where checks completed but report generation did not. +### Rebuilding `state.json` from run artifacts + +If `state.json` is missing or corrupted (for example after a disk-full write), rebuild it from completed directories under `data/pr-check/runs/` without re-running coverage checks: + +```bash +# Preview what would be written +./scripts/pr-check/rebuild-state.sh --dry-run + +# Rebuild state, then refresh reports +./scripts/pr-check/rebuild-state.sh +./scripts/pr-check/get-latest.sh +``` + +The rebuild: + +1. Scans `data/pr-check/runs/-/` and skips incomplete directories (no `commit_lines_report/target_lines_uncovered.csv`). +2. Recomputes `gap_count`, `lit_failure_count`, and `status` from on-disk artifacts. +3. Fills `title`, `head_sha`, and `checked_at` from `reports/latest.json` and `reports/runs/*.json` when available. +4. Falls back to `gh pr view` for any remaining runs (uses the **current** PR head SHA — if a PR was updated after the run, `--plan-only` may queue a re-check). + +Incomplete or empty run directories (for example a check interrupted before detection finished) are skipped and are not added to state. + The main orchestrator also refreshes reports at normal exit. An EXIT trap regenerates the report if at least one check ran but the process did not finish cleanly (Ctrl+C, OOM, or unexpected early exit during `--drain-queue`). ## Daily AMDGPU cron diff --git a/scripts/pr-check/rebuild-state.sh b/scripts/pr-check/rebuild-state.sh new file mode 100755 index 00000000..088ca228 --- /dev/null +++ b/scripts/pr-check/rebuild-state.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Rebuild data/pr-check/state.json from on-disk run artifacts. +# +# Scans data/pr-check/runs/-/, re-evaluates gap/LIT counts, and +# fills title/head_sha from saved reports (latest.json, runs/*.json) or GitHub. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +PR_CHECK_STATE_FILE="${PR_CHECK_STATE_FILE:-${REPO_ROOT}/data/pr-check/state.json}" +PR_CHECK_OUTPUT_ROOT="${PR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/data/pr-check/runs}" +PR_CHECK_REPORT_DIR="${PR_CHECK_REPORT_DIR:-${REPO_ROOT}/data/pr-check/reports}" +PR_CHECK_LOG_LEVEL="${PR_CHECK_LOG_LEVEL:-info}" +GITHUB_REPO="${GITHUB_REPO:-llvm/llvm-project}" + +CONFIG_FILE="${PR_CHECK_CONFIG:-${SCRIPT_DIR}/config.env}" + +usage() { + cat < Output state file (default: ${PR_CHECK_STATE_FILE}) + --output-root Run artifact root (default: ${PR_CHECK_OUTPUT_ROOT}) + --report-dir Report directory (default: ${PR_CHECK_REPORT_DIR}) + --config Load environment overrides from a file + --help, -h Show this help + +Examples: + $(basename "$0") --dry-run + $(basename "$0") + $(basename "$0") && ./scripts/pr-check/get-latest.sh +EOF +} + +load_config() { + if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck source=/dev/null + source "$CONFIG_FILE" + fi +} + +dry_run=0 +extra_args=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + dry_run=1 + extra_args+=(--dry-run) + shift + ;; + --no-fetch-pr-metadata) + extra_args+=(--no-fetch-pr-metadata) + shift + ;; + --state-file) + [[ $# -ge 2 ]] || { echo "error: --state-file requires a value" >&2; exit 2; } + PR_CHECK_STATE_FILE="$2" + shift 2 + ;; + --output-root) + [[ $# -ge 2 ]] || { echo "error: --output-root requires a value" >&2; exit 2; } + PR_CHECK_OUTPUT_ROOT="$2" + shift 2 + ;; + --report-dir) + [[ $# -ge 2 ]] || { echo "error: --report-dir requires a value" >&2; exit 2; } + PR_CHECK_REPORT_DIR="$2" + shift 2 + ;; + --config) + [[ $# -ge 2 ]] || { echo "error: --config requires a value" >&2; exit 2; } + CONFIG_FILE="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +load_config +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +require_command python3 +if [[ "${extra_args[*]}" != *"--no-fetch-pr-metadata"* ]]; then + require_command gh +fi + +mkdir -p "$(dirname "$PR_CHECK_STATE_FILE")" + +PYTHONPATH="${REPO_ROOT}/src" python3 -m pr_check rebuild-state \ + --state-file "$PR_CHECK_STATE_FILE" \ + --output-root "$PR_CHECK_OUTPUT_ROOT" \ + --report-dir "$PR_CHECK_REPORT_DIR" \ + --github-repo "$GITHUB_REPO" \ + --log-level "$PR_CHECK_LOG_LEVEL" \ + "${extra_args[@]}" diff --git a/src/pr_check/checker.py b/src/pr_check/checker.py index 720b0499..a221f5fb 100644 --- a/src/pr_check/checker.py +++ b/src/pr_check/checker.py @@ -713,6 +713,248 @@ def evaluate_output_dir(output_dir: Path) -> dict[str, Any]: } +_STATE_ENTRY_FIELDS = ( + "pr_number", + "backend", + "title", + "head_sha", + "status", + "gap_count", + "lit_failure_count", + "checked_at", + "output_dir", + "error", +) + + +def parse_run_dir_name(name: str) -> tuple[int, str]: + """Parse a run directory name ``-``.""" + if "-" not in name: + raise PrCheckerError(f"invalid run directory name (expected -): {name!r}") + backend = name.rsplit("-", 1)[1] + if backend not in BACKEND_SEARCH_QUERIES: + raise PrCheckerError( + f"invalid run directory name {name!r}: unknown backend {backend!r}" + ) + try: + pr_number = int(name[: -(len(backend) + 1)]) + except ValueError as exc: + raise PrCheckerError(f"invalid run directory name (bad PR number): {name!r}") from exc + if pr_number <= 0: + raise PrCheckerError(f"invalid run directory name (bad PR number): {name!r}") + return pr_number, backend + + +def is_evaluable_run(output_dir: Path) -> bool: + """Return True when *output_dir* has the artifacts produced by a completed check.""" + return (output_dir / "commit_lines_report" / "target_lines_uncovered.csv").is_file() + + +def output_dir_checked_at(output_dir: Path) -> str: + """Approximate check completion time from the newest artifact under *output_dir*.""" + newest_mtime = output_dir.stat().st_mtime + for path in output_dir.rglob("*"): + if path.is_file(): + newest_mtime = max(newest_mtime, path.stat().st_mtime) + return datetime.fromtimestamp(newest_mtime, tz=timezone.utc).replace(microsecond=0).isoformat() + + +def _report_entry_to_state_entry(entry: dict[str, Any]) -> dict[str, Any]: + return {field: entry.get(field) for field in _STATE_ENTRY_FIELDS} + + +def _checked_at_sort_key(checked_at: str) -> str: + return checked_at or "" + + +def load_report_entry_index(report_dir: Path | None) -> dict[str, dict[str, Any]]: + """Load PR/backend metadata from ``latest.json`` and ``runs/*.json`` report snapshots.""" + if report_dir is None: + return {} + + candidates: list[Path] = [] + latest_json = report_dir / "latest.json" + if latest_json.is_file(): + candidates.append(latest_json) + runs_dir = report_dir / "runs" + if runs_dir.is_dir(): + candidates.extend(sorted(runs_dir.glob("*.json"))) + + indexed: dict[str, dict[str, Any]] = {} + for path in candidates: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + log.warning("Skipping invalid report file %s: %s", path, exc.msg) + continue + if not isinstance(payload, dict): + log.warning("Skipping invalid report file %s: expected object at top level", path) + continue + + for raw in payload.get("all_entries", []): + if not isinstance(raw, dict): + continue + try: + key = _entry_key_from_record(raw) + state_entry = _report_entry_to_state_entry(raw) + except (KeyError, TypeError, ValueError): + log.warning("Skipping malformed report entry in %s", path) + continue + + existing = indexed.get(key) + if existing is None or _checked_at_sort_key(state_entry["checked_at"]) >= _checked_at_sort_key( + existing.get("checked_at", "") + ): + indexed[key] = state_entry + + log.info( + "Loaded metadata for %d PR/backend pair(s) from report files under %s", + len(indexed), + report_dir, + ) + return indexed + + +def fetch_pr_metadata(pr_number: int, *, github_repo: str) -> tuple[str, str]: + """Return ``(title, head_sha)`` for an open PR via ``gh pr view``.""" + payload = _view_pr(pr_number, github_repo) + head_sha = payload.get("headRefOid") + if not head_sha: + raise PrCheckerError(f"could not resolve headRefOid for {github_repo}#{pr_number}") + title = str(payload.get("title") or "") + return title, str(head_sha) + + +def rebuild_state_from_runs( + *, + output_root: Path, + github_repo: str = DEFAULT_GITHUB_REPO, + report_dir: Path | None = None, + existing_state: dict[str, Any] | None = None, + fetch_missing_pr_metadata: bool = True, +) -> tuple[dict[str, Any], dict[str, int]]: + """Rebuild ``state.json`` entries from on-disk run artifacts. + + Completed runs under *output_root* are re-evaluated with + :func:`evaluate_output_dir`. ``title``, ``head_sha``, and ``checked_at`` are + taken from prior report snapshots when available; remaining PRs can optionally + be resolved via ``gh pr view`` (current head SHA — see README caveat). + """ + if not output_root.is_dir(): + raise PrCheckerError(f"output root is not a directory: {output_root}") + + entries: dict[str, dict[str, Any]] = {} + if existing_state is not None: + for key, raw in existing_state.get("entries", {}).items(): + if isinstance(raw, dict): + entries[key] = dict(raw) + + report_index = load_report_entry_index(report_dir) + stats = { + "runs_seen": 0, + "runs_evaluated": 0, + "runs_skipped_incomplete": 0, + "metadata_from_reports": 0, + "metadata_from_existing_state": 0, + "metadata_from_github": 0, + "metadata_missing": 0, + } + + for run_dir in sorted(output_root.iterdir()): + if not run_dir.is_dir(): + continue + stats["runs_seen"] += 1 + if not is_evaluable_run(run_dir): + stats["runs_skipped_incomplete"] += 1 + log.info("Skipping incomplete run directory (no gap report): %s", run_dir.name) + continue + + pr_number, backend = parse_run_dir_name(run_dir.name) + key = entry_key(pr_number, backend) + evaluation = evaluate_output_dir(run_dir) + output_dir = str(run_dir.resolve()) + + entry: dict[str, Any] = { + "pr_number": pr_number, + "backend": backend, + "title": "", + "head_sha": "", + "status": evaluation["status"], + "gap_count": evaluation["gap_count"], + "lit_failure_count": evaluation["lit_failure_count"], + "checked_at": output_dir_checked_at(run_dir), + "output_dir": output_dir, + "error": None, + } + + metadata_source = report_index.get(key) + metadata_origin: str | None = None + if metadata_source is not None: + metadata_origin = "reports" + elif key in entries: + metadata_source = entries[key] + metadata_origin = "existing_state" + + if metadata_source is not None: + entry["title"] = str(metadata_source.get("title") or "") + entry["head_sha"] = str(metadata_source.get("head_sha") or "") + if metadata_source.get("checked_at"): + entry["checked_at"] = str(metadata_source["checked_at"]) + if metadata_source.get("status") == "failed": + entry["status"] = "failed" + entry["error"] = metadata_source.get("error") + if metadata_source.get("output_dir"): + entry["output_dir"] = str(metadata_source["output_dir"]) + if metadata_origin == "reports": + stats["metadata_from_reports"] += 1 + elif metadata_origin == "existing_state": + stats["metadata_from_existing_state"] += 1 + + if fetch_missing_pr_metadata and not entry["head_sha"]: + try: + title, head_sha = fetch_pr_metadata(pr_number, github_repo=github_repo) + except PrCheckerError as exc: + stats["metadata_missing"] += 1 + log.warning( + "Skipping %s: could not resolve PR metadata (%s)", + run_dir.name, + exc, + ) + continue + entry["title"] = title + entry["head_sha"] = head_sha + stats["metadata_from_github"] += 1 + log.info( + "Resolved PR metadata from GitHub for #%d (%s): head=%s", + pr_number, + backend, + head_sha[:12], + ) + elif not entry["head_sha"]: + stats["metadata_missing"] += 1 + log.warning( + "Skipping %s: missing head_sha (pass --fetch-pr-metadata or provide report snapshots)", + run_dir.name, + ) + continue + + entries[key] = entry + stats["runs_evaluated"] += 1 + + state = {"version": STATE_VERSION, "entries": entries} + log.info( + "Rebuild summary: %d run dir(s) seen, %d evaluated, %d incomplete skipped, " + "%d metadata from reports, %d from GitHub, %d missing", + stats["runs_seen"], + stats["runs_evaluated"], + stats["runs_skipped_incomplete"], + stats["metadata_from_reports"], + stats["metadata_from_github"], + stats["metadata_missing"], + ) + return state, stats + + def pr_url(github_repo: str, pr_number: int) -> str: return f"https://github.com/{github_repo}/pull/{pr_number}" @@ -1116,6 +1358,58 @@ def cmd_record(args: argparse.Namespace) -> int: return 0 +def cmd_rebuild_state(args: argparse.Namespace) -> int: + existing_state: dict[str, Any] | None = None + if args.merge_existing_state and args.state_file.is_file(): + try: + existing_state = load_state(args.state_file) + except PrCheckerError as exc: + log.warning("Ignoring invalid existing state file %s: %s", args.state_file, exc) + + report_dir = args.report_dir + if report_dir is None: + report_dir = args.state_file.parent / "reports" + + state, stats = rebuild_state_from_runs( + output_root=args.output_root, + github_repo=args.github_repo, + report_dir=report_dir, + existing_state=existing_state, + fetch_missing_pr_metadata=args.fetch_pr_metadata, + ) + + if args.dry_run: + print( + json.dumps( + { + "dry_run": True, + "entry_count": len(state["entries"]), + "stats": stats, + }, + indent=2, + ) + ) + return 0 + + if args.state_file.is_file() and args.backup: + backup_path = args.state_file.with_suffix(args.state_file.suffix + ".bak") + backup_path.write_text(args.state_file.read_text(encoding="utf-8"), encoding="utf-8") + log.info("Backed up existing state to %s", backup_path) + + save_state(args.state_file, state) + print( + json.dumps( + { + "state_file": str(args.state_file), + "entry_count": len(state["entries"]), + "stats": stats, + }, + indent=2, + ) + ) + return 0 + + def cmd_evaluate_output(args: argparse.Namespace) -> int: log.info("Evaluating output directory: %s", args.output_dir) payload = evaluate_output_dir(args.output_dir) @@ -1266,6 +1560,60 @@ def build_parser() -> argparse.ArgumentParser: evaluate_parser.add_argument("--output-dir", type=Path, required=True) evaluate_parser.set_defaults(func=cmd_evaluate_output) + rebuild_parser = subparsers.add_parser( + "rebuild-state", + help="Rebuild state.json from run artifacts and saved reports", + parents=[logging_parent, github_repo_parent], + ) + rebuild_parser.add_argument( + "--state-file", + type=Path, + required=True, + help="Path to write rebuilt state.json", + ) + rebuild_parser.add_argument( + "--output-root", + type=Path, + required=True, + help="Directory containing per-run output folders (-)", + ) + rebuild_parser.add_argument( + "--report-dir", + type=Path, + default=None, + help=( + "Directory with latest.json and runs/*.json snapshots " + "(default: /reports)" + ), + ) + rebuild_parser.add_argument( + "--fetch-pr-metadata", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Resolve title/head_sha via gh for runs missing report metadata " + "(default: enabled)" + ), + ) + rebuild_parser.add_argument( + "--merge-existing-state", + action=argparse.BooleanOptionalAction, + default=True, + help="Reuse metadata from an existing state file when valid (default: enabled)", + ) + rebuild_parser.add_argument( + "--backup", + action=argparse.BooleanOptionalAction, + default=True, + help="Back up the existing state file to state.json.bak (default: enabled)", + ) + rebuild_parser.add_argument( + "--dry-run", + action="store_true", + help="Print rebuild stats without writing state.json", + ) + rebuild_parser.set_defaults(func=cmd_rebuild_state) + report_parser = subparsers.add_parser( "report", help="Write latest.json and latest.md from state", diff --git a/tests/test_pr_checker.py b/tests/test_pr_checker.py index a4cc3f1f..5cd71f83 100644 --- a/tests/test_pr_checker.py +++ b/tests/test_pr_checker.py @@ -506,5 +506,90 @@ def fake_search(backend: str, **kwargs: object) -> list[dict[str, object]]: self.assertEqual(discovered, []) +class RebuildStateTest(unittest.TestCase): + def _write_completed_run( + self, + root: Path, + *, + pr_number: int, + backend: str, + gap_rows: list[list[object]] | None = None, + ) -> Path: + run_dir = root / f"{pr_number}-{backend}" + gap_csv = run_dir / "commit_lines_report" / "target_lines_uncovered.csv" + rows: list[list[object]] = [["file", "line_no", "text"]] + if gap_rows: + rows.extend(gap_rows) + _write_gap_csv(gap_csv, rows) + lit_json = run_dir / "baseline" / "lit_failures.json" + lit_json.parent.mkdir(parents=True, exist_ok=True) + lit_json.write_text(json.dumps({"tests": []}), encoding="utf-8") + return run_dir + + def test_rebuild_state_from_completed_run(self) -> None: + from pr_check.checker import rebuild_state_from_runs + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runs = root / "runs" + reports = root / "reports" + self._write_completed_run( + runs, + pr_number=42, + backend="amdgpu", + gap_rows=[["llvm/lib/Target/AMDGPU/Foo.cpp", "10", "assert(x);"]], + ) + report_payload = { + "all_entries": [ + { + "key": "42:amdgpu", + "pr_number": 42, + "backend": "amdgpu", + "title": "AMDGPU fix", + "head_sha": "abc123def456", + "status": "gaps", + "gap_count": 1, + "lit_failure_count": 0, + "checked_at": "2026-01-01T00:00:00+00:00", + "output_dir": str(runs / "42-amdgpu"), + "error": None, + } + ] + } + reports.mkdir() + (reports / "latest.json").write_text(json.dumps(report_payload), encoding="utf-8") + + state, stats = rebuild_state_from_runs( + output_root=runs, + report_dir=reports, + fetch_missing_pr_metadata=False, + ) + + self.assertEqual(stats["runs_evaluated"], 1) + self.assertEqual(stats["metadata_from_reports"], 1) + entry = state["entries"]["42:amdgpu"] + self.assertEqual(entry["title"], "AMDGPU fix") + self.assertEqual(entry["head_sha"], "abc123def456") + self.assertEqual(entry["status"], "gaps") + self.assertEqual(entry["gap_count"], 1) + + def test_rebuild_state_skips_incomplete_run(self) -> None: + from pr_check.checker import rebuild_state_from_runs + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + runs = root / "runs" + incomplete = runs / "99-spirv" + incomplete.mkdir(parents=True) + + state, stats = rebuild_state_from_runs( + output_root=runs, + fetch_missing_pr_metadata=False, + ) + + self.assertEqual(state["entries"], {}) + self.assertEqual(stats["runs_skipped_incomplete"], 1) + + if __name__ == "__main__": unittest.main()