diff --git a/Dockerfile b/Dockerfile index f9a5fd4b..01333878 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,6 +151,7 @@ COPY --chown=${UID}:${GID} pyproject.toml LICENCE.txt README.md /work/fuzz-fill/ COPY --chown=${UID}:${GID} src /work/fuzz-fill/src COPY --chown=${UID}:${GID} tests /work/fuzz-fill/tests COPY --chown=${UID}:${GID} integration-tests /work/fuzz-fill/integration-tests +COPY --chown=${UID}:${GID} scripts /work/fuzz-fill/scripts RUN echo "=== fuzz-fill image LLVM source: $(cat /work/.llvm-source) ===" \ && echo "=== fuzz-fill image SanitizerCoverage allowlist: $(cat /work/.sancov-allowlist) ===" \ diff --git a/README.md b/README.md index c075fee1..e2561e5d 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,34 @@ # fuzz-fill -Fuzzing to fill test suite coverage gaps in LLVM. +Fuzzing to fill LLVM coverage gaps with fuzz-generated tests. -fuzz-fill supports two main workflows: +fuzz-fill has two phases: -1. **Find coverage gaps in the existing LLVM test suite and fill them with fuzz-generated tests** — measure what the suite already covers, run a fuzz corpus, and identify which tests hit lines the suite misses; then reduce those tests into minimal LIT cases. -2. **Find uncovered lines in a commit** — list source lines added by a patch that the regression suite still does not fully cover. +1. **Gap finding** — measure baseline coverage achieved by LLVM's LIT test suite and produce a list of uncovered lines. At present `fuzz-fill` supports finding gaps in the AMDGPU and SPIR-V backends; this will be extended in the futuer to other backends and parts of the LLVM codebase. + - **Baseline** — all uncovered lines in a user-specified part of the LLVM codebase. + - **PR** — added or changed lines in a commit that baseline coverage still misses. +2. **Gap filling** — run a fuzz corpus against that list and report which tests cover the gaps; then reduce promising tests into minimal IR modules. -See [here](#contributions) for a list of tests contributed to LLVM. +``` +┌─ Gap finding ────────────────────────────────────────────────┐ +│ │ +│ Baseline ──┐ │ +│ ├──► Uncovered lines │ +│ PR ────────┘ │ │ +└────────────────────────────────│─────────────────────────────┘ + │ +┌─ Gap filling ──────────────────│─────────────────────────────┐ +│ ▼ │ +│ Fuzz corpus ─────────► Identify tests ────────► New tests │ +│ that cover line gaps │ +└──────────────────────────────────────────────────────────────┘ +``` + +See [Contributions](#contributions) for tests contributed to LLVM. ## Quick start (Docker) -Try [Workflow 2](#workflow-2-uncovered-lines-in-a-commit), which reports lines added as part of a commit that the LLVM test suite does not cover. +Try [PR gap finding](#gap-finding-pr), which reports lines changed in a PR or commit that are not covered by any LIT tests. Results are saved in `/commit_lines_report/target_lines_uncovered.csv`. See [Gap finding (PR)](#gap-finding-pr) and [Docker test image](#docker-test-image) for more options. Prerequisites: - [Docker](https://docs.docker.com/) @@ -22,35 +39,33 @@ git clone https://github.com/ROCm/fuzz-fill.git cd fuzz-fill ``` -**LLVM pull request** — requires [GitHub CLI](https://cli.github.com/) (`gh`). Builds a PR image and runs detection in one step (first build compiles LLVM in Docker and can take a while): +**LLVM pull request** — requires [GitHub CLI](https://cli.github.com/) (`gh`). Builds a PR image, squashes all commits onto the PR's base commit, and finds all coverage gaps in that PR: ```bash -./scripts/docker/pr-cov-gaps-detection.sh \ +./scripts/docker/gap-finding-pr.sh \ --build-image \ --llvm-repo /path/to/llvm-project \ --pr-id 203468 \ --backend-tests amdgpu \ - --output-dir ./data/pr-cov-gaps-203468 \ + --output-dir ./data/gap-finding-pr-203468 \ -j "$(nproc)" ``` Use `spirv` instead of `amdgpu` for SPIR-V backend tests. -**Local commit** — build from your `llvm-project` and then run: +**Local commit** — build from your `llvm-project` checkout (the local checkout remains unchanged). This command only finds gaps in lines changed in a single commit rather than a full PR. Replace `HEAD` with a hash, branch, or `main~3` as needed: ```bash ./scripts/docker/build-image.sh --llvm-dir /path/to/llvm-project --allowlist amdgpu -j "$(nproc)" -./scripts/docker/pr-cov-gaps-detection.sh \ +./scripts/docker/gap-finding-pr.sh \ --image fuzz-fill-test:latest \ --output-dir ./data/my-commit \ --commit HEAD \ -j "$(nproc)" ``` -Replace `HEAD` with a hash, branch, or `main~3` as needed. - -**Result (both paths):** `/commit_lines_report/target_lines_uncovered.csv` — added source lines that are not covered by the test suite. See [Workflow 2](#workflow-2-uncovered-lines-in-a-commit) and [Docker test image](#docker-test-image) for more options. +To **fill** those gaps with fuzz tests, chain [gap filling](#gap-filling) after gap finding (see [Chaining gap finding and filling](#chaining-gap-finding-and-filling)). ## Table of Contents @@ -58,15 +73,23 @@ Replace `HEAD` with a hash, branch, or `main~3` as needed. - [Setup](#setup) - [Python environment](#python-environment) - [LLVM builds](#llvm-builds) -- [Workflow 1: Fill suite coverage gaps with fuzz-generated tests](#workflow-1-fill-suite-coverage-gaps-with-fuzz-generated-tests) -- [Workflow 2: Uncovered lines in a commit](#workflow-2-uncovered-lines-in-a-commit) +- [Gap finding (baseline)](#gap-finding-baseline) +- [Gap finding (PR)](#gap-finding-pr) +- [Gap filling](#gap-filling) +- [Gap reducing](#gap-reducing) +- [Chaining gap finding, filling, and reducing](#chaining-gap-finding-filling-and-reducing) - [Reduce interesting tests](#reduce-interesting-tests) - [CLI reference](#cli-reference) + - [Uncovered-lines CSV contract](#uncovered-lines-csv-contract) - [Environment variables](#environment-variables) - [Docker test image](#docker-test-image) - [Build](#build) - [Build from an LLVM pull request](#build-from-an-llvm-pull-request) - - [Workflow 2: PR coverage gap detection](#workflow-2-pr-coverage-gap-detection) + - [PR image build and reuse](#pr-image-build-and-reuse) + - [Gap finding (baseline) in Docker](#gap-finding-baseline-in-docker) + - [Gap finding (PR) in Docker](#gap-finding-pr-in-docker) + - [Gap filling in Docker](#gap-filling-in-docker) + - [Gap reducing in Docker](#gap-reducing-in-docker) - [Run integration tests](#run-integration-tests) - [Run a container](#run-a-container) - [Tests](#tests) @@ -84,7 +107,7 @@ source venv/bin/activate pip install -e . ``` -### LLVM build +### LLVM builds You need an official **LLVM GitHub release** as bootstrap and one **SanitizerCoverage** build of llvm-project at the matching tag: @@ -107,100 +130,90 @@ Adjust these to match your trees before running. --- -## Workflow 1: Fill suite coverage gaps with fuzz-generated tests +## Gap finding (baseline) -**When to use this:** you want to improve LLVM test coverage in a target area (e.g. AMDGPU `CodeGen`) by finding lines the regression suite does not hit, then checking whether fuzz-generated tests can cover those gaps. +**When to use this:** you want a list of source lines that baseline LIT coverage does not fully hit in a target area (e.g. AMDGPU `CodeGen`). -**Reference script:** [`scripts/test_coverage.sh`](scripts/test_coverage.sh) +**Reference script:** [`scripts/gap-finding-baseline.sh`](scripts/gap-finding-baseline.sh) ### What it does ```text -baseline → candidate-test → incremental → reduce -(suite baseline) (fuzz corpus) (gaps filled) (minimal LIT tests) +baseline → line_coverage_uncovered.csv +(LIT run) (+ llc_address_line_map.csv) ``` -1. **`baseline`** — run a filtered slice of the LLVM LIT suite with SanitizerCoverage to establish baseline coverage: which source lines the existing tests already hit. -2. **`candidate-test`** — run a directory of fuzz-generated tests (`.ll` / `.bc`) through instrumented `llc` and collect their coverage. -3. **`incremental`** — compare fuzz-test coverage against the suite baseline and report which fuzz tests cover lines the suite misses — these are candidate gap-fillers. A fuzz test qualifies for a line only when it fully covers that line and the line appears in the baseline `line_coverage_uncovered.csv`. -4. **`reduce`** — shrink promising tests into minimal cases suitable for adding to the suite (see [Reduce interesting tests](#reduce-interesting-tests) below). +**`coverage baseline`** — identify uncovered lines in run a filtered slice of LLVM instrumented with SanitizerCoverage. ### Configure and run -Edit the variables at the top of `scripts/test_coverage.sh`: +Edit the variables at the top of [`scripts/gap-finding-baseline.sh`](scripts/gap-finding-baseline.sh): | Variable | Meaning | |----------|---------| -| `LLVM` | Path to your `llvm-project` checkout | -| `LLVM_BIN` | Uninstrumented `bin` directory | +| `LLVM_REPO` | Path to your `llvm-project` checkout | +| `LLVM_BIN` | Uninstrumented `bin` directory (`sancov`) | | `INSTRUMENTED_BIN_DIR` | Instrumented `bin` directory | -| `OUTPUT_DIR` | Root for all artifacts from this workflow | -| `TESTS_DIR` | Directory of fuzz-generated `.ll` / `.bc` files to scan | -| `FILTER` | LIT directory prefix for baseline (default: `AMDGPU`; repeat on CLI with multiple `--lit-filter`) | +| `OUTPUT_DIR` | Root for artifacts | +| `FILTER` | LIT directory prefix (default: `AMDGPU`) | Then run from the fuzz-fill repo root: ```bash -./scripts/test_coverage.sh +./scripts/gap-finding-baseline.sh ``` -The default LIT filter is the tests in the `CodeGen/AMDGPU` directory. A different set of tests can be specified using multiple paths, for example see [`scripts/test_coverage_amdgpu_workflow1.sh`](scripts/test_coverage_amdgpu_workflow1.sh). - -By default the script runs only **`baseline`**. Uncomment the **`candidate-test`** and **`incremental`** blocks when you are ready for the full pipeline. +For multiple LIT prefixes and optional inline baseline during gap filling, see [`scripts/gap-filling-amdgpu.sh`](scripts/gap-filling-amdgpu.sh) (`SKIP_BASELINE=0` runs baseline as step 1). ### Key outputs -Under `$OUTPUT_DIR`: +Under `$OUTPUT_DIR/baseline/`: | Path | Contents | |------|----------| -| `baseline/line_coverage_summary.csv` | Per-line baseline coverage (joint llc + opt): `covered`, `partially`, or `uncovered` | -| `baseline/line_coverage_uncovered.csv` | Baseline lines with no suite coverage — input to `incremental` and `target-lines` | -| `baseline/llc_address_line_map.csv` | llc address-to-line map — input to `incremental` | -| `baseline/lit_failures.json` | Failed lit tests from the baseline run (llvm-lit `--report-failures-only` JSON: `name`, `code`, `output`, `elapsed`) | -| `baseline/processed_sancov/` | Merged, symbolized symcov files — debugging artifact from baseline | -| `candidate_tests/raw_sancov/` | Per-test raw sancov shards | -| `incremental/new_coverage.csv` | **Main result** — columns `test`, `file`, `line`, `covered-points`: fuzz tests that fill suite coverage gaps | - -`new_coverage.csv` is the input for testcase reduction in step 4. +| `line_coverage_summary.csv` | Per-line baseline coverage: `covered`, `partially`, or `uncovered` | +| `line_coverage_uncovered.csv` | **Main gap list** — input to `incremental` and `target-lines` | +| `llc_address_line_map.csv` | LLC address-to-line map — **required** for gap filling | +| `lit_failures.json` | Failed LIT tests (`name`, `code`, `output`, `elapsed`) | +| `processed_sancov/` | Merged symcov (debugging) | --- -## Workflow 2: Uncovered lines in a commit +## Gap finding (PR) -**When to use this:** you landed a patch and want a precise list of **added** source lines that the regression suite still does not fully cover. +**When to use this:** you landed a patch and want **added** source lines that baseline coverage still does not fully cover. -**Reference script:** [`scripts/test_coverage_commit_lines.sh`](scripts/test_coverage_commit_lines.sh) +**Reference script:** [`scripts/gap-finding-pr.sh`](scripts/gap-finding-pr.sh) ### What it does ```text added-lines → baseline → target-lines -(from git) (baseline) (uncovered added lines) +(from git) (LIT run) (uncovered added lines) ``` 1. **`added-lines`** — parse `git show` for a commit and list every line added on the right-hand side of the diff. -2. **`baseline`** — same baseline coverage run as Workflow 1 (produces `line_coverage_uncovered.csv` and related CSVs). -3. **`target-lines`** — for each line in the target CSV, include it in the report when its `(file, line)` appears in `line_coverage_uncovered.csv` from the baseline run. +2. **`baseline`** — same baseline run as [gap finding (baseline)](#gap-finding-baseline). +3. **`target-lines`** — include each added line whose `(file, line)` appears in `line_coverage_uncovered.csv`. -Step 3 does **not** re-run LIT, so you can repeat it with different `added-lines.csv` inputs as long as the `baseline` symcov artifacts are still present. +Step 3 does **not** re-run LIT; you can repeat it with different `added-lines.csv` inputs while baseline artifacts remain valid. ### Configure and run -Edit the variables at the top of `scripts/test_coverage_commit_lines.sh`: +Edit the variables at the top of [`scripts/gap-finding-pr.sh`](scripts/gap-finding-pr.sh): | Variable | Meaning | |----------|---------| -| `LLVM` | `llvm-project` checkout (same tree `added-lines` diffs against) | -| `LLVM_BIN` / `INSTRUMENTED_BIN_DIR` | Same as Workflow 1 | +| `LLVM_REPO` | `llvm-project` checkout (same tree `added-lines` diffs against) | +| `LLVM_BIN` / `INSTRUMENTED_BIN_DIR` | Same as baseline gap finding | | `OUTPUT_DIR` | Root for all artifacts | -| `FILTER` | LIT directory prefix for the baseline run (default in commit-lines script: `CodeGen/SPIRV`) | +| `FILTER` | LIT directory prefix for baseline | | `COMMIT` | Revision to analyse (`HEAD`, a hash, `main~3`, …) | -Then run from the fuzz-fill repo root: +Then run: ```bash -./scripts/test_coverage_commit_lines.sh +./scripts/gap-finding-pr.sh ``` ### Key outputs @@ -210,18 +223,157 @@ Under `$OUTPUT_DIR`: | Path | Contents | |------|----------| | `added-lines/added-lines.csv` | Added lines from the commit (`path`, `line_no`, `text`) | -| `test_suite/line_coverage_uncovered.csv` | Baseline uncovered lines — **required by `target-lines`** | -| `baseline/lit_failures.json` | Failed lit tests from the baseline run (llvm-lit `--report-failures-only` JSON: `name`, `code`, `output`, `elapsed`) | -| `baseline/processed_sancov/` | Merged symcov (still produced for debugging; not read by `target-lines`) | -| `target_lines_report/target_lines_uncovered.csv` | **Main result** — added lines where every suite point on that line is off | +| `baseline/line_coverage_uncovered.csv` | Baseline uncovered lines | +| `baseline/llc_address_line_map.csv` | LLC map — **required** if you gap-fill PR targets | +| `baseline/lit_failures.json` | Failed LIT tests from baseline | +| `commit_lines_report/target_lines_uncovered.csv` | **Main result** — PR-added lines still uncovered (`file`, `line`, optional `text`; absolute paths) | + +--- + +## Gap filling + +**When to use this:** you have a gap list from baseline or PR gap finding and want to find fuzz-generated tests that cover those lines. + +**Reference script:** [`scripts/gap-filling-amdgpu.sh`](scripts/gap-filling-amdgpu.sh) + +### What it does + +```text +candidate-test → incremental +(fuzz corpus) (gaps filled) +``` + +1. **`candidate-test`** — run a directory of fuzz-generated tests (`.ll` / `.bc`) through instrumented `llc` and collect coverage. +2. **`incremental`** — report fuzz tests that fully cover lines in the gap list. A test qualifies only when the line appears in the uncovered-lines CSV passed to `incremental`. + +Gap filling stops at `incremental/new_coverage.csv`. Run [gap reducing](#gap-reducing) next to shrink one promising row into a minimal testcase. + +The local AMDGPU script can run baseline inline (step 1/3) or reuse an existing gap list (`SKIP_BASELINE=1`). The [Docker gap-filling runner](#gap-filling-in-docker) always requires explicit profile CSV paths. + +### Configure and run + +```bash +# Full local run (baseline + fill) +./scripts/gap-filling-amdgpu.sh + +# Reuse gap list from a prior baseline or PR gap-finding run +SKIP_BASELINE=1 OUTPUT_DIR=./data/my_run ./scripts/gap-filling-amdgpu.sh +``` + +Environment variables: `OUTPUT_DIR`, `JOBS`, `CORPUS_N`, `TESTS_DIR`, `REFRESH`, `SKIP_BASELINE`, `SKIP_CANDIDATE`, `SKIP_INCREMENTAL`. LIT filters come from [`scripts/lit-filters-amdgpu.sh`](scripts/lit-filters-amdgpu.sh). + +When `SKIP_BASELINE=1`, point `OUTPUT_DIR` at a tree that already has `baseline/line_coverage_uncovered.csv` and `baseline/llc_address_line_map.csv` (from baseline gap finding) or copy PR outputs into that layout (`target_lines_uncovered.csv` can replace `line_coverage_uncovered.csv` for `incremental` if you adjust paths in the script or pass CSVs explicitly via the Python CLI). + +### Key outputs + +Under `$OUTPUT_DIR`: + +| Path | Contents | +|------|----------| +| `candidate_tests/raw_sancov/` | Per-test raw sancov shards | +| `incremental/new_coverage.csv` | **Main result** — `test_name`, `file`, `line`, `covered-points` | + +--- + +## Gap reducing + +**When to use this:** you have `new_coverage.csv` from gap filling and want to reduce **one row** into a minimal testcase (example / smoke run, not a full batch). + +**Reference script:** [`scripts/gap-reducing-amdgpu.sh`](scripts/gap-reducing-amdgpu.sh) + +### What it does + +```text +new_coverage.csv + candidate_tests/ → reduce (one row) + → reduced/t-00001-*/ +``` + +Wraps [`scripts/batch_reduce_using_coverage.py`](scripts/batch_reduce_using_coverage.py) with `--n 1`. Default pipeline: **`llvm_reduce_ir`** only (fast). Append creduce with `WITH_CREDUCE=1` or `--pipeline llvm_reduce_ir,creduce`. + +### Configure and run + +```bash +# after gap-filling wrote ./data/my_run/incremental/new_coverage.csv +./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run + +# harness only (no llvm-reduce run) +./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run --prepare-only + +# second row, with creduce +./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run --row 2 +WITH_CREDUCE=1 ./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run +``` + +Requires under `--output-dir`: + +| Path | Role | +|------|------| +| `incremental/new_coverage.csv` | Input CSV (from gap filling) | +| `candidate_tests/` | Per-test dirs with `test.sh` (from gap filling) | +| `reduced/` | **Output** — one case dir per invocation | + +For batch reduction of many rows, use [`scripts/batch_reduce_using_coverage.sh`](scripts/batch_reduce_using_coverage.sh) directly. + +--- + +## Chaining gap finding, filling, and reducing + +Typical end-to-end flow: + +```text +gap finding (baseline or PR) → gap filling → reduce +``` + +**Baseline gaps → fill** (Docker): + +```bash +./scripts/docker/gap-finding-baseline.sh --output-dir ./data/baseline-run -j "$(nproc)" + +./scripts/docker/gap-filling.sh \ + --output-dir ./data/fill-100 \ + --line-coverage-uncovered-csv ./data/baseline-run/baseline/line_coverage_uncovered.csv \ + --llc-address-line-map-csv ./data/baseline-run/baseline/llc_address_line_map.csv \ + --candidate-tests-dir /path/to/irtests/bitcode/amdgpu/all \ + -n 100 -j "$(nproc)" +``` + +**PR gaps → fill** (Docker): + +```bash +./scripts/docker/gap-finding-pr.sh \ + --pr-id 203468 --output-dir ./data/gap-finding-pr-203468 -j "$(nproc)" + +./scripts/docker/gap-filling.sh \ + --pr-id 203468 \ + --output-dir ./data/pr-fill-100 \ + --line-coverage-uncovered-csv ./data/gap-finding-pr-203468/commit_lines_report/target_lines_uncovered.csv \ + --llc-address-line-map-csv ./data/gap-finding-pr-203468/baseline/llc_address_line_map.csv \ + --candidate-tests-dir /path/to/irtests/bitcode/amdgpu/all \ + -n 100 -j "$(nproc)" +``` + +The LLC map must come from the **same** baseline run (and LIT filters / Docker image) as the uncovered-lines CSV. + +**Candidate corpus:** pass an external path such as `irtests/bitcode/amdgpu/all` via `--candidate-tests-dir`. The directory is bind-mounted read-only at run time (not copied into the image). `-n` limits how many tests `candidate-test` processes. + +**Image selection:** gap-finding and gap-filling Docker runners accept `--pr-id` (PR LLVM image), `--image` (local or custom tag), or default `fuzz-fill-test:latest`. PR gap finding additionally supports `--build-image` + `--llvm-repo` + `--pr-id`; local commits use `build-image.sh` then `--image fuzz-fill-test:latest --commit `. + +**Reduce one row** after gap filling (same `--output-dir`): + +```bash +./scripts/docker/gap-reducing.sh --output-dir ./data/fill-100 +./scripts/docker/gap-reducing.sh --bind-repo --output-dir ./data/fill-100 --prepare-only +``` + +See [Gap reducing in Docker](#gap-reducing-in-docker). --- ## Reduce interesting tests -Once you have `new_coverage.csv` (Workflow 1) or a specific uncovered line you want to target (Workflow 2), use the **`reduce`** module to shrink a testcase while preserving coverage or crash behaviour. +Once you have `new_coverage.csv` from gap filling, use [gap reducing](#gap-reducing) for a single-row example, or the **`reduce`** module directly for custom configs. -Each row in `new_coverage.csv` maps a test file to a source location and one or more SanitizerCoverage point ids (`covered-points`). Turn a row into a reduction job by pointing a JSON config at the testcase, setting `file` / `line`, and wiring an interestingness script that checks the coverage address. +Each row in `new_coverage.csv` maps a test file to a source location and SanitizerCoverage point ids (`covered-points`). Turn a row into a reduction job with a JSON config, `file` / `line`, and an interestingness script that checks the coverage address. Examples: @@ -240,17 +392,40 @@ See `python -m reduce --help` and the checked-in `example/*/config.json` files f ## CLI reference -The workflows above call these modules. Use `--help` on any command for the full flag list. +The scripts above call these modules. Use `--help` on any command for the full flag list. -| Command | Role in workflows | -|---------|-------------------| -| `python -m coverage baseline` | Baseline LIT coverage (both workflows) | -| `python -m coverage candidate-test` | Coverage from a fuzz-generated test corpus (Workflow 1) | -| `python -m coverage incremental` | Suite gaps filled by fuzz tests (Workflow 1) | -| `python -m coverage target-lines` | Uncovered target lines vs `line_coverage_uncovered.csv` (Workflow 2) | -| `python -m added_lines` | Lines added by a git commit (Workflow 2) | +| Command | Role | +|---------|------| +| `python -m coverage baseline` | Baseline LIT coverage (gap finding) | +| `python -m coverage candidate-test` | Coverage from a fuzz corpus (gap filling) | +| `python -m coverage incremental` | Match fuzz tests against a gap list (gap filling) | +| `python -m coverage target-lines` | PR added lines vs `line_coverage_uncovered.csv` | +| `python -m added_lines` | Lines added by a git commit | | `python -m reduce` | Testcase reduction | +### Uncovered-lines CSV contract + +`coverage incremental` and PR gap finding both use a baseline uncovered-lines CSV with columns **`file`** and **`line`**. Paths are **absolute** and must match those in `llc_address_line_map.csv` (as produced by `coverage baseline`). + +| File | Role | +|------|------| +| `line_coverage_uncovered.csv` | Baseline gap list | +| `target_lines_uncovered.csv` | PR-added lines still uncovered; same `file`/`line` schema, optional `text` | + +PR gap finding input to `target-lines` remains `added-lines.csv` (`path`, `line_no`, `text` with git-relative paths). The **output** report uses the shared contract above. + +`coverage incremental` requires explicit CSV paths: + +```bash +python -m coverage incremental \ + --output-dir data/incremental \ + --line-coverage-uncovered-csv data/baseline/line_coverage_uncovered.csv \ + --llc-address-line-map-csv data/baseline/llc_address_line_map.csv \ + --candidate-tests-output-dir data/candidate_tests +``` + +For PR targets, pass `target_lines_uncovered.csv` as `--line-coverage-uncovered-csv` instead. + ### `coverage baseline` filters | Flag | Meaning | @@ -286,12 +461,11 @@ export FUZZ_FILL_LLC=/work/llvm-build-sancov/bin/llc export FUZZ_FILL_OPT=/work/llvm-build-sancov/bin/opt export FUZZ_FILL_LLVM_REPO=/work/llvm-project -python -m coverage baseline \ - --output-dir data/baseline +python -m coverage baseline --output-dir data/baseline python -m added_lines --commit HEAD ``` -By default, `coverage baseline` uses `--lit-filter AMDGPU` (all LIT tests whose path contains `AMDGPU`). For a faster CodeGen-only run: +CodeGen-only baseline: ```bash python -m coverage baseline \ @@ -299,26 +473,10 @@ python -m coverage baseline \ --lit-filter CodeGen/AMDGPU ``` -Multiple directory prefixes: - -```bash -python -m coverage baseline \ - --output-dir data/baseline-multi \ - --lit-filter CodeGen/AMDGPU \ - --lit-filter MC/AMDGPU \ - -j "$(nproc)" -``` - -Or via [`scripts/test_coverage.sh`](scripts/test_coverage.sh) (default `FILTER=AMDGPU`): +Or via the reference script: ```bash -./scripts/test_coverage.sh -``` - -CodeGen-only via script: - -```bash -FILTER=CodeGen/AMDGPU ./scripts/test_coverage.sh +FILTER=CodeGen/AMDGPU ./scripts/gap-finding-baseline.sh ``` Workflow shell scripts under `scripts/` may use their own names (`LLVM_BIN`, `INSTRUMENTED_BIN_DIR`, …); only the `FUZZ_FILL_*` variables are read by the Python CLIs. @@ -329,7 +487,9 @@ Workflow shell scripts under `scripts/` may use their own names (`LLVM_BIN`, `IN The Docker image bundles an official LLVM release bootstrap, a dual-build SanitizerCoverage LLVM tree (instrumented `llc`/`opt` plus Release helpers), and a fuzz-fill venv. Use it when you want to run integration tests or experiment without building LLVM locally. -**Scripts** (under [`scripts/docker/`](scripts/docker/)): [`build-image.sh`](scripts/docker/build-image.sh), [`build-image-pr.sh`](scripts/docker/build-image-pr.sh), [`pr-cov-gaps-detection.sh`](scripts/docker/pr-cov-gaps-detection.sh), [`test-image.sh`](scripts/docker/test-image.sh), [`tmp-container.sh`](scripts/docker/tmp-container.sh) +**Scripts** (under [`scripts/docker/`](scripts/docker/)): [`build-image.sh`](scripts/docker/build-image.sh), [`build-image-pr.sh`](scripts/docker/build-image-pr.sh), [`ensure-image.sh`](scripts/docker/ensure-image.sh), [`gap-finding-baseline.sh`](scripts/docker/gap-finding-baseline.sh), [`gap-finding-pr.sh`](scripts/docker/gap-finding-pr.sh), [`gap-filling.sh`](scripts/docker/gap-filling.sh), [`gap-reducing.sh`](scripts/docker/gap-reducing.sh), [`test-image.sh`](scripts/docker/test-image.sh), [`tmp-container.sh`](scripts/docker/tmp-container.sh) + +The image bakes a copy of fuzz-fill at `/work/fuzz-fill` when built. Pass **`--bind-repo`** on a docker runner to mount your local checkout over that path (venv stays at `/work/fuzz-fill-venv`) when you need code that is newer than the image. ### Build @@ -357,6 +517,8 @@ Examples: ./scripts/docker/build-image.sh -j "$(nproc)" ``` +Use this image with **`--image fuzz-fill-test:latest`** for local-commit gap finding (see [Quick start](#quick-start-docker)). + ### Build from an LLVM pull request [`scripts/docker/build-image-pr.sh`](scripts/docker/build-image-pr.sh) builds a Docker image from an LLVM PR. Pass a local `llvm-project` clone; the PR is squashed in a standalone fuzz-fill clone so your llvm checkout is unchanged. Requires local `gh` and Docker (BuildKit). PRs are assumed to live on **`llvm/llvm-project`** unless you pass `--github-repo`. @@ -373,36 +535,165 @@ Examples: | `--github-repo ` | GitHub repo hosting the PR (default: `llvm/llvm-project`) | | `-j `, `--jobs ` | Limit ninja parallelism for both LLVM builds (default: unconstrained) | -For the full coverage-gap workflow (build + detect), use [`scripts/docker/pr-cov-gaps-detection.sh --build-image`](#pr-coverage-gap-detection) instead. +For build + gap finding in one step, use [`gap-finding-pr.sh --build-image`](#gap-finding-pr-in-docker) instead. + +### PR image build and reuse + +Gap-finding and gap-filling Docker runners share [`ensure-image.sh`](scripts/docker/ensure-image.sh) for PR images tagged `fuzz-fill-test:llvm-pr-`: + +| Flag | Meaning | +|------|---------| +| `--build-image` | Build via `build-image-pr.sh` when the tag is missing | +| `--force-build` | Rebuild even when the tag already exists | +| `--keep-image` | Do not remove the image after a `--build-image` run (default: remove) | +| `--pr-id ` | Select `fuzz-fill-test:llvm-pr-` | +| `--llvm-repo ` | Required with `--build-image` | +| `--backend-tests amdgpu\|spirv` | Required with `--build-image` | + +Build once, then reuse on later runs (omit `--build-image`): + +```bash +./scripts/docker/gap-finding-pr.sh \ + --build-image --keep-image \ + --llvm-repo /path/llvm-project --pr-id 203468 \ + --backend-tests amdgpu --output-dir ./data/gap-finding-pr-203468 -j "$(nproc)" + +./scripts/docker/gap-finding-pr.sh \ + --pr-id 203468 --output-dir ./data/gap-finding-pr-203468 +``` -### Workflow 2: PR coverage gap detection +### Gap finding (baseline) in Docker -[`scripts/docker/pr-cov-gaps-detection.sh`](scripts/docker/pr-cov-gaps-detection.sh) runs Workflow 2 in Docker (baseline → `added_lines` → `target-lines`). Use `--build-image` to build the PR image and run detection in one step. For AMDGPU images, the baseline defaults to the twelve LIT prefixes in [`scripts/lit-filters-amdgpu.sh`](scripts/lit-filters-amdgpu.sh) (same as [`scripts/test_coverage_amdgpu_workflow1.sh`](scripts/test_coverage_amdgpu_workflow1.sh)); SPIRV defaults to `CodeGen/SPIRV`. Override with one or more `--lit-filter` directory prefixes. +[`scripts/docker/gap-finding-baseline.sh`](scripts/docker/gap-finding-baseline.sh) runs `coverage baseline` in a container. Output: `/baseline/`. ```bash -./scripts/docker/pr-cov-gaps-detection.sh \ +./scripts/docker/gap-finding-baseline.sh \ + --output-dir ./data/baseline-run \ + -j "$(nproc)" +``` + +| Option | Meaning | +|--------|---------| +| `--output-dir ` | Host output directory (required) | +| `--image ` | Docker image (default: `fuzz-fill-test:latest`) | +| `--pr-id ` | PR image `fuzz-fill-test:llvm-pr-` | +| `--build-image` | Build PR image when missing (see [PR image build and reuse](#pr-image-build-and-reuse)) | +| `--bind-repo` | Mount local fuzz-fill checkout over `/work/fuzz-fill` | +| `--lit-filter ` | LIT filter override (default: from image `/work/.sancov-allowlist`) | +| `-j `, `--jobs ` | Parallel jobs for llvm-lit and ninja (when building) | + +### Gap finding (PR) in Docker + +[`scripts/docker/gap-finding-pr.sh`](scripts/docker/gap-finding-pr.sh) runs baseline → `added_lines` → `target-lines`. Requires **`--image` or `--pr-id`**. + +**GitHub PR** (build + run): + +```bash +./scripts/docker/gap-finding-pr.sh \ --build-image \ --llvm-repo /path/llvm-project \ --pr-id 203468 \ --backend-tests amdgpu \ - --output-dir /path/pr-cov-gaps-203468 \ + --output-dir ./data/gap-finding-pr-203468 \ + -j "$(nproc)" +``` + +**Local commit** (after [`build-image.sh`](#build)): + +```bash +./scripts/docker/gap-finding-pr.sh \ + --image fuzz-fill-test:latest \ + --output-dir ./data/my-commit \ + --commit HEAD \ -j "$(nproc)" ``` +For AMDGPU images, baseline defaults to the twelve LIT prefixes in [`scripts/lit-filters-amdgpu.sh`](scripts/lit-filters-amdgpu.sh) (same as [`scripts/gap-filling-amdgpu.sh`](scripts/gap-filling-amdgpu.sh)); SPIRV defaults to `CodeGen/SPIRV`. Override with one or more `--lit-filter` prefixes. + | Option | Meaning | |--------|---------| -| `--build-image` | Build PR image first via `scripts/docker/build-image-pr.sh` | +| `--image ` | Docker image (required unless `--pr-id`) | +| `--pr-id ` | PR image tag `llvm-pr-` (required unless `--image`) | +| `--commit ` | Revision for `added_lines` (default: `HEAD` in container llvm-project) | +| `--build-image` | Build PR image when missing | +| `--force-build` / `--keep-image` | Rebuild or retain PR image | | `--llvm-repo ` | Required with `--build-image` | | `--backend-tests amdgpu\|spirv` | Required with `--build-image` | -| `--pr-id ` | PR number (image tag `llvm-pr-`) | | `--output-dir ` | Host output directory | -| `-j `, `--jobs ` | Parallel jobs (ninja when building, llvm-lit when detecting) | -| `--lit-filter ` | LIT directory prefix; repeat for multiple (default: [`scripts/lit-filters-amdgpu.sh`](scripts/lit-filters-amdgpu.sh) for AMDGPU, `CodeGen/SPIRV` for SPIRV) | -| `--github-repo ` | Optional; default `llvm/llvm-project` when building | +| `-j `, `--jobs ` | Parallel jobs | +| `--lit-filter ` | LIT prefix; repeat for multiple | +| `--github-repo ` | When building (default: `llvm/llvm-project`) | + +Main output: `/commit_lines_report/target_lines_uncovered.csv`. + +### Gap filling in Docker + +[`scripts/docker/gap-filling.sh`](scripts/docker/gap-filling.sh) runs `candidate-test` → `incremental`. + +**Candidate tests are not baked into the image.** By default, `--candidate-tests-dir` is bind-mounted read-only from the host (e.g. an external `irtests` corpus). No host-side copy is made; `-n` limits how many `.ll`/`.bc` files `candidate-test` processes (sorted path order). Pass **`--stage-candidate-tests`** to copy the first N inputs into a temp dir before mounting instead (previous behaviour). + +**Requires** both profile CSV flags (gap list + LLC map from the same baseline run): + +```bash +./scripts/docker/gap-filling.sh \ + --output-dir ./data/fill-100 \ + --line-coverage-uncovered-csv ./data/baseline-run/baseline/line_coverage_uncovered.csv \ + --llc-address-line-map-csv ./data/baseline-run/baseline/llc_address_line_map.csv \ + --candidate-tests-dir /path/to/irtests/bitcode/amdgpu/all \ + -n 100 \ + -j "$(nproc)" +``` + +PR gap list (use `target_lines_uncovered.csv` as the uncovered-lines CSV): + +```bash +./scripts/docker/gap-filling.sh \ + --pr-id 203468 \ + --output-dir ./data/pr-fill-100 \ + --line-coverage-uncovered-csv ./data/gap-finding-pr-203468/commit_lines_report/target_lines_uncovered.csv \ + --llc-address-line-map-csv ./data/gap-finding-pr-203468/baseline/llc_address_line_map.csv \ + --candidate-tests-dir /path/to/irtests/bitcode/amdgpu/all \ + -n 100 -j "$(nproc)" +``` + +| Option | Meaning | +|--------|---------| +| `--output-dir ` | Host output directory (required) | +| `--line-coverage-uncovered-csv ` | Gap list CSV (required) | +| `--llc-address-line-map-csv ` | LLC map from same baseline (required) | +| `--candidate-tests-dir ` | Host corpus root, bind-mounted read-only (required) | +| `-n `, `--n ` | First N candidate tests to run (required) | +| `--stage-candidate-tests` | Copy first N inputs to a temp dir before mounting (default: bind-mount full dir) | +| `--image ` / `--pr-id ` | Docker image | +| `--build-image` | Build PR image when missing | +| `--bind-repo` | Mount local fuzz-fill checkout | +| `-j `, `--jobs ` | Parallel jobs | + +Main output: `/incremental/new_coverage.csv`. + +### Gap reducing in Docker + +[`scripts/docker/gap-reducing.sh`](scripts/docker/gap-reducing.sh) reduces **one row** from a prior gap-filling run. Mounts `--output-dir` at `/mounted-output/` (must already contain `incremental/new_coverage.csv` and `candidate_tests/`). + +```bash +./scripts/docker/gap-reducing.sh --output-dir ./data/fill-100 +./scripts/docker/gap-reducing.sh --pr-id 203468 --output-dir ./data/fill-100 --row 1 +./scripts/docker/gap-reducing.sh --bind-repo --output-dir ./data/fill-100 --prepare-only +``` + +| Option | Meaning | +|--------|---------| +| `--output-dir ` | Gap-fill output directory (required) | +| `--row ` | CSV row to reduce (default: 1) | +| `--prepare-only` | Create harness under `reduced/` without running reduce | +| `--pipeline ` | Reduce pipeline (default: `llvm_reduce_ir`) | +| `--with-creduce` | Append creduce to the pipeline | +| `--image ` / `--pr-id ` | Docker image (same as gap filling) | +| `--bind-repo` | Mount local fuzz-fill checkout | -If the image `fuzz-fill-test:llvm-pr-` already exists, omit `--build-image` to run detection only. +Default pipeline is `llvm_reduce_ir` only (no creduce). The image includes `llvm-reduce` and `creduce` ([`Dockerfile`](Dockerfile)). -Main output: `/commit_lines_report/target_lines_uncovered.csv`. See [Workflow 2](#workflow-2-uncovered-lines-in-a-commit) for report semantics. +Main output: `/reduced/t-00001-*/` (case harness; reduced artifacts under `reduced/` inside that dir when not using `--prepare-only`). ### Run integration tests diff --git a/integration-tests/coverage-pipeline.test b/integration-tests/coverage-pipeline.test index c817d139..57f74565 100644 --- a/integration-tests/coverage-pipeline.test +++ b/integration-tests/coverage-pipeline.test @@ -65,7 +65,7 @@ # COM: The same coverage gaps reported by incremental (GAP_CSV) must be reported as # COM: all-points-uncovered by target-lines when fed as target lines. # RUN: %FileCheck %s --input-file=%t/target_lines/target_lines_uncovered.csv --check-prefix=TARGET_CSV -# TARGET_CSV-DAG: file,line_no,text +# TARGET_CSV-DAG: file,line,text # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,211, # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,222, # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,581, diff --git a/integration-tests/gap-filling.test b/integration-tests/gap-filling.test new file mode 100644 index 00000000..7fca9ded --- /dev/null +++ b/integration-tests/gap-filling.test @@ -0,0 +1,41 @@ +# COM: Gap filling: candidate-test -> incremental using explicit profile CSVs +# COM: (mirrors Docker gap-filling.sh). Scenario A uses baseline gap list; scenario B +# COM: uses target_lines_uncovered.csv from target-lines as the PR gap list. + +# RUN: rm -rf %t && mkdir -p %t/profile %t/candidate_tests %t/incremental %t/pr_gaps %t/candidate_tests_pr %t/incremental_pr +# RUN: %coverage baseline --output-dir %t/profile --sancov %sancov --llvm-lit %llvm-lit --llc %sancov-llc --opt %sancov-opt --lit-filter CodeGen/AMDGPU/loop -j2 + +# COM: Scenario A — baseline gap fill +# RUN: %coverage candidate-test --output-dir %t/candidate_tests --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1 +# RUN: %coverage incremental --output-dir %t/incremental --sancov %sancov --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests + +# RUN: test -f %t/incremental/new_coverage.csv +# RUN: %FileCheck %s --input-file=%t/incremental/new_coverage.csv --check-prefix=GAP_CSV +# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,211, +# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,222, +# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,581, +# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,590, +# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,680, + +# RUN: %FileCheck %s --input-file=%t/incremental/new_coverage.csv --check-prefix=NO_BASELINE_GAIN +# NO_BASELINE_GAIN-NOT: /AMDGPUAlwaysInlinePass.cpp,160, + +# COM: Scenario B — PR gap fill (target_lines_uncovered.csv as gap list) +# RUN: %coverage target-lines --output-dir %t/pr_gaps --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llvm-repo %llvm-repo --target-lines-csv %S/fixtures/target-lines-sample.csv +# RUN: test -f %t/pr_gaps/target_lines_uncovered.csv +# RUN: %FileCheck %s --input-file=%t/pr_gaps/target_lines_uncovered.csv --check-prefix=PR_TARGET_CSV +# PR_TARGET_CSV-DAG: file,line,text + +# RUN: %coverage candidate-test --output-dir %t/candidate_tests_pr --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1 +# RUN: %coverage incremental --output-dir %t/incremental_pr --sancov %sancov --line-coverage-uncovered-csv %t/pr_gaps/target_lines_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests_pr + +# RUN: test -f %t/incremental_pr/new_coverage.csv +# RUN: %FileCheck %s --input-file=%t/incremental_pr/new_coverage.csv --check-prefix=PR_GAP_CSV +# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,211, +# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,222, +# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,581, +# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,590, +# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,680, + +# RUN: %FileCheck %s --input-file=%t/incremental_pr/new_coverage.csv --check-prefix=PR_GAP_CSV_NOT +# PR_GAP_CSV_NOT-NOT: /AMDGPUAliasAnalysis.cpp,103, diff --git a/integration-tests/gap-reducing.test b/integration-tests/gap-reducing.test new file mode 100644 index 00000000..853d801c --- /dev/null +++ b/integration-tests/gap-reducing.test @@ -0,0 +1,26 @@ +# COM: Gap reducing: build a single-row reduce harness from gap-filling output. +# COM: Omits --llc/--llvm-reduce (prepare-only): config.json, interesting_ir.sh, +# COM: and a copy of the candidate IR input. + +# RUN: rm -rf %t && mkdir -p %t/profile %t/candidate_tests %t/incremental %t/reduced +# RUN: %coverage baseline --output-dir %t/profile --sancov %sancov --llvm-lit %llvm-lit --llc %sancov-llc --opt %sancov-opt --lit-filter CodeGen/AMDGPU/loop -j2 +# RUN: %coverage candidate-test --output-dir %t/candidate_tests --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1 +# RUN: %coverage incremental --output-dir %t/incremental --sancov %sancov --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests + +# RUN: test -f %t/incremental/new_coverage.csv +# RUN: %batch-reduce --csv %t/incremental/new_coverage.csv --candidate-tests %t/candidate_tests --output %t/reduced --n 1 --pipeline llvm_reduce_ir + +# RUN: test -f %t/reduced/t-00001-test_0_s/config.json +# RUN: test -f %t/reduced/t-00001-test_0_s/interesting_ir.sh +# RUN: test -f %t/reduced/t-00001-test_0_s/standalone-empty-kernel.ll +# RUN: test -x %t/reduced/t-00001-test_0_s/interesting_ir.sh + +# RUN: %FileCheck %s --input-file=%t/reduced/t-00001-test_0_s/config.json --check-prefix=CONFIG +# CONFIG: "input": "standalone-empty-kernel.ll", +# CONFIG: "file": "llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp", +# CONFIG: "line": 91, +# CONFIG: "llvm_reduce_ir" + +# RUN: %FileCheck %s --input-file=%t/reduced/t-00001-test_0_s/interesting_ir.sh --check-prefix=INTERESTING +# INTERESTING: LLC_FLAGS="-O0" +# INTERESTING: COVERED="0x15a77d0" diff --git a/integration-tests/lit.cfg.py b/integration-tests/lit.cfg.py index fb4dd0b3..4041c8fc 100644 --- a/integration-tests/lit.cfg.py +++ b/integration-tests/lit.cfg.py @@ -83,8 +83,12 @@ def _tool(bin_dir: str, name: str) -> str: _llvm_build_dir = os.path.dirname(_llvm_bin_dir) +_repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) _venv_cmd = f". {shlex.quote(_venv_activate)}" _venv_python = shlex.quote(os.path.join(_venv_dir, "bin", "python")) +_batch_reduce = shlex.quote( + os.path.join(_repo_root, "scripts", "batch_reduce_using_coverage.py") +) config.name = "fuzz-fill-integration-tests" config.suffixes = [".test"] @@ -103,6 +107,7 @@ def _tool(bin_dir: str, name: str) -> str: ("%venv", _venv_cmd), ("%coverage", f"{_venv_python} -m coverage"), ("%reduce", f"{_venv_python} -m reduce"), + ("%batch-reduce", f"{_venv_python} {_batch_reduce}"), ("%added-lines", f"{_venv_cmd} && python -m added_lines"), ] ) diff --git a/scripts/batch_reduce_using_coverage.py b/scripts/batch_reduce_using_coverage.py index b2d8b7f9..c8e669a2 100644 --- a/scripts/batch_reduce_using_coverage.py +++ b/scripts/batch_reduce_using_coverage.py @@ -6,7 +6,7 @@ CSV columns: test_name, file, line, covered-points (test_name is a directory under --candidate-tests containing test.sh). -Each test.sh records the instrumented llc invocation (binary, flags, .bc path). +Each test.sh records the instrumented llc invocation (binary, flags, .bc or .ll path). Llvm flags on that command are copied into interesting_ir.sh and extract_* pipeline steps (config ``llc_O``). interesting_mir.sh uses the same COVERED sancov check as interesting_ir.sh. Use ``--mir-codegen-only`` when the pass @@ -136,8 +136,30 @@ class TestShInfo: llc_flags: tuple[str, ...] +def resolve_input_path(input_path: Path, test_sh: Path) -> Path: + """Resolve IR input from test.sh, including Docker/host path remapping.""" + if input_path.is_file(): + return input_path + + repo = _repo_root() + posix = input_path.as_posix() + marker = "/fuzz-fill/" + if marker in posix: + candidate = repo / posix.split(marker, 1)[1] + if candidate.is_file(): + return candidate + + sibling = test_sh.parent / input_path.name + if sibling.is_file(): + return sibling + + raise FileNotFoundError( + f"Input IR file from test.sh does not exist: {input_path}" + ) + + def parse_test_sh(test_sh: Path) -> TestShInfo: - """Parse candidate_tests//test.sh for llc binary, flags, and .bc path.""" + """Parse candidate_tests//test.sh for llc binary, flags, and IR input.""" if not test_sh.is_file(): raise FileNotFoundError(f"test.sh not found: {test_sh}") @@ -160,22 +182,28 @@ def parse_test_sh(test_sh: Path) -> TestShInfo: if len(parts) < 2: raise ValueError(f"Could not parse llc command in {test_sh}: {llc_line!r}") - llc_exe = Path(parts[0]) - if llc_exe.name != "llc": - raise ValueError(f"Expected llc executable, got {llc_exe!r} in {test_sh}") - - bc_indices = [i for i, p in enumerate(parts) if p.endswith(".bc")] - if len(bc_indices) != 1: + llc_indices = [i for i, p in enumerate(parts) if Path(p).name == "llc"] + if len(llc_indices) != 1: + raise ValueError( + f"Expected exactly one llc executable in {test_sh}, got {llc_indices!r}" + ) + llc_idx = llc_indices[0] + llc_exe = Path(parts[llc_idx]) + + input_indices = [ + i + for i, p in enumerate(parts) + if i > llc_idx and (p.endswith(".bc") or p.endswith(".ll")) + ] + if len(input_indices) != 1: raise ValueError( - f"Expected exactly one .bc argument in {test_sh}, got {bc_indices!r}" + f"Expected exactly one .bc or .ll argument in {test_sh}, got {input_indices!r}" ) - bc_idx = bc_indices[0] - bc_path = Path(parts[bc_idx]) - if not bc_path.is_file(): - raise FileNotFoundError(f"Bitcode file from test.sh does not exist: {bc_path}") + input_idx = input_indices[0] + input_path = resolve_input_path(Path(parts[input_idx]), test_sh) - flags = tuple(parts[1:bc_idx]) - return TestShInfo(llvm_bin=llc_exe.parent, bc_path=bc_path, llc_flags=flags) + flags = tuple(parts[llc_idx + 1 : input_idx]) + return TestShInfo(llvm_bin=llc_exe.parent, bc_path=input_path, llc_flags=flags) def render_interesting_ir( @@ -509,7 +537,7 @@ def validate_pipeline_cli( def copy_input_bc(test_info: TestShInfo, dest_dir: Path) -> str: - """Copy the .bc from test.sh into dest_dir; return config input basename.""" + """Copy the IR input from test.sh into dest_dir; return config input basename.""" dest_name = test_info.bc_path.name shutil.copy2(test_info.bc_path, dest_dir / dest_name) return dest_name diff --git a/scripts/count_lit_failures.py b/scripts/count_lit_failures.py new file mode 100755 index 00000000..d20a8f9f --- /dev/null +++ b/scripts/count_lit_failures.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Count LIT tests with failure-like result codes in a lit_failures.json report.""" + +import argparse +import json + +FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"}) + + +def count_failures(path: str) -> int: + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return 0 + return sum(1 for t in data.get("tests", []) if t.get("code") in FAILURE_CODES) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("lit_failures_json", help="Path to lit_failures.json from coverage baseline") + args = parser.parse_args() + print(count_failures(args.lit_failures_json)) + + +if __name__ == "__main__": + main() diff --git a/scripts/docker/docker-image-cli.sh b/scripts/docker/docker-image-cli.sh new file mode 100644 index 00000000..97ce8dd6 --- /dev/null +++ b/scripts/docker/docker-image-cli.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Shared Docker image flag parsing and prepare helpers for gap docker runners. +# Source after SCRIPT_DIR is set; requires ensure-image.sh. + +: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing docker-image-cli.sh}" + +docker_image_cli_init_vars() { + image_name="${IMAGE_NAME:-fuzz-fill-test}" + image_tag="${IMAGE_TAG:-latest}" + image_ref="" + pr_id="" + build_image=0 + keep_image=0 + force_build=0 + llvm_repo="" + backend_tests="" + github_repo="" +} + +# Parse one image-related flag. Returns 0 if consumed; sets DOCKER_IMAGE_CLI_SHIFT. +docker_image_cli_try_parse() { + DOCKER_IMAGE_CLI_SHIFT=0 + + case "$1" in + --image) + [[ $# -ge 2 ]] || { echo "error: --image requires a value" >&2; exit 2; } + image_ref="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + --pr-id) + [[ $# -ge 2 ]] || { echo "error: --pr-id requires a value" >&2; exit 2; } + pr_id="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + --build-image) + build_image=1 + DOCKER_IMAGE_CLI_SHIFT=1 + return 0 + ;; + --keep-image) + keep_image=1 + DOCKER_IMAGE_CLI_SHIFT=1 + return 0 + ;; + --force-build) + force_build=1 + DOCKER_IMAGE_CLI_SHIFT=1 + return 0 + ;; + --llvm-repo) + [[ $# -ge 2 ]] || { echo "error: --llvm-repo requires a value" >&2; exit 2; } + llvm_repo="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + --backend-tests) + [[ $# -ge 2 ]] || { echo "error: --backend-tests requires a value" >&2; exit 2; } + backend_tests="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + --github-repo) + [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; } + github_repo="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + --image-name) + [[ $# -ge 2 ]] || { echo "error: --image-name requires a value" >&2; exit 2; } + image_name="$2" + DOCKER_IMAGE_CLI_SHIFT=2 + return 0 + ;; + *) + return 1 + ;; + esac +} + +docker_image_cli_prepare() { + docker_image_validate_build_flags + docker_image_normalize_backend_tests + docker_image_validate_pr_id + docker_image_resolve_ref + docker_image_ensure +} diff --git a/scripts/docker/docker-run.sh b/scripts/docker/docker-run.sh new file mode 100644 index 00000000..1ab792ce --- /dev/null +++ b/scripts/docker/docker-run.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Shared docker run helpers for gap docker runners. +# Source after SCRIPT_DIR and REPO_ROOT are set. + +: "${REPO_ROOT:?REPO_ROOT must be set before sourcing docker-run.sh}" + +DOCKER_GAP_CONTAINER_WORKDIR="/work/fuzz-fill" + +docker_gap_container_script() { + printf '%s/scripts/docker/%s' "$DOCKER_GAP_CONTAINER_WORKDIR" "$(basename "$1")" +} + +# Parse --bind-repo or -j/--jobs. Returns 0 if consumed; sets DOCKER_GAP_CLI_SHIFT. +docker_gap_cli_try_parse_common() { + DOCKER_GAP_CLI_SHIFT=0 + + case "$1" in + --bind-repo) + bind_repo=1 + DOCKER_GAP_CLI_SHIFT=1 + return 0 + ;; + -j|--jobs) + [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 2; } + jobs="$2" + DOCKER_GAP_CLI_SHIFT=2 + return 0 + ;; + *) + return 1 + ;; + esac +} + +docker_gap_append_bind_repo_mount() { + local -n _mounts=$1 + if [[ "${bind_repo:-0}" -eq 1 ]]; then + _mounts+=(-v "${REPO_ROOT}:${DOCKER_GAP_CONTAINER_WORKDIR}") + echo "Using local fuzz-fill checkout: ${REPO_ROOT}" + fi +} + +docker_gap_append_jobs_env() { + local -n _env=$1 + if [[ -n "${jobs:-}" ]]; then + _env+=(-e "JOBS=${jobs}") + fi +} + +# Run the current docker gap workflow script inside a container. +docker_gap_run() { + local container_script="$1" + local host_output_dir="$2" + local image="$3" + local -n _env=$4 + local -n _mounts=$5 + + docker run --rm \ + -v "${host_output_dir}:/mounted-output" \ + "${_mounts[@]}" \ + "${_env[@]}" \ + -w "${DOCKER_GAP_CONTAINER_WORKDIR}" \ + "${image}" \ + bash "${container_script}" +} diff --git a/scripts/docker/ensure-image.sh b/scripts/docker/ensure-image.sh new file mode 100644 index 00000000..fa6863fc --- /dev/null +++ b/scripts/docker/ensure-image.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Shared Docker image resolution, PR build, and reuse helpers for fuzz-fill docker runners. +# Source this file from runner scripts; do not execute directly. + +: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing ensure-image.sh}" + +docker_image_cleanup_built() { + if [[ "${build_image:-0}" -eq 1 && "${keep_image:-0}" -eq 0 && -n "${image_ref:-}" ]]; then + if docker image inspect "${image_ref}" >/dev/null 2>&1; then + echo "Removing Docker image ${image_ref}" + docker rmi "${image_ref}" + fi + fi +} + +docker_image_validate_pr_id() { + if [[ -z "${pr_id:-}" ]]; then + return 0 + fi + if [[ ! "$pr_id" =~ ^[0-9]+$ ]] || [[ "$pr_id" -eq 0 ]]; then + echo "error: --pr-id must be a positive integer: ${pr_id}" >&2 + exit 1 + fi +} + +docker_image_normalize_backend_tests() { + if [[ -z "${backend_tests:-}" ]]; then + return 0 + fi + backend_tests="$(printf '%s' "$backend_tests" | tr '[:upper:]' '[:lower:]')" + case "$backend_tests" in + amdgpu|spirv) ;; + *) + echo "error: --backend-tests must be amdgpu or spirv: ${backend_tests}" >&2 + exit 1 + ;; + esac +} + +docker_image_resolve_ref() { + local name="${image_name:-${IMAGE_NAME:-fuzz-fill-test}}" + if [[ -n "${pr_id:-}" ]]; then + image_ref="${name}:llvm-pr-${pr_id}" + elif [[ -z "${image_ref:-}" ]]; then + image_ref="${name}:${image_tag:-latest}" + fi +} + +docker_image_validate_build_flags() { + if [[ -n "${image_ref:-}" && -n "${pr_id:-}" ]]; then + echo "error: pass only one of --image or --pr-id" >&2 + exit 1 + fi + + if [[ -n "${image_ref:-}" && "${build_image:-0}" -eq 1 ]]; then + echo "error: --build-image cannot be used with --image" >&2 + exit 1 + fi + + if [[ "${force_build:-0}" -eq 1 && "${build_image:-0}" -eq 0 ]]; then + echo "error: --force-build requires --build-image" >&2 + exit 1 + fi + + if [[ "${build_image:-0}" -eq 0 ]]; then + if [[ -n "${llvm_repo:-}" || -n "${backend_tests:-}" || -n "${github_repo:-}" \ + || "${keep_image:-0}" -eq 1 || "${force_build:-0}" -eq 1 ]]; then + echo "error: --llvm-repo, --backend-tests, --github-repo, --keep-image, and --force-build require --build-image" >&2 + exit 1 + fi + return 0 + fi + + if [[ -z "${llvm_repo:-}" ]]; then + echo "error: --llvm-repo is required with --build-image" >&2 + exit 1 + fi + if [[ -z "${pr_id:-}" ]]; then + echo "error: --pr-id is required with --build-image" >&2 + exit 1 + fi + if [[ -z "${backend_tests:-}" ]]; then + echo "error: --backend-tests is required with --build-image" >&2 + exit 1 + fi +} + +docker_image_read_allowlist() { + local allowlist + if ! allowlist="$(docker run --rm --entrypoint cat "${image_ref}" /work/.sancov-allowlist 2>/dev/null | tr -d '[:space:]')"; then + echo "error: failed to read /work/.sancov-allowlist from image: ${image_ref}" >&2 + exit 1 + fi + if [[ -z "$allowlist" ]]; then + echo "error: /work/.sancov-allowlist is empty in image: ${image_ref}" >&2 + exit 1 + fi + printf '%s' "$allowlist" +} + +docker_image_ensure() { + if [[ "${build_image:-0}" -eq 1 && "${keep_image:-0}" -eq 0 ]]; then + trap docker_image_cleanup_built EXIT + fi + + if [[ "${build_image:-0}" -eq 1 ]]; then + if docker image inspect "${image_ref}" >/dev/null 2>&1 && [[ "${force_build:-0}" -eq 0 ]]; then + echo "Reusing existing Docker image ${image_ref}" + else + if [[ "${force_build:-0}" -eq 1 ]]; then + echo "=== rebuild PR image (--force-build) ===" + else + echo "=== build PR image ===" + fi + + build_args=( + --llvm-repo "$llvm_repo" + --pr-id "$pr_id" + --allowlist "$backend_tests" + ) + if [[ -n "${github_repo:-}" ]]; then + build_args+=(--github-repo "$github_repo") + fi + if [[ -n "${jobs:-}" ]]; then + build_args+=(-j "$jobs") + fi + + "${SCRIPT_DIR}/build-image-pr.sh" "${build_args[@]}" + fi + fi + + if ! docker image inspect "${image_ref}" >/dev/null 2>&1; then + echo "error: image not found: ${image_ref}" >&2 + if [[ -n "${DOCKER_IMAGE_MISSING_HINT:-}" ]]; then + echo "hint: ${DOCKER_IMAGE_MISSING_HINT}" >&2 + fi + exit 1 + fi +} diff --git a/scripts/docker/gap-filling.sh b/scripts/docker/gap-filling.sh new file mode 100755 index 00000000..26fff483 --- /dev/null +++ b/scripts/docker/gap-filling.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# Gap filling: candidate-test -> incremental inside a one-shot Docker container. +# +# Requires a gap list from gap finding (baseline or PR), passed as explicit CSV +# paths (same flags as ``coverage incremental``). By default the host corpus is +# bind-mounted read-only at /mounted-candidate-tests (no copy); -n limits work +# inside candidate-test. Pass --stage-candidate-tests to copy the first N inputs +# into a temp dir before mounting. Nothing is baked into the image. +# +# Pass --bind-repo to run the local fuzz-fill checkout instead of the copy baked +# into the image at build time. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +if [[ "${IN_CONTAINER:-}" == 1 ]]; then + # shellcheck source=scripts/lib/gap-filling.sh + source "${REPO_ROOT}/scripts/lib/gap-filling.sh" + + : "${CANDIDATE_N:?CANDIDATE_N is required}" + : "${UNCOVERED_CSV:?UNCOVERED_CSV is required}" + : "${LLC_MAP_CSV:?LLC_MAP_CSV is required}" + + run_candidate_test /mounted-output/candidate_tests /mounted-candidate-tests "$CANDIDATE_N" + + run_incremental \ + /mounted-output/incremental \ + "$UNCOVERED_CSV" \ + "$LLC_MAP_CSV" \ + /mounted-output/candidate_tests + exit 0 +fi + +# shellcheck source=scripts/docker/ensure-image.sh +source "${SCRIPT_DIR}/ensure-image.sh" +# shellcheck source=scripts/docker/docker-image-cli.sh +source "${SCRIPT_DIR}/docker-image-cli.sh" +# shellcheck source=scripts/docker/docker-run.sh +source "${SCRIPT_DIR}/docker-run.sh" +# shellcheck source=scripts/lib/common.sh +source "${REPO_ROOT}/scripts/lib/common.sh" +# shellcheck source=scripts/lib/gap-artifacts.sh +source "${REPO_ROOT}/scripts/lib/gap-artifacts.sh" +# shellcheck source=scripts/lib/candidate-inputs.sh +source "${REPO_ROOT}/scripts/lib/candidate-inputs.sh" + +CONTAINER_WORKDIR="${DOCKER_GAP_CONTAINER_WORKDIR}" +CONTAINER_SCRIPT="$(docker_gap_container_script "$0")" + +docker_image_cli_init_vars +output_dir="" +llc_address_line_map_csv="" +line_coverage_uncovered_csv="" +jobs="" +bind_repo=0 +stage_candidate_tests=0 +candidate_tests_dir="" +candidate_n="" + +usage() { + cat < [options] + +Run candidate-test and incremental in a temporary container. +Artifacts are written under --output-dir (mounted at /mounted-output/). + +Required: + --output-dir Host output directory (created if missing) + --line-coverage-uncovered-csv + Uncovered-lines CSV (``file``, ``line`` columns) + --llc-address-line-map-csv + LLC address-to-line map CSV from the same baseline run + --candidate-tests-dir Host corpus root (bind-mounted read-only by default) + -n , --n Run only the first N candidate tests (.ll/.bc, sorted) + +Image (one of): + --image Docker image ref (default: \${IMAGE_NAME:-fuzz-fill-test}:\${IMAGE_TAG:-latest}) + --pr-id Use \${IMAGE_NAME:-fuzz-fill-test}:llvm-pr- + +Image build (optional; reuses existing image when omitted): + --build-image Build PR image via build-image-pr.sh when missing + --force-build Rebuild PR image even when the tag already exists + --keep-image Keep PR image after run (default: remove when --build-image) + --llvm-repo Local llvm-project clone (required with --build-image) + --backend-tests amdgpu or spirv (required with --build-image) + --github-repo GitHub repo hosting the PR (default: llvm/llvm-project) + --image-name Image name when using --pr-id (default: fuzz-fill-test) + +Options: + --bind-repo Mount the local fuzz-fill checkout at ${CONTAINER_WORKDIR} + --stage-candidate-tests Copy the first N inputs to a temp dir before mounting + (default: bind-mount the full --candidate-tests-dir) + -j , --jobs Parallel jobs for candidate-test and ninja (build) + --help, -h Show this help + +Examples: + $(basename "$0") --output-dir ./data \\ + --line-coverage-uncovered-csv line_coverage_uncovered.csv \\ + --llc-address-line-map-csv llc_address_line_map.csv \\ + --candidate-tests-dir /path/to/corpus -n 100 -j "\$(nproc)" + + $(basename "$0") --pr-id 203468 --output-dir ./data \\ + --line-coverage-uncovered-csv line_coverage_uncovered.csv \\ + --llc-address-line-map-csv llc_address_line_map.csv \\ + --candidate-tests-dir /path/to/corpus -n 100 -j "\$(nproc)" +EOF +} + +validate_candidate_n() { + if [[ ! "$candidate_n" =~ ^[0-9]+$ ]] || [[ "$candidate_n" -eq 0 ]]; then + echo "error: -n/--n must be a positive integer: ${candidate_n}" >&2 + exit 1 + fi +} + +while [[ $# -gt 0 ]]; do + if docker_image_cli_try_parse "$1" "${2:-}"; then + shift "${DOCKER_IMAGE_CLI_SHIFT}" + continue + fi + if docker_gap_cli_try_parse_common "$1" "${2:-}"; then + shift "${DOCKER_GAP_CLI_SHIFT}" + continue + fi + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 2; } + output_dir="$2" + shift 2 + ;; + --line-coverage-uncovered-csv) + [[ $# -ge 2 ]] || { echo "error: --line-coverage-uncovered-csv requires a value" >&2; exit 2; } + line_coverage_uncovered_csv="$2" + shift 2 + ;; + --llc-address-line-map-csv) + [[ $# -ge 2 ]] || { echo "error: --llc-address-line-map-csv requires a value" >&2; exit 2; } + llc_address_line_map_csv="$2" + shift 2 + ;; + --stage-candidate-tests) + stage_candidate_tests=1 + shift + ;; + --candidate-tests-dir) + [[ $# -ge 2 ]] || { echo "error: --candidate-tests-dir requires a value" >&2; exit 2; } + candidate_tests_dir="$2" + shift 2 + ;; + -n|--n) + [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 2; } + candidate_n="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ $# -gt 0 ]]; then + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 +fi + +if [[ -z "$output_dir" ]]; then + echo "error: --output-dir is required" >&2 + usage >&2 + exit 1 +fi + +if [[ -z "$candidate_tests_dir" || -z "$candidate_n" ]]; then + echo "error: --candidate-tests-dir and -n are required" >&2 + usage >&2 + exit 1 +fi + +validate_candidate_n +if [[ ! -d "$candidate_tests_dir" ]]; then + echo "error: --candidate-tests-dir is not a directory: ${candidate_tests_dir}" >&2 + exit 1 +fi + +resolve_gap_profile_csv_paths line_coverage_uncovered_csv llc_address_line_map_csv +validate_jobs "$jobs" + +DOCKER_IMAGE_MISSING_HINT="build with ${SCRIPT_DIR}/build-image.sh, or pass --build-image with --llvm-repo and --backend-tests" +docker_image_cli_prepare + +container_uncovered_csv="/mounted-profile/line_coverage_uncovered.csv" +container_llc_map_csv="/mounted-profile/llc_address_line_map.csv" + +echo "Using coverage profile:" +echo " uncovered lines: ${line_coverage_uncovered_csv}" +echo " address map: ${llc_address_line_map_csv}" + +mkdir -p "$output_dir" +output_dir="$(realpath "$output_dir")" +candidate_tests_dir="$(realpath "$candidate_tests_dir")" + +if [[ "${stage_candidate_tests}" -eq 1 ]]; then + candidate_staging_dir="$(mktemp -d -t fuzz-fill-candidate-tests.XXXXXX)" + cleanup_staging() { + rm -rf "${candidate_staging_dir}" + } + trap cleanup_staging EXIT + stage_candidate_tests "$candidate_tests_dir" "$candidate_n" "$candidate_staging_dir" + candidate_mount=(-v "${candidate_staging_dir}:/mounted-candidate-tests:ro") +else + mapfile -t candidate_files < <(collect_candidate_inputs "$candidate_tests_dir") + if [[ "${#candidate_files[@]}" -eq 0 ]]; then + echo "error: no .ll or .bc files under --candidate-tests-dir: ${candidate_tests_dir}" >&2 + exit 1 + fi + candidate_mount=(-v "${candidate_tests_dir}:/mounted-candidate-tests:ro") + echo "Bind-mounting candidate corpus (no copy): ${candidate_tests_dir}" +fi + +rm -rf "${output_dir}/candidate_tests" "${output_dir}/incremental" + +docker_env=( + -e "IN_CONTAINER=1" + -e "CANDIDATE_N=${candidate_n}" + -e "UNCOVERED_CSV=${container_uncovered_csv}" + -e "LLC_MAP_CSV=${container_llc_map_csv}" +) +docker_gap_append_jobs_env docker_env + +extra_mounts=( + -v "${line_coverage_uncovered_csv}:${container_uncovered_csv}:ro" + -v "${llc_address_line_map_csv}:${container_llc_map_csv}:ro" +) +extra_mounts+=("${candidate_mount[@]}") +docker_gap_append_bind_repo_mount extra_mounts + +docker_gap_run "$CONTAINER_SCRIPT" "$output_dir" "$image_ref" docker_env extra_mounts + +echo "Image: ${image_ref}" +report="${output_dir}/incremental/new_coverage.csv" +echo "Wrote ${report}" diff --git a/scripts/docker/gap-finding-baseline.sh b/scripts/docker/gap-finding-baseline.sh new file mode 100755 index 00000000..32714c07 --- /dev/null +++ b/scripts/docker/gap-finding-baseline.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Gap finding (baseline): run coverage baseline inside a one-shot Docker container. +# +# Host output is bind-mounted at /mounted-output/. Writes baseline/ under --output-dir. +# +# Pass --bind-repo to run the local fuzz-fill checkout instead of the copy baked +# into the image at build time. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +if [[ "${IN_CONTAINER:-}" == 1 ]]; then + # shellcheck source=scripts/lib/coverage-baseline.sh + source "${REPO_ROOT}/scripts/lib/coverage-baseline.sh" + + : "${LIT_FILTER:?LIT_FILTER is required}" + + export LIT_ALLOW_FAILURES=1 + + echo "=== coverage baseline (lit-filter=${LIT_FILTER}) ===" + run_coverage_baseline /mounted-output/baseline "$LIT_FILTER" + exit 0 +fi + +# shellcheck source=scripts/docker/gap-finding-docker.sh +source "${SCRIPT_DIR}/gap-finding-docker.sh" +docker_gap_finding_source_host_libs "$0" + +usage() { + cat < [options] + +Run coverage baseline in a temporary container. +Artifacts are written under --output-dir/baseline/ (mounted at /mounted-output/). + +Required: + --output-dir Host output directory (created if missing) + +$(docker_gap_finding_usage_image_select_options) + +Image build (optional; reuses existing image when omitted): +$(docker_gap_finding_usage_image_build_options) + +Options: + --bind-repo Mount the local fuzz-fill checkout at ${CONTAINER_WORKDIR} + --lit-filter LIT --filter= prefix (default: from image /work/.sancov-allowlist) + -j , --jobs Parallel jobs for llvm-lit and ninja (build) + --help, -h Show this help + +Examples: + $(basename "$0") --output-dir ./data/baseline -j "\$(nproc)" + $(basename "$0") --pr-id 203468 --output-dir ./data/pr-baseline -j "\$(nproc)" + $(basename "$0") --build-image --llvm-repo /path/llvm-project --pr-id 203468 \\ + --backend-tests amdgpu --output-dir ./data/pr-baseline -j "\$(nproc)" +EOF +} + +while [[ $# -gt 0 ]]; do + if docker_image_cli_try_parse "$1" "${2:-}"; then + shift "${DOCKER_IMAGE_CLI_SHIFT}" + continue + fi + if docker_gap_cli_try_parse_common "$1" "${2:-}"; then + shift "${DOCKER_GAP_CLI_SHIFT}" + continue + fi + if docker_gap_finding_try_parse_workflow "$1" "${2:-}"; then + shift "${DOCKER_GAP_FINDING_SHIFT}" + continue + fi + case "$1" in + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if ! docker_gap_finding_finish_arg_parse "$@"; then + usage >&2 + exit 2 +fi + +if ! docker_gap_finding_require_output_dir; then + usage >&2 + exit 1 +fi + +validate_jobs "$jobs" + +DOCKER_IMAGE_MISSING_HINT="build with ${SCRIPT_DIR}/build-image.sh, or pass --build-image with --llvm-repo and --backend-tests" +docker_image_cli_prepare + +docker_gap_default_lit_filters_from_image single +lit_filter="${lit_filters[0]}" + +docker_gap_finding_prepare_output_dir +rm -rf "${output_dir}/baseline" + +docker_env=(-e "IN_CONTAINER=1" -e "LIT_FILTER=${lit_filter}") +docker_gap_append_jobs_env docker_env + +extra_mounts=() +docker_gap_append_bind_repo_mount extra_mounts + +docker_gap_run "$CONTAINER_SCRIPT" "$output_dir" "$image_ref" docker_env extra_mounts + +echo "Image: ${image_ref}" +echo "LIT filter: ${lit_filter}" +echo "Wrote ${output_dir}/baseline/" + +emit_lit_failures_warning "$output_dir" "downstream gap lists" diff --git a/scripts/docker/gap-finding-docker.sh b/scripts/docker/gap-finding-docker.sh new file mode 100644 index 00000000..f6951493 --- /dev/null +++ b/scripts/docker/gap-finding-docker.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Shared host-side helpers for docker gap-finding runners (baseline and PR). +# Source after SCRIPT_DIR and REPO_ROOT are set. + +: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing gap-finding-docker.sh}" +: "${REPO_ROOT:?REPO_ROOT must be set before sourcing gap-finding-docker.sh}" + +docker_gap_finding_source_host_libs() { + local entrypoint="$1" + + # shellcheck source=scripts/docker/ensure-image.sh + source "${SCRIPT_DIR}/ensure-image.sh" + # shellcheck source=scripts/docker/docker-image-cli.sh + source "${SCRIPT_DIR}/docker-image-cli.sh" + # shellcheck source=scripts/docker/docker-run.sh + source "${SCRIPT_DIR}/docker-run.sh" + # shellcheck source=scripts/lib/common.sh + source "${REPO_ROOT}/scripts/lib/common.sh" + # shellcheck source=scripts/lib/lit-filters.sh + source "${REPO_ROOT}/scripts/lib/lit-filters.sh" + # shellcheck source=scripts/lib/lit-failures.sh + source "${REPO_ROOT}/scripts/lib/lit-failures.sh" + + CONTAINER_WORKDIR="${DOCKER_GAP_CONTAINER_WORKDIR}" + CONTAINER_SCRIPT="$(docker_gap_container_script "$entrypoint")" + + docker_image_cli_init_vars + output_dir="" + lit_filters=() + jobs="" + bind_repo=0 +} + +docker_gap_finding_usage_image_select_options() { + cat < Docker image ref (default: \${IMAGE_NAME:-fuzz-fill-test}:\${IMAGE_TAG:-latest}) + --pr-id Use \${IMAGE_NAME:-fuzz-fill-test}:llvm-pr- + +EOF +} + +docker_gap_finding_usage_image_build_options() { + cat < Local llvm-project clone (required with --build-image) + --backend-tests amdgpu or spirv (required with --build-image) + --github-repo GitHub repo hosting the PR (default: llvm/llvm-project) + --image-name Image name when using --pr-id (default: fuzz-fill-test) +EOF +} + +# Parse --output-dir or --lit-filter. Returns 0 if consumed; sets DOCKER_GAP_FINDING_SHIFT. +docker_gap_finding_try_parse_workflow() { + DOCKER_GAP_FINDING_SHIFT=0 + + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 2; } + output_dir="$2" + DOCKER_GAP_FINDING_SHIFT=2 + return 0 + ;; + --lit-filter) + [[ $# -ge 2 ]] || { echo "error: --lit-filter requires a value" >&2; exit 2; } + lit_filters+=("$2") + DOCKER_GAP_FINDING_SHIFT=2 + return 0 + ;; + *) + return 1 + ;; + esac +} + +docker_gap_finding_finish_arg_parse() { + if [[ $# -gt 0 ]]; then + echo "error: unexpected argument: $1" >&2 + return 1 + fi + return 0 +} + +docker_gap_finding_require_output_dir() { + if [[ -z "$output_dir" ]]; then + echo "error: --output-dir is required" >&2 + return 1 + fi + return 0 +} + +docker_gap_finding_prepare_output_dir() { + mkdir -p "$output_dir" + output_dir="$(realpath "$output_dir")" +} + +# mode: single (one prefix) or multi (full allowlist list). +docker_gap_default_lit_filters_from_image() { + local mode="$1" + local allowlist + + if [[ ${#lit_filters[@]} -gt 0 ]]; then + return 0 + fi + + allowlist="$(docker_image_read_allowlist)" + case "$mode" in + single) + lit_filters=("$(lit_filter_for_allowlist "$allowlist")") + echo "Image allowlist: ${allowlist} -> lit-filter: ${lit_filters[0]}" + ;; + multi) + mapfile -t lit_filters < <(default_lit_filters_for_allowlist "$allowlist") + echo "Image allowlist: ${allowlist} -> ${#lit_filters[@]} lit-filter prefix(es)" + ;; + *) + echo "error: docker_gap_default_lit_filters_from_image: unknown mode: ${mode}" >&2 + exit 1 + ;; + esac +} + +docker_gap_finding_write_lit_filters_file() { + local filters_file="${output_dir}/.lit-filters" + printf '%s\n' "${lit_filters[@]}" > "$filters_file" +} diff --git a/scripts/docker/gap-finding-pr.sh b/scripts/docker/gap-finding-pr.sh new file mode 100755 index 00000000..47333f52 --- /dev/null +++ b/scripts/docker/gap-finding-pr.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Gap finding (PR): baseline -> added_lines -> target-lines in a one-shot Docker container. +# +# Host output is bind-mounted at /mounted-output/ in the container. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +if [[ "${IN_CONTAINER:-}" == 1 ]]; then + # shellcheck source=scripts/lib/gap-finding-pr.sh + source "${REPO_ROOT}/scripts/lib/gap-finding-pr.sh" + + commit="${COMMIT_REV:-$(git -C /work/llvm-project rev-parse HEAD)}" + + mapfile -t lit_filters < /mounted-output/.lit-filters + + export LIT_ALLOW_FAILURES=1 + + run_gap_finding_pr /mounted-output "$commit" "" "${lit_filters[@]}" + exit 0 +fi + +# shellcheck source=scripts/docker/gap-finding-docker.sh +source "${SCRIPT_DIR}/gap-finding-docker.sh" +docker_gap_finding_source_host_libs "$0" + +commit_rev="" + +usage() { + cat < | --pr-id ) --output-dir [options] + +Run coverage baseline, added_lines, and target-lines in a temporary container. +Artifacts are written under --output-dir (mounted at /mounted-output/). + +Required (one of): + --image Full Docker image ref (e.g. fuzz-fill-test:llvm-pr-203468) + --pr-id Derive image as \${IMAGE_NAME:-fuzz-fill-test}:llvm-pr- + +Required: + --output-dir Host output directory (created if missing) + +Options: +$(docker_gap_finding_usage_image_build_options) + --lit-filter LIT regex prefix; repeat for multiple (default values available in: scripts/lit-filters-amdgpu.sh for amdgpu, CodeGen/SPIRV for spirv) + --commit Revision for added_lines (default: HEAD in image llvm-project) + --bind-repo Mount the local fuzz-fill checkout at ${CONTAINER_WORKDIR} + -j , --jobs Parallel jobs for llvm-lit; with --build-image, also for ninja + --help, -h Show this help + +Examples: + $(basename "$0") --build-image --llvm-repo /path/llvm-project --pr-id 203468 \\ + --backend-tests amdgpu --output-dir ./data/gap-finding-pr-203468 -j "\$(nproc)" + $(basename "$0") --pr-id 203468 --output-dir ./data/gap-finding-pr-203468 + $(basename "$0") --pr-id 203468 --output-dir ./data/gap-finding-pr-203468 \\ + --lit-filter CodeGen/AMDGPU -j "\$(nproc)" +EOF +} + +while [[ $# -gt 0 ]]; do + if docker_image_cli_try_parse "$1" "${2:-}"; then + shift "${DOCKER_IMAGE_CLI_SHIFT}" + continue + fi + if docker_gap_cli_try_parse_common "$1" "${2:-}"; then + shift "${DOCKER_GAP_CLI_SHIFT}" + continue + fi + if docker_gap_finding_try_parse_workflow "$1" "${2:-}"; then + shift "${DOCKER_GAP_FINDING_SHIFT}" + continue + fi + case "$1" in + --commit) + [[ $# -ge 2 ]] || { echo "error: --commit requires a value" >&2; exit 2; } + commit_rev="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + echo "error: unexpected argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if ! docker_gap_finding_finish_arg_parse "$@"; then + usage >&2 + exit 2 +fi + +if [[ -n "$image_ref" && -n "$pr_id" ]]; then + echo "error: pass only one of --image or --pr-id" >&2 + exit 1 +fi + +if [[ -z "$image_ref" && -z "$pr_id" ]]; then + echo "error: one of --image or --pr-id is required" >&2 + usage >&2 + exit 1 +fi + +if ! docker_gap_finding_require_output_dir; then + usage >&2 + exit 1 +fi + +validate_jobs "$jobs" + +DOCKER_IMAGE_MISSING_HINT="pass --build-image with --llvm-repo and --backend-tests to build it first" +docker_image_cli_prepare + +docker_gap_default_lit_filters_from_image multi + +docker_gap_finding_prepare_output_dir +docker_gap_finding_write_lit_filters_file + +docker_env=(-e "IN_CONTAINER=1") +if [[ -n "$commit_rev" ]]; then + docker_env+=(-e "COMMIT_REV=${commit_rev}") +fi +docker_gap_append_jobs_env docker_env + +extra_mounts=() +docker_gap_append_bind_repo_mount extra_mounts + +docker_gap_run "$CONTAINER_SCRIPT" "$output_dir" "$image_ref" docker_env extra_mounts + +report="${output_dir}/commit_lines_report/target_lines_uncovered.csv" +echo "Wrote ${report}" +echo "Image: ${image_ref}" +echo "LIT filters: ${lit_filters[*]}" + +emit_lit_failures_warning "$output_dir" "target_lines_uncovered.csv" diff --git a/scripts/docker/gap-reducing.sh b/scripts/docker/gap-reducing.sh new file mode 100755 index 00000000..7f42c54b --- /dev/null +++ b/scripts/docker/gap-reducing.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Gap reducing: reduce one row from gap-filling output inside a one-shot Docker container. +# +# Requires --output-dir from a prior gap-filling run: +# incremental/new_coverage.csv +# candidate_tests/ +# +# Writes reduced/ under --output-dir. Pass --bind-repo to use the local fuzz-fill checkout. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +if [[ "${IN_CONTAINER:-}" == 1 ]]; then + # shellcheck source=scripts/lib/gap-reducing-cli.sh + source "${REPO_ROOT}/scripts/lib/gap-reducing-cli.sh" + # shellcheck source=scripts/lib/gap-reducing.sh + source "${REPO_ROOT}/scripts/lib/gap-reducing.sh" + + : "${REDUCE_ROW:?REDUCE_ROW is required}" + : "${PIPELINE:?PIPELINE is required}" + : "${PREPARE_ONLY:?PREPARE_ONLY is required}" + + export BATCH_REDUCE_PYTHON=python + + gap_reducing_apply_env \ + "${PREPARE_ONLY}" \ + "${WITH_CREDUCE:-0}" \ + "${CREDUCE_N:-}" \ + "${PASS_UNDER_TEST:-}" \ + "${MTRIPLE:-}" \ + "${EXTRACT_MIR_OUTPUT:-}" \ + "${MIR_CODEGEN_ONLY:-0}" + + if [[ "${PREPARE_ONLY}" -eq 0 ]]; then + export COVERAGE_LLC=/work/llvm-build-sancov/bin/llc + export COVERAGE_LLVM_REDUCE=/work/llvm-build-sancov/bin/llvm-reduce + fi + + run_gap_reducing \ + /mounted-output/incremental/new_coverage.csv \ + /mounted-output/candidate_tests \ + /mounted-output/reduced \ + "$REDUCE_ROW" \ + "$PIPELINE" + exit 0 +fi + +# shellcheck source=scripts/docker/ensure-image.sh +source "${SCRIPT_DIR}/ensure-image.sh" +# shellcheck source=scripts/docker/docker-image-cli.sh +source "${SCRIPT_DIR}/docker-image-cli.sh" +# shellcheck source=scripts/docker/docker-run.sh +source "${SCRIPT_DIR}/docker-run.sh" + +CONTAINER_WORKDIR="${DOCKER_GAP_CONTAINER_WORKDIR}" +CONTAINER_SCRIPT="$(docker_gap_container_script "$0")" + +docker_image_cli_init_vars +output_dir="" +reduce_row=1 +pipeline="llvm_reduce_ir" +prepare_only=0 +bind_repo=0 +with_creduce=0 +creduce_n="" +pass_under_test="" +mtriple="" +extract_mir_output="" +mir_codegen_only=0 + +usage() { + cat < [options] + +Reduce one row from gap-filling output in a temporary container. +Requires incremental/new_coverage.csv and candidate_tests/ under --output-dir. + +Required: + --output-dir Gap-fill output directory (mounted at /mounted-output/) + +Image (one of): + --image Docker image ref (default: \${IMAGE_NAME:-fuzz-fill-test}:\${IMAGE_TAG:-latest}) + --pr-id Use \${IMAGE_NAME:-fuzz-fill-test}:llvm-pr- + +Image build (optional; reuses existing image when omitted): + --build-image Build PR image via build-image-pr.sh when missing + --force-build Rebuild PR image even when the tag already exists + --keep-image Keep PR image after run (default: remove when --build-image) + --llvm-repo Local llvm-project clone (required with --build-image) + --backend-tests amdgpu or spirv (required with --build-image) + --github-repo GitHub repo hosting the PR (default: llvm/llvm-project) + --image-name Image name when using --pr-id (default: fuzz-fill-test) + +Options: + --row CSV row to reduce (default: 1) + --prepare-only Create harness under reduced/ without running reduce + --pipeline Reduce pipeline pass ids (default: llvm_reduce_ir) + --with-creduce Append creduce to the pipeline + --creduce-n Parallelism for creduce steps + --pass-under-test For MIR pipelines + --mtriple For MIR pipelines + --extract-mir-output Basename for extract_mir_before_pass output + --mir-codegen-only Use codegen-only MIR template + --bind-repo Mount the local fuzz-fill checkout at ${CONTAINER_WORKDIR} + --help, -h Show this help + +Examples: + $(basename "$0") --output-dir ./data/fill-100 + $(basename "$0") --pr-id 203468 --output-dir ./data/fill-100 --prepare-only + $(basename "$0") --bind-repo --output-dir ./data/fill-100 --with-creduce +EOF +} + +while [[ $# -gt 0 ]]; do + if docker_image_cli_try_parse "$1" "${2:-}"; then + shift "${DOCKER_IMAGE_CLI_SHIFT}" + continue + fi + if docker_gap_cli_try_parse_common "$1" "${2:-}"; then + shift "${DOCKER_GAP_CLI_SHIFT}" + continue + fi + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 2; } + output_dir="$2" + shift 2 + ;; + --row) + [[ $# -ge 2 ]] || { echo "error: --row requires a value" >&2; exit 2; } + reduce_row="$2" + shift 2 + ;; + --prepare-only) + prepare_only=1 + shift + ;; + --pipeline) + [[ $# -ge 2 ]] || { echo "error: --pipeline requires a value" >&2; exit 2; } + pipeline="$2" + shift 2 + ;; + --with-creduce) + with_creduce=1 + shift + ;; + --creduce-n) + [[ $# -ge 2 ]] || { echo "error: --creduce-n requires a value" >&2; exit 2; } + creduce_n="$2" + shift 2 + ;; + --pass-under-test) + [[ $# -ge 2 ]] || { echo "error: --pass-under-test requires a value" >&2; exit 2; } + pass_under_test="$2" + shift 2 + ;; + --mtriple) + [[ $# -ge 2 ]] || { echo "error: --mtriple requires a value" >&2; exit 2; } + mtriple="$2" + shift 2 + ;; + --extract-mir-output) + [[ $# -ge 2 ]] || { echo "error: --extract-mir-output requires a value" >&2; exit 2; } + extract_mir_output="$2" + shift 2 + ;; + --mir-codegen-only) + mir_codegen_only=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$output_dir" ]]; then + echo "error: --output-dir is required" >&2 + usage >&2 + exit 1 +fi + +# shellcheck source=scripts/lib/gap-artifacts.sh +source "${REPO_ROOT}/scripts/lib/gap-artifacts.sh" +# shellcheck source=scripts/lib/gap-reducing-cli.sh +source "${REPO_ROOT}/scripts/lib/gap-reducing-cli.sh" + +gap_reducing_validate_row "$reduce_row" +gap_reducing_validate_creduce_n "$creduce_n" + +output_dir="$(realpath "$output_dir")" +coverage_csv="$(gap_fill_coverage_csv "$output_dir")" +candidate_tests_dir="$(gap_fill_candidate_tests_dir "$output_dir")" +validate_gap_fill_artifacts "$coverage_csv" "$candidate_tests_dir" \ + "run scripts/docker/gap-filling.sh first" + +DOCKER_IMAGE_MISSING_HINT="build with ${SCRIPT_DIR}/build-image.sh, or pass --build-image with --llvm-repo and --backend-tests" +docker_image_cli_prepare + +docker_env=(-e "IN_CONTAINER=1") +gap_reducing_append_docker_env docker_env \ + "$reduce_row" "$pipeline" "$prepare_only" "$with_creduce" \ + "$creduce_n" "$pass_under_test" "$mtriple" "$extract_mir_output" "$mir_codegen_only" + +extra_mounts=() +docker_gap_append_bind_repo_mount extra_mounts + +gap_reducing_print_summary "$output_dir" "$reduce_row" "$pipeline" "$prepare_only" "Gap-fill output" + +docker_gap_run "$CONTAINER_SCRIPT" "$output_dir" "$image_ref" docker_env extra_mounts + +echo "Image: ${image_ref}" +echo "Wrote ${output_dir}/reduced/" diff --git a/scripts/docker/pr-cov-gaps-detection.sh b/scripts/docker/pr-cov-gaps-detection.sh deleted file mode 100755 index 35dc9c2b..00000000 --- a/scripts/docker/pr-cov-gaps-detection.sh +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env bash -# Run PR coverage-gap detection inside a one-shot Docker container: -# coverage baseline (allow failures) -> added_lines -> target-lines -# -# Host output is bind-mounted at /mounted-output/ in the container. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -image_ref="" -pr_id="" -output_dir="" -lit_filters=() -image_name="${IMAGE_NAME:-fuzz-fill-test}" -commit_rev="" -jobs="" -build_image=0 -keep_image=0 -llvm_repo="" -backend_tests="" -github_repo="" - -usage() { - cat < | --pr-id ) --output-dir [options] - -Run coverage baseline, added_lines, and target-lines in a temporary container. -Artifacts are written under --output-dir (mounted at /mounted-output/). - -Required (one of): - --image Full Docker image ref (e.g. fuzz-fill-test:llvm-pr-203468) - --pr-id Derive image as \${IMAGE_NAME:-fuzz-fill-test}:llvm-pr- - -Required: - --output-dir Host output directory (created if missing) - -Options: - --build-image Build PR image via build-image-pr.sh before detection - --keep-image Keep the PR image after detection (default: remove it - when --build-image was used) - --llvm-repo Local llvm-project clone (required with --build-image) - --backend-tests amdgpu or spirv (required with --build-image) - --github-repo - GitHub repo hosting the PR (default: llvm/llvm-project) - --lit-filter LIT regex prefix; repeat for multiple (default values available in: scripts/lit-filters-amdgpu.sh for amdgpu, CodeGen/SPIRV for spirv) - --image-name Image name when using --pr-id (default: fuzz-fill-test) - --commit Revision for added_lines (default: HEAD in image llvm-project) - -j , --jobs Parallel jobs for llvm-lit; with --build-image, also for ninja - --help, -h Show this help - -Examples: - $(basename "$0") --build-image --llvm-repo /path/llvm-project --pr-id 203468 \\ - --backend-tests amdgpu --output-dir ./data/pr-cov-gaps-203468 -j "\$(nproc)" - $(basename "$0") --pr-id 203468 --output-dir ./data/pr-cov-gaps-203468 - $(basename "$0") --pr-id 203468 --output-dir ./data/pr-cov-gaps-203468 \\ - --lit-filter CodeGen/AMDGPU -j "\$(nproc)" -EOF -} - -default_lit_filters_for_allowlist() { - case "$1" in - amdgpu) - # shellcheck source=scripts/lit-filters-amdgpu.sh - source "${REPO_ROOT}/scripts/lit-filters-amdgpu.sh" - printf '%s\n' "${AMDGPU_LIT_FILTERS[@]}" - ;; - spirv) - printf '%s\n' "CodeGen/SPIRV" - ;; - *) - echo "error: unsupported image allowlist: ${1} (expected amdgpu or spirv)" >&2 - return 1 - ;; - esac -} - -validate_jobs() { - if [[ -n "$jobs" ]] && { [[ ! "$jobs" =~ ^[0-9]+$ ]] || [[ "$jobs" -eq 0 ]]; }; then - echo "error: -j/--jobs must be a positive integer: ${jobs}" >&2 - exit 1 - fi -} - -cleanup_built_image() { - if [[ "${build_image:-0}" -eq 1 && "${keep_image:-0}" -eq 0 && -n "${image_ref:-}" ]]; then - if docker image inspect "${image_ref}" >/dev/null 2>&1; then - echo "Removing Docker image ${image_ref}" - docker rmi "${image_ref}" - fi - fi -} - -read_image_allowlist() { - local allowlist - if ! allowlist="$(docker run --rm --entrypoint cat "${image_ref}" /work/.sancov-allowlist 2>/dev/null | tr -d '[:space:]')"; then - echo "error: failed to read /work/.sancov-allowlist from image: ${image_ref}" >&2 - exit 1 - fi - if [[ -z "$allowlist" ]]; then - echo "error: /work/.sancov-allowlist is empty in image: ${image_ref}" >&2 - exit 1 - fi - printf '%s' "$allowlist" -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --image) - [[ $# -ge 2 ]] || { echo "error: --image requires a value" >&2; exit 2; } - image_ref="$2" - shift 2 - ;; - --pr-id) - [[ $# -ge 2 ]] || { echo "error: --pr-id requires a value" >&2; exit 2; } - pr_id="$2" - shift 2 - ;; - --output-dir) - [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 2; } - output_dir="$2" - shift 2 - ;; - --build-image) - build_image=1 - shift - ;; - --keep-image) - keep_image=1 - shift - ;; - --llvm-repo) - [[ $# -ge 2 ]] || { echo "error: --llvm-repo requires a value" >&2; exit 2; } - llvm_repo="$2" - shift 2 - ;; - --backend-tests) - [[ $# -ge 2 ]] || { echo "error: --backend-tests requires a value" >&2; exit 2; } - backend_tests="$2" - shift 2 - ;; - --github-repo) - [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; } - github_repo="$2" - shift 2 - ;; - --lit-filter) - [[ $# -ge 2 ]] || { echo "error: --lit-filter requires a value" >&2; exit 2; } - lit_filters+=("$2") - shift 2 - ;; - --image-name) - [[ $# -ge 2 ]] || { echo "error: --image-name requires a value" >&2; exit 2; } - image_name="$2" - shift 2 - ;; - --commit) - [[ $# -ge 2 ]] || { echo "error: --commit requires a value" >&2; exit 2; } - commit_rev="$2" - shift 2 - ;; - -j|--jobs) - [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 2; } - jobs="$2" - shift 2 - ;; - --help|-h) - usage - exit 0 - ;; - --) - shift - break - ;; - -*) - echo "error: unknown option: $1" >&2 - usage >&2 - exit 2 - ;; - *) - echo "error: unexpected argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -if [[ $# -gt 0 ]]; then - echo "error: unexpected argument: $1" >&2 - usage >&2 - exit 2 -fi - -if [[ -n "$image_ref" && -n "$pr_id" ]]; then - echo "error: pass only one of --image or --pr-id" >&2 - exit 1 -fi - -if [[ -n "$image_ref" && "$build_image" -eq 1 ]]; then - echo "error: --build-image cannot be used with --image" >&2 - exit 1 -fi - -if [[ -z "$image_ref" && -z "$pr_id" ]]; then - echo "error: one of --image or --pr-id is required" >&2 - usage >&2 - exit 1 -fi - -if [[ -z "$output_dir" ]]; then - echo "error: --output-dir is required" >&2 - usage >&2 - exit 1 -fi - -if [[ "$build_image" -eq 0 ]]; then - if [[ -n "$llvm_repo" || -n "$backend_tests" || -n "$github_repo" || "$keep_image" -eq 1 ]]; then - echo "error: --llvm-repo, --backend-tests, --github-repo, and --keep-image require --build-image" >&2 - exit 1 - fi -else - if [[ -z "$llvm_repo" ]]; then - echo "error: --llvm-repo is required with --build-image" >&2 - exit 1 - fi - if [[ -z "$pr_id" ]]; then - echo "error: --pr-id is required with --build-image" >&2 - exit 1 - fi - if [[ -z "$backend_tests" ]]; then - echo "error: --backend-tests is required with --build-image" >&2 - exit 1 - fi -fi - -validate_jobs - -if [[ -n "$backend_tests" ]]; then - backend_tests="$(printf '%s' "$backend_tests" | tr '[:upper:]' '[:lower:]')" - case "$backend_tests" in - amdgpu|spirv) ;; - *) - echo "error: --backend-tests must be amdgpu or spirv: ${backend_tests}" >&2 - exit 1 - ;; - esac -fi - -if [[ -n "$pr_id" ]]; then - if [[ ! "$pr_id" =~ ^[0-9]+$ ]] || [[ "$pr_id" -eq 0 ]]; then - echo "error: --pr-id must be a positive integer: ${pr_id}" >&2 - exit 1 - fi - image_ref="${image_name}:llvm-pr-${pr_id}" -fi - -if [[ "$build_image" -eq 1 && "$keep_image" -eq 0 ]]; then - trap cleanup_built_image EXIT -fi - -if [[ "$build_image" -eq 1 ]]; then - build_args=( - --llvm-repo "$llvm_repo" - --pr-id "$pr_id" - --allowlist "$backend_tests" - ) - if [[ -n "$github_repo" ]]; then - build_args+=(--github-repo "$github_repo") - fi - if [[ -n "$jobs" ]]; then - build_args+=(-j "$jobs") - fi - - echo "=== build PR image ===" - "${SCRIPT_DIR}/build-image-pr.sh" "${build_args[@]}" -fi - -if ! docker image inspect "${image_ref}" >/dev/null 2>&1; then - echo "error: image not found: ${image_ref}" >&2 - if [[ "$build_image" -eq 0 ]]; then - echo "hint: pass --build-image with --llvm-repo and --backend-tests to build it first" >&2 - fi - exit 1 -fi - -if [[ ${#lit_filters[@]} -eq 0 ]]; then - image_allowlist="$(read_image_allowlist)" - mapfile -t lit_filters < <(default_lit_filters_for_allowlist "$image_allowlist") - echo "Image allowlist: ${image_allowlist} -> ${#lit_filters[@]} lit-filter prefix(es)" -fi - -mkdir -p "$output_dir" -output_dir="$(realpath "$output_dir")" - -lit_filters_file="${output_dir}/.lit-filters" -printf '%s\n' "${lit_filters[@]}" > "$lit_filters_file" - -docker_env=() -if [[ -n "$commit_rev" ]]; then - docker_env+=(-e "COMMIT_REV=${commit_rev}") -fi -if [[ -n "$jobs" ]]; then - docker_env+=(-e "JOBS=${jobs}") -fi - -docker run --rm \ - -v "${output_dir}:/mounted-output" \ - "${docker_env[@]}" \ - -w /work/fuzz-fill \ - "${image_ref}" \ - bash -lc ' -set -euo pipefail - -commit="${COMMIT_REV:-$(git -C /work/llvm-project rev-parse HEAD)}" - -mapfile -t lit_filters < /mounted-output/.lit-filters - -baseline_args=( - python -m coverage baseline - --output-dir /mounted-output/baseline - --lit-allow-failures -) -for lit_filter in "${lit_filters[@]}"; do - baseline_args+=(--lit-filter "${lit_filter}") -done -if [[ -n "${JOBS:-}" ]]; then - baseline_args+=(-j "${JOBS}") -fi - -echo "=== coverage baseline (${#lit_filters[@]} lit-filter prefix(es)) ===" -"${baseline_args[@]}" - -echo "=== added_lines (commit=${commit}) ===" -python -m added_lines \ - --commit "${commit}" \ - --output-dir /mounted-output/added-lines - -echo "=== coverage target-lines ===" -python -m coverage target-lines \ - --output-dir /mounted-output/commit_lines_report \ - --line-coverage-uncovered-csv /mounted-output/baseline/line_coverage_uncovered.csv \ - --target-lines-csv /mounted-output/added-lines/added-lines.csv -' - -report="${output_dir}/commit_lines_report/target_lines_uncovered.csv" -echo "Wrote ${report}" -echo "Image: ${image_ref}" -echo "LIT filters: ${lit_filters[*]}" - -lit_failures_json="${output_dir}/baseline/lit_failures.json" -warning_file="${output_dir}/README-WARNING" - -fail_count=0 -if [[ -f "$lit_failures_json" ]]; then - fail_count="$(python3 -c ' -import json, sys -FAILURE_CODES = {"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"} -try: - with open(sys.argv[1]) as f: - data = json.load(f) - print(sum(1 for t in data.get("tests", []) if t.get("code") in FAILURE_CODES)) -except Exception: - print(0) -' "$lit_failures_json")" -fi - -if [[ "$fail_count" -gt 0 ]]; then - msg="WARNING: ${fail_count} LIT test(s) failed during the baseline run (e.g., failed a check, timed out, unresolved, or unexpectedly passed). -Failed tests may lead to an incomplete coverage profile. As a result, -target_lines_uncovered.csv may report lines as uncovered even though they are -actually covered by a (failing) test. -Review baseline/lit_failures.json for the list of failing tests." - - if [[ -t 1 ]]; then - printf '%b\n' "\033[1;31m${msg}\033[0m" - else - printf '%s\n' "$msg" - fi - - printf '%s\n' "$msg" > "$warning_file" - echo "Wrote ${warning_file}" -fi diff --git a/scripts/gap-filling-amdgpu.sh b/scripts/gap-filling-amdgpu.sh new file mode 100755 index 00000000..e2ae4867 --- /dev/null +++ b/scripts/gap-filling-amdgpu.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Gap filling (AMDGPU): candidate-test -> incremental for a fuzz corpus. +# +# Requires a gap list from gap finding (baseline or PR): +# baseline/line_coverage_uncovered.csv or commit_lines_report/target_lines_uncovered.csv +# plus baseline/llc_address_line_map.csv from the same baseline run. +# +# Usage: +# ./scripts/gap-filling-amdgpu.sh +# OUTPUT_DIR=./data/my_run ./scripts/gap-filling-amdgpu.sh +# JOBS="$(nproc)" ./scripts/gap-filling-amdgpu.sh +# SKIP_BASELINE=1 ./scripts/gap-filling-amdgpu.sh # reuse existing gap list under output/baseline/ +# REFRESH=all ./scripts/gap-filling-amdgpu.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-${REPO_ROOT}/data/coverage_output/bb_coverage_amdgpu_$(date +%y%m%d)}" +BASELINE_OUTPUT_DIR="$OUTPUT_DIR/baseline" +CANDIDATE_TESTS_OUTPUT_DIR="$OUTPUT_DIR/candidate_tests" +INCREMENTAL_OUTPUT_DIR="$OUTPUT_DIR/incremental" +TESTS_DIR="${TESTS_DIR:-${REPO_ROOT}/../irtests/bitcode/amdgpu/all}" +CORPUS_N="${CORPUS_N:-100}" + +# shellcheck source=scripts/lit-filters-amdgpu.sh +source "${SCRIPT_DIR}/lit-filters-amdgpu.sh" +LIT_FILTERS=("${AMDGPU_LIT_FILTERS[@]}") + +# shellcheck source=scripts/lib/local-llvm-env.sh +source "${SCRIPT_DIR}/lib/local-llvm-env.sh" +# shellcheck source=scripts/lib/coverage-baseline.sh +source "${SCRIPT_DIR}/lib/coverage-baseline.sh" +# shellcheck source=scripts/lib/gap-filling.sh +source "${SCRIPT_DIR}/lib/gap-filling.sh" + +activate_venv_if_present "$REPO_ROOT" +setup_local_llvm_env "$REPO_ROOT" "build-sancov/bin" +require_local_coverage_bins + +case "${REFRESH:-}" in + all) + rm -rf "$BASELINE_OUTPUT_DIR" "$CANDIDATE_TESTS_OUTPUT_DIR" "$INCREMENTAL_OUTPUT_DIR" + ;; + baseline) + rm -rf "$BASELINE_OUTPUT_DIR" + ;; + candidate) + rm -rf "$CANDIDATE_TESTS_OUTPUT_DIR" + ;; + incremental) + rm -rf "$INCREMENTAL_OUTPUT_DIR" + ;; + "") + ;; + *) + echo "error: unknown REFRESH=$REFRESH (use all, baseline, candidate, or incremental)" >&2 + exit 1 + ;; +esac + +mkdir -p "$OUTPUT_DIR" +cd "$REPO_ROOT" + +echo "=== AMDGPU gap filling ===" +echo "output: $OUTPUT_DIR" +echo "llvm-bin: $LLVM_BIN" +echo "instrumented: $INSTRUMENTED_BIN_DIR" +echo "candidate dir: $TESTS_DIR (n=$CORPUS_N)" +echo "lit filters: ${LIT_FILTERS[*]}" +echo + +if [[ -z "${SKIP_BASELINE:-}" ]]; then + echo ">>> Step 1/3: gap finding — baseline (LIT)" + run_coverage_baseline "$BASELINE_OUTPUT_DIR" "${LIT_FILTERS[@]}" +else + echo ">>> Step 1/3: skipped (SKIP_BASELINE=1; using existing gap list under $BASELINE_OUTPUT_DIR)" +fi + +if [[ -z "${SKIP_CANDIDATE:-}" ]]; then + echo ">>> Step 2/3: candidate-test" + run_candidate_test "$CANDIDATE_TESTS_OUTPUT_DIR" "$TESTS_DIR" "$CORPUS_N" +else + echo ">>> Step 2/3: skipped (SKIP_CANDIDATE=1)" +fi + +if [[ -z "${SKIP_INCREMENTAL:-}" ]]; then + echo ">>> Step 3/3: incremental" + run_incremental \ + "$INCREMENTAL_OUTPUT_DIR" \ + "$BASELINE_OUTPUT_DIR/line_coverage_uncovered.csv" \ + "$BASELINE_OUTPUT_DIR/llc_address_line_map.csv" \ + "$CANDIDATE_TESTS_OUTPUT_DIR" +else + echo ">>> Step 3/3: skipped (SKIP_INCREMENTAL=1)" +fi + +echo +echo "Gap-fill report: $INCREMENTAL_OUTPUT_DIR/new_coverage.csv" diff --git a/scripts/gap-finding-baseline.sh b/scripts/gap-finding-baseline.sh new file mode 100755 index 00000000..cd6f9757 --- /dev/null +++ b/scripts/gap-finding-baseline.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Gap finding (baseline): run coverage baseline over a filtered LIT slice. +# +# Produces line_coverage_uncovered.csv and llc_address_line_map.csv under +# /baseline/. Use scripts/gap-filling-amdgpu.sh (or gap-filling Docker) +# to find fuzz tests that cover those gaps. +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUTPUT_DIR="${REPO_ROOT}/data/coverage_output/bb_coverage_amdgpu_230726_full_filter" +BASELINE_OUTPUT_DIR=$OUTPUT_DIR/baseline + +FILTER="${FILTER:-AMDGPU}" + +# shellcheck source=scripts/lib/local-llvm-env.sh +source "${SCRIPT_DIR}/lib/local-llvm-env.sh" +# shellcheck source=scripts/lib/coverage-baseline.sh +source "${SCRIPT_DIR}/lib/coverage-baseline.sh" + +rm -rf "$BASELINE_OUTPUT_DIR" + +cd "$REPO_ROOT" + +setup_local_llvm_env_with_lit_failures "$REPO_ROOT" "build-amdgpu-bb/bin" + +run_coverage_baseline "$BASELINE_OUTPUT_DIR" "$FILTER" + +echo "Uncovered baseline lines: $BASELINE_OUTPUT_DIR/line_coverage_uncovered.csv" diff --git a/scripts/gap-finding-pr.sh b/scripts/gap-finding-pr.sh new file mode 100755 index 00000000..b49b48cc --- /dev/null +++ b/scripts/gap-finding-pr.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Gap finding (PR-specific): baseline -> added_lines -> target-lines. +# +# Produces commit_lines_report/target_lines_uncovered.csv — added source lines +# the baseline still does not cover. Run from fuzz-fill repo root. +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUTPUT_DIR="${REPO_ROOT}/data/coverage_output/bb_coverage_commit_lines_170626" +TARGET_LINES_REPORT_DIR=$OUTPUT_DIR/commit_lines_report + +# Faster CodeGen-only subset: FILTER=CodeGen/AMDGPU +FILTER="${FILTER:-AMDGPU}" + +COMMIT=b01fe4e + +# shellcheck source=scripts/lib/local-llvm-env.sh +source "${SCRIPT_DIR}/lib/local-llvm-env.sh" +# shellcheck source=scripts/lib/gap-finding-pr.sh +source "${SCRIPT_DIR}/lib/gap-finding-pr.sh" + +cd "$REPO_ROOT" + +setup_local_llvm_env_with_lit_failures "$REPO_ROOT" "build-amdgpu-bb/bin" + +run_gap_finding_pr "$OUTPUT_DIR" "$COMMIT" "${LLVM_REPO}" "$FILTER" + +echo "Uncovered PR target lines: $TARGET_LINES_REPORT_DIR/target_lines_uncovered.csv" diff --git a/scripts/gap-reducing-amdgpu.sh b/scripts/gap-reducing-amdgpu.sh new file mode 100755 index 00000000..a435d34e --- /dev/null +++ b/scripts/gap-reducing-amdgpu.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Gap reducing (AMDGPU): reduce one row from gap-filling output. +# +# Requires gap-filling artifacts under --output-dir: +# incremental/new_coverage.csv +# candidate_tests/ +# +# Usage: +# ./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run +# ./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run --row 2 +# ./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run --prepare-only +# WITH_CREDUCE=1 ./scripts/gap-reducing-amdgpu.sh --output-dir ./data/my_run +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +output_dir="" +reduce_row="${REDUCE_ROW:-1}" +pipeline="${PIPELINE:-llvm_reduce_ir}" +prepare_only=0 + +usage() { + cat < [options] + +Reduce one row from gap-filling output (new_coverage.csv + candidate_tests/). + +Required: + --output-dir Gap-fill output root (incremental/ and candidate_tests/ below) + +Options: + --row CSV row to reduce (default: 1, or \$REDUCE_ROW) + --prepare-only Create harness under reduced/ without running reduce + --pipeline Reduce pipeline pass ids (default: llvm_reduce_ir) + --help, -h Show this help + +Environment (optional): + OUTPUT_DIR Default for --output-dir when omitted + REDUCE_ROW Default for --row + PIPELINE Default pipeline (default: llvm_reduce_ir) + WITH_CREDUCE=1 Append creduce to the pipeline + CREDUCE_N Parallelism for creduce steps + PASS_UNDER_TEST, MTRIPLE, EXTRACT_MIR_OUTPUT, MIR_CODEGEN_ONLY + Forwarded for MIR pipelines (see batch_reduce_using_coverage.sh) + +Examples: + $(basename "$0") --output-dir ./data/my_run + $(basename "$0") --output-dir ./data/my_run --prepare-only + PIPELINE=llvm_reduce_ir,creduce $(basename "$0") --output-dir ./data/my_run +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir) + [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 2; } + output_dir="$2" + shift 2 + ;; + --row) + [[ $# -ge 2 ]] || { echo "error: --row requires a value" >&2; exit 2; } + reduce_row="$2" + shift 2 + ;; + --prepare-only) + prepare_only=1 + shift + ;; + --pipeline) + [[ $# -ge 2 ]] || { echo "error: --pipeline requires a value" >&2; exit 2; } + pipeline="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$output_dir" ]]; then + output_dir="${OUTPUT_DIR:-}" +fi +if [[ -z "$output_dir" ]]; then + echo "error: --output-dir is required" >&2 + usage >&2 + exit 1 +fi + +# shellcheck source=scripts/lib/local-llvm-env.sh +source "${SCRIPT_DIR}/lib/local-llvm-env.sh" +# shellcheck source=scripts/lib/gap-artifacts.sh +source "${SCRIPT_DIR}/lib/gap-artifacts.sh" +# shellcheck source=scripts/lib/gap-reducing-cli.sh +source "${SCRIPT_DIR}/lib/gap-reducing-cli.sh" + +gap_reducing_validate_row "$reduce_row" + +coverage_csv="$(gap_fill_coverage_csv "$output_dir")" +candidate_tests_dir="$(gap_fill_candidate_tests_dir "$output_dir")" +reduced_dir="$(gap_fill_reduced_dir "$output_dir")" +validate_gap_fill_artifacts "$coverage_csv" "$candidate_tests_dir" \ + "run gap-filling first (e.g. scripts/gap-filling-amdgpu.sh)" +# shellcheck source=scripts/lib/gap-reducing.sh +source "${SCRIPT_DIR}/lib/gap-reducing.sh" + +activate_venv_if_present "$REPO_ROOT" +setup_local_llvm_env "$REPO_ROOT" "build-sancov/bin" + +gap_reducing_apply_env "$prepare_only" + +if [[ "$prepare_only" -eq 0 ]]; then + require_local_reduce_bins + export_local_reduce_tools +fi + +cd "$REPO_ROOT" + +echo "=== AMDGPU gap reducing (row ${reduce_row}) ===" +echo "coverage csv: $coverage_csv" +echo "candidate dir: $candidate_tests_dir" +echo "output: $reduced_dir" +echo "pipeline: $pipeline" +if [[ "$prepare_only" -eq 1 ]]; then + echo "mode: prepare-only" +fi +echo + +run_gap_reducing "$coverage_csv" "$candidate_tests_dir" "$reduced_dir" "$reduce_row" "$pipeline" diff --git a/scripts/lib/candidate-inputs.sh b/scripts/lib/candidate-inputs.sh new file mode 100644 index 00000000..0e08bd56 --- /dev/null +++ b/scripts/lib/candidate-inputs.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Host-side candidate corpus enumeration and staging for Docker gap-filling. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +# Enumerate .ll/.bc under src (recursive), sorted — matches TestRunner.collect_llc_input_files(). +collect_candidate_inputs() { + local src="$1" + find "$src" -type f \( -name '*.ll' -o -name '*.bc' \) | LC_ALL=C sort +} + +# Copy the first n candidate inputs into staging_dir, preserving relative paths. +stage_candidate_tests() { + local src="$1" + local n="$2" + local staging_dir="$3" + local src_real count rel dest_parent + + src_real="$(realpath "$src")" + mapfile -t candidate_files < <(collect_candidate_inputs "$src_real") + + if [[ "${#candidate_files[@]}" -eq 0 ]]; then + echo "error: no .ll or .bc files under --candidate-tests-dir: ${src_real}" >&2 + exit 1 + fi + + count="$n" + if [[ "${#candidate_files[@]}" -lt "$count" ]]; then + count="${#candidate_files[@]}" + echo "note: corpus has ${#candidate_files[@]} file(s); staging all of them (requested ${n})" + fi + + local i + for (( i = 0; i < count; i++ )); do + rel="${candidate_files[$i]#"${src_real}/"}" + dest_parent="${staging_dir}/$(dirname "$rel")" + mkdir -p "$dest_parent" + cp -- "${candidate_files[$i]}" "${staging_dir}/${rel}" + done + + echo "Staged ${count} candidate test file(s) under ${staging_dir}" +} diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 00000000..460969b8 --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Shared helpers for fuzz-fill pipeline scripts. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +: "${SCRIPTS_DIR:=$(cd "${LIB_DIR}/.." && pwd)}" +: "${REPO_ROOT:=$(cd "${SCRIPTS_DIR}/.." && pwd)}" + +validate_jobs() { + local jobs="$1" + if [[ -n "$jobs" ]] && { [[ ! "$jobs" =~ ^[0-9]+$ ]] || [[ "$jobs" -eq 0 ]]; }; then + echo "error: -j/--jobs must be a positive integer: ${jobs}" >&2 + exit 1 + fi +} + +require_bin() { + local dir="$1" tool="$2" + if [[ ! -x "$dir/$tool" ]]; then + echo "error: missing $dir/$tool" >&2 + exit 1 + fi +} + +activate_venv_if_present() { + local repo_root="${1:-$REPO_ROOT}" + if [[ -f "$repo_root/venv/bin/activate" ]]; then + # shellcheck disable=SC1091 + source "$repo_root/venv/bin/activate" + fi +} diff --git a/scripts/lib/coverage-baseline.sh b/scripts/lib/coverage-baseline.sh new file mode 100644 index 00000000..0b5d5c6b --- /dev/null +++ b/scripts/lib/coverage-baseline.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# coverage baseline pipeline step. +# Source from entrypoints or other scripts/lib modules; do not execute directly. +# +# Optional env (local runs set these; Docker omits them and uses image PATH): +# COVERAGE_SANCOV, COVERAGE_LLVM_LIT, COVERAGE_LLC, COVERAGE_OPT +# LIT_ALLOW_FAILURES — any non-empty value adds --lit-allow-failures +# JOBS — parallel llvm-lit jobs (-j) + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +run_coverage_baseline() { + local output_dir="$1" + shift + local -a lit_filters=("$@") + + local -a args=( + python -m coverage baseline + --output-dir "$output_dir" + ) + + if [[ -n "${COVERAGE_SANCOV:-}" ]]; then + args+=(--sancov "$COVERAGE_SANCOV") + fi + if [[ -n "${COVERAGE_LLVM_LIT:-}" ]]; then + args+=(--llvm-lit "$COVERAGE_LLVM_LIT") + fi + if [[ -n "${COVERAGE_LLC:-}" ]]; then + args+=(--llc "$COVERAGE_LLC") + fi + if [[ -n "${COVERAGE_OPT:-}" ]]; then + args+=(--opt "$COVERAGE_OPT") + fi + + local lit_filter + for lit_filter in "${lit_filters[@]}"; do + args+=(--lit-filter "$lit_filter") + done + + if [[ -n "${LIT_ALLOW_FAILURES:-}" ]]; then + args+=(--lit-allow-failures) + fi + if [[ -n "${JOBS:-}" ]]; then + args+=(-j "$JOBS") + fi + + "${args[@]}" +} diff --git a/scripts/lib/gap-artifacts.sh b/scripts/lib/gap-artifacts.sh new file mode 100644 index 00000000..ce3b135b --- /dev/null +++ b/scripts/lib/gap-artifacts.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Gap-filling output paths and profile CSV validation. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +gap_fill_coverage_csv() { + printf '%s/incremental/new_coverage.csv' "$1" +} + +gap_fill_candidate_tests_dir() { + printf '%s/candidate_tests' "$1" +} + +gap_fill_reduced_dir() { + printf '%s/reduced' "$1" +} + +validate_gap_fill_artifacts() { + local coverage_csv="$1" + local candidate_tests_dir="$2" + local fill_hint="${3:-run gap-filling first}" + + if [[ ! -f "$coverage_csv" ]]; then + echo "error: gap-fill CSV not found: ${coverage_csv}" >&2 + echo "hint: ${fill_hint}" >&2 + exit 1 + fi + if [[ ! -d "$candidate_tests_dir" ]]; then + echo "error: candidate_tests directory not found: ${candidate_tests_dir}" >&2 + exit 1 + fi +} + +validate_gap_profile_csv() { + local flag_name="$1" + local path="$2" + if [[ ! -f "$path" ]]; then + echo "error: ${flag_name} not found: ${path}" >&2 + exit 1 + fi +} + +# Resolve and validate gap-list profile CSVs (names of caller variables). +resolve_gap_profile_csv_paths() { + local uncovered_var="$1" + local llc_map_var="$2" + local uncovered llc_map + + uncovered="${!uncovered_var:-}" + llc_map="${!llc_map_var:-}" + + if [[ -z "$uncovered" ]]; then + echo "error: --line-coverage-uncovered-csv is required" >&2 + exit 1 + fi + if [[ -z "$llc_map" ]]; then + echo "error: --llc-address-line-map-csv is required" >&2 + exit 1 + fi + + validate_gap_profile_csv "--line-coverage-uncovered-csv" "$uncovered" + validate_gap_profile_csv "--llc-address-line-map-csv" "$llc_map" + + printf -v "$uncovered_var" '%s' "$(realpath "$uncovered")" + printf -v "$llc_map_var" '%s' "$(realpath "$llc_map")" +} diff --git a/scripts/lib/gap-filling.sh b/scripts/lib/gap-filling.sh new file mode 100644 index 00000000..0222b513 --- /dev/null +++ b/scripts/lib/gap-filling.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Gap-filling pipeline steps: candidate-test and incremental. +# Source from entrypoints or other scripts/lib modules; do not execute directly. +# +# Optional env (local runs): +# COVERAGE_SANCOV, COVERAGE_LLC +# JOBS + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +run_candidate_test() { + local output_dir="$1" + local candidate_tests_dir="$2" + local n="$3" + + local -a args=( + python -m coverage candidate-test + --output-dir "$output_dir" + --candidate-tests-dir "$candidate_tests_dir" + --n "$n" + ) + + if [[ -n "${COVERAGE_LLC:-}" ]]; then + args+=(--llc "$COVERAGE_LLC") + fi + if [[ -n "${JOBS:-}" ]]; then + args+=(-j "$JOBS") + fi + + echo "=== coverage candidate-test (n=${n}) ===" + "${args[@]}" +} + +run_incremental() { + local output_dir="$1" + local uncovered_csv="$2" + local llc_map_csv="$3" + local candidate_tests_output_dir="$4" + + local -a args=( + python -m coverage incremental + --output-dir "$output_dir" + --line-coverage-uncovered-csv "$uncovered_csv" + --llc-address-line-map-csv "$llc_map_csv" + --candidate-tests-output-dir "$candidate_tests_output_dir" + ) + + if [[ -n "${COVERAGE_SANCOV:-}" ]]; then + args+=(--sancov "$COVERAGE_SANCOV") + fi + + echo "=== coverage incremental ===" + "${args[@]}" +} diff --git a/scripts/lib/gap-finding-pr.sh b/scripts/lib/gap-finding-pr.sh new file mode 100644 index 00000000..640bd4d1 --- /dev/null +++ b/scripts/lib/gap-finding-pr.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# PR gap-finding pipeline: baseline -> added_lines -> target-lines. +# Source from entrypoints or other scripts/lib modules; do not execute directly. +# +# run_gap_finding_pr [llvm_repo] [lit_filter ...] +# Optional env: same tool-path and JOBS vars as coverage-baseline.sh; +# LIT_ALLOW_FAILURES applies to the baseline step. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/coverage-baseline.sh +source "${LIB_DIR}/coverage-baseline.sh" + +run_gap_finding_pr() { + local output_root="$1" + local commit="$2" + local llvm_repo="${3:-}" + shift 3 + local -a lit_filters=("$@") + + local baseline_dir="${output_root}/baseline" + local added_lines_dir="${output_root}/added-lines" + local target_lines_dir="${output_root}/commit_lines_report" + + mkdir -p "$added_lines_dir" "$target_lines_dir" + + if [[ ${#lit_filters[@]} -gt 0 ]]; then + echo "=== coverage baseline (${#lit_filters[@]} lit-filter prefix(es)) ===" + else + echo "=== coverage baseline ===" + fi + run_coverage_baseline "$baseline_dir" "${lit_filters[@]}" + + echo "=== added_lines (commit=${commit}) ===" + local -a added_args=( + python -m added_lines + --commit "$commit" + --output-dir "$added_lines_dir" + ) + if [[ -n "$llvm_repo" ]]; then + added_args+=(--llvm-repo "$llvm_repo") + fi + "${added_args[@]}" + + echo "=== coverage target-lines ===" + local -a target_args=( + python -m coverage target-lines + --output-dir "$target_lines_dir" + --line-coverage-uncovered-csv "${baseline_dir}/line_coverage_uncovered.csv" + --target-lines-csv "${added_lines_dir}/added-lines.csv" + ) + if [[ -n "$llvm_repo" ]]; then + target_args+=(--llvm-repo "$llvm_repo") + fi + "${target_args[@]}" +} diff --git a/scripts/lib/gap-reducing-cli.sh b/scripts/lib/gap-reducing-cli.sh new file mode 100644 index 00000000..745e9d7a --- /dev/null +++ b/scripts/lib/gap-reducing-cli.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Shared gap-reducing validation, env setup, and Docker env helpers. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +gap_reducing_validate_row() { + local row="$1" + if [[ ! "$row" =~ ^[0-9]+$ ]] || [[ "$row" -eq 0 ]]; then + echo "error: --row must be a positive integer: ${row}" >&2 + exit 1 + fi +} + +gap_reducing_validate_creduce_n() { + local n="$1" + if [[ -n "$n" ]] && { [[ ! "$n" =~ ^[0-9]+$ ]] || [[ "$n" -eq 0 ]]; }; then + echo "error: --creduce-n must be a positive integer: ${n}" >&2 + exit 1 + fi +} + +# Apply reduce extras to the environment for run_gap_reducing (local host or in-container). +# Optional args fall back to existing env when omitted (local WITH_CREDUCE=1 workflow). +gap_reducing_apply_env() { + local prepare_only="${1:-0}" + local with_creduce="${2:-${WITH_CREDUCE:-0}}" + local creduce_n="${3:-${CREDUCE_N:-}}" + local pass_under_test="${4:-${PASS_UNDER_TEST:-}}" + local mtriple="${5:-${MTRIPLE:-}}" + local extract_mir_output="${6:-${EXTRACT_MIR_OUTPUT:-}}" + local mir_codegen_only="${7:-${MIR_CODEGEN_ONLY:-0}}" + + if [[ "$prepare_only" -eq 1 ]]; then + export PREPARE_ONLY=1 + else + unset PREPARE_ONLY + fi + + if [[ "$with_creduce" == 1 ]]; then + export WITH_CREDUCE=1 + else + unset WITH_CREDUCE + fi + + if [[ -n "$creduce_n" ]]; then + export CREDUCE_N="$creduce_n" + else + unset CREDUCE_N + fi + + if [[ -n "$pass_under_test" ]]; then + export PASS_UNDER_TEST="$pass_under_test" + else + unset PASS_UNDER_TEST + fi + + if [[ -n "$mtriple" ]]; then + export MTRIPLE="$mtriple" + else + unset MTRIPLE + fi + + if [[ -n "$extract_mir_output" ]]; then + export EXTRACT_MIR_OUTPUT="$extract_mir_output" + else + unset EXTRACT_MIR_OUTPUT + fi + + if [[ "$mir_codegen_only" == 1 ]]; then + export MIR_CODEGEN_ONLY=1 + else + unset MIR_CODEGEN_ONLY + fi +} + +# Append reduce-related -e flags to a docker_env array (nameref). +gap_reducing_append_docker_env() { + local -n _env=$1 + local reduce_row="$2" + local pipeline="$3" + local prepare_only="${4:-0}" + local with_creduce="${5:-0}" + local creduce_n="${6:-}" + local pass_under_test="${7:-}" + local mtriple="${8:-}" + local extract_mir_output="${9:-}" + local mir_codegen_only="${10:-0}" + + _env+=( + -e "REDUCE_ROW=${reduce_row}" + -e "PIPELINE=${pipeline}" + -e "PREPARE_ONLY=${prepare_only}" + -e "WITH_CREDUCE=${with_creduce}" + ) + + if [[ -n "$creduce_n" ]]; then + _env+=(-e "CREDUCE_N=${creduce_n}") + fi + if [[ -n "$pass_under_test" ]]; then + _env+=(-e "PASS_UNDER_TEST=${pass_under_test}") + fi + if [[ -n "$mtriple" ]]; then + _env+=(-e "MTRIPLE=${mtriple}") + fi + if [[ -n "$extract_mir_output" ]]; then + _env+=(-e "EXTRACT_MIR_OUTPUT=${extract_mir_output}") + fi + if [[ "$mir_codegen_only" -eq 1 ]]; then + _env+=(-e "MIR_CODEGEN_ONLY=1") + fi +} + +gap_reducing_print_summary() { + local output_dir="$1" + local reduce_row="$2" + local pipeline="$3" + local prepare_only="${4:-0}" + local output_label="${5:-Gap-fill output}" + + echo "${output_label}: ${output_dir}" + echo "Reducing row: ${reduce_row}" + echo "Pipeline: ${pipeline}" + if [[ "$prepare_only" -eq 1 ]]; then + echo "Mode: prepare-only" + fi +} diff --git a/scripts/lib/gap-reducing.sh b/scripts/lib/gap-reducing.sh new file mode 100644 index 00000000..08aa99d4 --- /dev/null +++ b/scripts/lib/gap-reducing.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Gap-reducing pipeline step via batch_reduce_using_coverage.py. +# Source from entrypoints or other scripts/lib modules; do not execute directly. +# +# run_gap_reducing +# +# Optional env: +# BATCH_REDUCE_SCRIPT — path to batch_reduce_using_coverage.py +# (default: scripts/batch_reduce_using_coverage.py under SCRIPTS_DIR) +# COVERAGE_LLC, COVERAGE_LLVM_REDUCE — tool paths (omitted in prepare-only or Docker) +# WITH_CREDUCE, CREDUCE_N, PASS_UNDER_TEST, MTRIPLE, EXTRACT_MIR_OUTPUT, MIR_CODEGEN_ONLY +# PREPARE_ONLY — any non-empty value skips --llc/--llvm-reduce + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +run_gap_reducing() { + local coverage_csv="$1" + local candidate_tests_dir="$2" + local reduced_dir="$3" + local reduce_row="$4" + local pipeline="$5" + + local batch_script="${BATCH_REDUCE_SCRIPT:-${SCRIPTS_DIR}/batch_reduce_using_coverage.py}" + local python="${BATCH_REDUCE_PYTHON:-python3}" + + local -a batch_args=( + "$python" "$batch_script" + --csv "$coverage_csv" + --candidate-tests "$candidate_tests_dir" + --output "$reduced_dir" + --n "$reduce_row" + --pipeline "$pipeline" + ) + + if [[ "${WITH_CREDUCE:-0}" == 1 ]]; then + batch_args+=(--with-creduce) + fi + if [[ -n "${CREDUCE_N:-}" ]]; then + batch_args+=(--creduce-n "$CREDUCE_N") + fi + if [[ -n "${PASS_UNDER_TEST:-}" ]]; then + batch_args+=(--pass-under-test "$PASS_UNDER_TEST") + fi + if [[ -n "${MTRIPLE:-}" ]]; then + batch_args+=(--mtriple "$MTRIPLE") + fi + if [[ -n "${EXTRACT_MIR_OUTPUT:-}" ]]; then + batch_args+=(--extract-mir-output "$EXTRACT_MIR_OUTPUT") + fi + if [[ -n "${MIR_CODEGEN_ONLY:-}" ]]; then + batch_args+=(--mir-codegen-only) + fi + if [[ "${PREPARE_ONLY:-0}" != 1 ]]; then + if [[ -n "${COVERAGE_LLC:-}" ]]; then + batch_args+=(--llc "$COVERAGE_LLC") + fi + if [[ -n "${COVERAGE_LLVM_REDUCE:-}" ]]; then + batch_args+=(--llvm-reduce "$COVERAGE_LLVM_REDUCE") + fi + fi + + echo "=== gap reducing (row=${reduce_row}, pipeline=${pipeline}) ===" + "${batch_args[@]}" +} diff --git a/scripts/lib/lit-failures.sh b/scripts/lib/lit-failures.sh new file mode 100644 index 00000000..99b50030 --- /dev/null +++ b/scripts/lib/lit-failures.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Post-run warning when LIT tests failed during a baseline coverage run. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +: "${SCRIPTS_DIR:=$(cd "${LIB_DIR}/.." && pwd)}" + +emit_lit_failures_warning() { + local output_dir="$1" + local downstream_context="$2" + + local lit_failures_json="${output_dir}/baseline/lit_failures.json" + local warning_file="${output_dir}/README-WARNING" + local fail_count=0 + + if [[ -f "$lit_failures_json" ]]; then + fail_count="$(python3 "${SCRIPTS_DIR}/count_lit_failures.py" "$lit_failures_json")" + fi + + if [[ "$fail_count" -gt 0 ]]; then + local msg="WARNING: ${fail_count} LIT test(s) failed during the baseline run (e.g., failed a check, timed out, unresolved, or unexpectedly passed). +Failed tests may lead to an incomplete coverage profile. As a result, +${downstream_context} may report lines as uncovered even though they are +actually covered by a (failing) test. +Review baseline/lit_failures.json for the list of failing tests." + + if [[ -t 1 ]]; then + printf '%b\n' "\033[1;31m${msg}\033[0m" + else + printf '%s\n' "$msg" + fi + + printf '%s\n' "$msg" > "$warning_file" + echo "Wrote ${warning_file}" + fi +} diff --git a/scripts/lib/lit-filters.sh b/scripts/lib/lit-filters.sh new file mode 100644 index 00000000..a68dbf02 --- /dev/null +++ b/scripts/lib/lit-filters.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# LIT filter resolution for baseline coverage runs. +# Source from entrypoints or other scripts/lib modules; do not execute directly. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +: "${SCRIPTS_DIR:=$(cd "${LIB_DIR}/.." && pwd)}" +: "${REPO_ROOT:=$(cd "${SCRIPTS_DIR}/.." && pwd)}" + +# Single default prefix for docker gap-finding-baseline (one --lit-filter). +lit_filter_for_allowlist() { + case "$1" in + amdgpu) echo "CodeGen/AMDGPU" ;; + spirv) echo "CodeGen/SPIRV" ;; + *) + echo "error: unsupported image allowlist: ${1} (expected amdgpu or spirv)" >&2 + exit 1 + ;; + esac +} + +# Full filter list for docker gap-finding-pr and gap-filling-amdgpu defaults. +# Prints one prefix per line (for mapfile). +default_lit_filters_for_allowlist() { + case "$1" in + amdgpu) + # shellcheck source=scripts/lit-filters-amdgpu.sh + source "${SCRIPTS_DIR}/lit-filters-amdgpu.sh" + printf '%s\n' "${AMDGPU_LIT_FILTERS[@]}" + ;; + spirv) + printf '%s\n' "CodeGen/SPIRV" + ;; + *) + echo "error: unsupported image allowlist: ${1} (expected amdgpu or spirv)" >&2 + return 1 + ;; + esac +} diff --git a/scripts/lib/local-llvm-env.sh b/scripts/lib/local-llvm-env.sh new file mode 100644 index 00000000..1fa036f0 --- /dev/null +++ b/scripts/lib/local-llvm-env.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Local LLVM path setup and coverage tool exports for gap pipeline scripts. +# Source from entrypoints or other scripts/lib modules; do not execute directly. +# +# setup_local_llvm_env [instrumented_subdir] +# Sets LLVM_REPO, LLVM_BIN, INSTRUMENTED_BIN_DIR and exports COVERAGE_* tool paths. +# setup_local_llvm_env_with_lit_failures — same, plus LIT_ALLOW_FAILURES=1. + +: "${LIB_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +# shellcheck source=scripts/lib/common.sh +source "${LIB_DIR}/common.sh" + +setup_local_llvm_env() { + local repo_root="$1" + local instrumented_subdir="${2:-build-amdgpu-bb/bin}" + + LLVM_REPO="${LLVM_REPO:-$(cd "${repo_root}/../llvm-project" && pwd)}" + LLVM_BIN="${LLVM_BIN:-${LLVM_REPO}/build/bin}" + INSTRUMENTED_BIN_DIR="${INSTRUMENTED_BIN_DIR:-${LLVM_REPO}/${instrumented_subdir}}" + + export COVERAGE_SANCOV="${LLVM_BIN}/sancov" + export COVERAGE_LLVM_LIT="${INSTRUMENTED_BIN_DIR}/llvm-lit" + export COVERAGE_LLC="${INSTRUMENTED_BIN_DIR}/llc" + export COVERAGE_OPT="${INSTRUMENTED_BIN_DIR}/opt" +} + +setup_local_llvm_env_with_lit_failures() { + setup_local_llvm_env "$@" + export LIT_ALLOW_FAILURES=1 +} + +require_local_coverage_bins() { + require_bin "$LLVM_BIN" sancov + require_bin "$INSTRUMENTED_BIN_DIR" llvm-lit + require_bin "$INSTRUMENTED_BIN_DIR" llc + require_bin "$INSTRUMENTED_BIN_DIR" opt +} + +require_local_reduce_bins() { + require_bin "$INSTRUMENTED_BIN_DIR" llc + require_bin "$LLVM_BIN" llvm-reduce +} + +export_local_reduce_tools() { + export COVERAGE_LLC="${INSTRUMENTED_BIN_DIR}/llc" + export COVERAGE_LLVM_REDUCE="${LLVM_BIN}/llvm-reduce" +} diff --git a/scripts/lit-filters-amdgpu.sh b/scripts/lit-filters-amdgpu.sh index 8c10574e..1aa30c22 100644 --- a/scripts/lit-filters-amdgpu.sh +++ b/scripts/lit-filters-amdgpu.sh @@ -1,6 +1,6 @@ # Shared AMDGPU llvm-lit --filter= regex fragments for baseline coverage runs. -# Sourced by scripts/test_coverage_amdgpu_workflow1.sh and -# scripts/docker/pr-cov-gaps-detection.sh (default for amdgpu allowlist). +# Sourced by scripts/gap-filling-amdgpu.sh and +# scripts/docker/gap-finding-pr.sh (default for amdgpu allowlist). AMDGPU_LIT_FILTERS=( CodeGen/AMDGPU diff --git a/scripts/test_coverage.sh b/scripts/test_coverage.sh deleted file mode 100755 index b831cb11..00000000 --- a/scripts/test_coverage.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -LLVM_REPO="${LLVM_REPO:-$(cd "${REPO_ROOT}/../llvm-project" && pwd)}" -LLVM_BIN="${LLVM_BIN:-${LLVM_REPO}/build/bin}" -INSTRUMENTED_BIN_DIR="${INSTRUMENTED_BIN_DIR:-${LLVM_REPO}/build-amdgpu-bb/bin}" -OUTPUT_DIR="${REPO_ROOT}/data/coverage_output/bb_coverage_amdgpu_230726_full_filter" -BASELINE_OUTPUT_DIR=$OUTPUT_DIR/baseline -CANDIDATE_TESTS_OUTPUT_DIR=$OUTPUT_DIR/candidate_tests -INCREMENTAL_OUTPUT_DIR=$OUTPUT_DIR/incremental -TESTS_DIR="${TESTS_DIR:-${REPO_ROOT}/../irtests/bitcode/amdgpu/all}" - -FILTER="${FILTER:-AMDGPU}" - -# Clear old output directories -rm -rf $BASELINE_OUTPUT_DIR -rm -rf $CANDIDATE_TESTS_OUTPUT_DIR -rm -rf $INCREMENTAL_OUTPUT_DIR - -cd "$REPO_ROOT" - -python -m coverage baseline \ - --output-dir $BASELINE_OUTPUT_DIR \ - --sancov "$LLVM_BIN/sancov" \ - --llvm-lit "$INSTRUMENTED_BIN_DIR/llvm-lit" \ - --llc "$INSTRUMENTED_BIN_DIR/llc" \ - --opt "$INSTRUMENTED_BIN_DIR/opt" \ - --lit-filter "$FILTER" \ - --lit-allow-failures - -python -m coverage candidate-test \ - --output-dir $CANDIDATE_TESTS_OUTPUT_DIR \ - --llc "$INSTRUMENTED_BIN_DIR/llc" \ - --candidate-tests-dir $TESTS_DIR \ - --n 100 - -python -m coverage incremental \ - --output-dir $INCREMENTAL_OUTPUT_DIR \ - --sancov "$LLVM_BIN/sancov" \ - --line-coverage-uncovered-csv $BASELINE_OUTPUT_DIR/line_coverage_uncovered.csv \ - --llc-address-line-map-csv $BASELINE_OUTPUT_DIR/llc_address_line_map.csv \ - --candidate-tests-output-dir $CANDIDATE_TESTS_OUTPUT_DIR diff --git a/scripts/test_coverage_amdgpu_workflow1.sh b/scripts/test_coverage_amdgpu_workflow1.sh deleted file mode 100755 index 7a62aca3..00000000 --- a/scripts/test_coverage_amdgpu_workflow1.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash -# Workflow 1: baseline -> candidate-test -> incremental for AMDGPU LIT subsets. -# -# Usage: -# ./scripts/test_coverage_amdgpu_workflow1.sh -# OUTPUT_DIR=./data/my_run ./scripts/test_coverage_amdgpu_workflow1.sh -# JOBS="$(nproc)" ./scripts/test_coverage_amdgpu_workflow1.sh -# SKIP_BASELINE=1 ./scripts/test_coverage_amdgpu_workflow1.sh -# REFRESH=all ./scripts/test_coverage_amdgpu_workflow1.sh -# -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -LLVM_REPO="${LLVM_REPO:-$(cd "${REPO_ROOT}/../llvm-project" && pwd)}" -LLVM_BIN="${LLVM_BIN:-${LLVM_REPO}/build/bin}" -INSTRUMENTED_BIN_DIR="${INSTRUMENTED_BIN_DIR:-${LLVM_REPO}/build-sancov/bin}" -OUTPUT_DIR="${OUTPUT_DIR:-${REPO_ROOT}/data/coverage_output/bb_coverage_amdgpu_$(date +%y%m%d)}" -BASELINE_OUTPUT_DIR="$OUTPUT_DIR/baseline" -CANDIDATE_TESTS_OUTPUT_DIR="$OUTPUT_DIR/candidate_tests" -INCREMENTAL_OUTPUT_DIR="$OUTPUT_DIR/incremental" -TESTS_DIR="${TESTS_DIR:-${REPO_ROOT}/../irtests/bitcode/amdgpu/all}" -CORPUS_N="${CORPUS_N:-100}" - -# shellcheck source=scripts/lit-filters-amdgpu.sh -source "${SCRIPT_DIR}/lit-filters-amdgpu.sh" -LIT_FILTERS=("${AMDGPU_LIT_FILTERS[@]}") - -require_bin() { - local dir="$1" tool="$2" - if [[ ! -x "$dir/$tool" ]]; then - echo "error: missing $dir/$tool" >&2 - exit 1 - fi -} - -if [[ -f "$REPO_ROOT/venv/bin/activate" ]]; then - # shellcheck disable=SC1091 - source "$REPO_ROOT/venv/bin/activate" -fi - -require_bin "$LLVM_BIN" sancov -require_bin "$INSTRUMENTED_BIN_DIR" llvm-lit -require_bin "$INSTRUMENTED_BIN_DIR" llc -require_bin "$INSTRUMENTED_BIN_DIR" opt - -case "${REFRESH:-}" in - all) - rm -rf "$BASELINE_OUTPUT_DIR" "$CANDIDATE_TESTS_OUTPUT_DIR" "$INCREMENTAL_OUTPUT_DIR" - ;; - baseline) - rm -rf "$BASELINE_OUTPUT_DIR" - ;; - candidate) - rm -rf "$CANDIDATE_TESTS_OUTPUT_DIR" - ;; - incremental) - rm -rf "$INCREMENTAL_OUTPUT_DIR" - ;; - "") - ;; - *) - echo "error: unknown REFRESH=$REFRESH (use all, baseline, candidate, or incremental)" >&2 - exit 1 - ;; -esac - -mkdir -p "$OUTPUT_DIR" -cd "$REPO_ROOT" - -echo "=== AMDGPU Workflow 1 coverage ===" -echo "output: $OUTPUT_DIR" -echo "llvm-bin: $LLVM_BIN" -echo "instrumented: $INSTRUMENTED_BIN_DIR" -echo "candidate dir: $TESTS_DIR (n=$CORPUS_N)" -echo "lit filters: ${LIT_FILTERS[*]}" -echo - -if [[ -z "${SKIP_BASELINE:-}" ]]; then - echo ">>> Step 1/3: baseline (LIT suite)" - baseline_args=( - python -m coverage baseline - --output-dir "$BASELINE_OUTPUT_DIR" - --sancov "$LLVM_BIN/sancov" - --llvm-lit "$INSTRUMENTED_BIN_DIR/llvm-lit" - --llc "$INSTRUMENTED_BIN_DIR/llc" - --opt "$INSTRUMENTED_BIN_DIR/opt" - ) - for lit_filter in "${LIT_FILTERS[@]}"; do - baseline_args+=(--lit-filter "$lit_filter") - done - if [[ -n "${JOBS:-}" ]]; then - baseline_args+=(-j "$JOBS") - fi - "${baseline_args[@]}" -else - echo ">>> Step 1/3: skipped (SKIP_BASELINE=1)" -fi - -if [[ -z "${SKIP_CANDIDATE:-}" ]]; then - echo ">>> Step 2/3: candidate-test" - python -m coverage candidate-test \ - --output-dir "$CANDIDATE_TESTS_OUTPUT_DIR" \ - --llc "$INSTRUMENTED_BIN_DIR/llc" \ - --candidate-tests-dir "$TESTS_DIR" \ - --n "$CORPUS_N" -else - echo ">>> Step 2/3: skipped (SKIP_CANDIDATE=1)" -fi - -if [[ -z "${SKIP_INCREMENTAL:-}" ]]; then - echo ">>> Step 3/3: incremental" - python -m coverage incremental \ - --output-dir "$INCREMENTAL_OUTPUT_DIR" \ - --sancov "$LLVM_BIN/sancov" \ - --baseline-output-dir "$BASELINE_OUTPUT_DIR" \ - --candidate-tests-output-dir "$CANDIDATE_TESTS_OUTPUT_DIR" -else - echo ">>> Step 3/3: skipped (SKIP_INCREMENTAL=1)" -fi - -echo -echo "Baseline summary: $BASELINE_OUTPUT_DIR/line_coverage_summary.csv" -echo "Gap-fill report: $INCREMENTAL_OUTPUT_DIR/new_coverage.csv" diff --git a/scripts/test_coverage_commit_lines.sh b/scripts/test_coverage_commit_lines.sh deleted file mode 100755 index 0a70ccaa..00000000 --- a/scripts/test_coverage_commit_lines.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# End-to-end: added-lines -> LIT baseline symcov -> target-lines uncovered list. -# Run from fuzz-fill repo root (same layout as scripts/test_coverage.sh). - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -LLVM_REPO="${LLVM_REPO:-$(cd "${REPO_ROOT}/../llvm-project" && pwd)}" -LLVM_BIN="${LLVM_BIN:-${LLVM_REPO}/build/bin}" -INSTRUMENTED_BIN_DIR="${INSTRUMENTED_BIN_DIR:-${LLVM_REPO}/build-amdgpu-bb/bin}" - -OUTPUT_DIR="${REPO_ROOT}/data/coverage_output/bb_coverage_commit_lines_170626" -BASELINE_OUTPUT_DIR=$OUTPUT_DIR/baseline -ADDED_LINES_DIR=$OUTPUT_DIR/added-lines -TARGET_LINES_REPORT_DIR=$OUTPUT_DIR/target_lines_report - -# Faster CodeGen-only subset: FILTER=CodeGen/AMDGPU -FILTER="${FILTER:-AMDGPU}" - -COMMIT=b01fe4e - -cd "$REPO_ROOT" - -#rm -rf "$BASELINE_OUTPUT_DIR" -mkdir -p "$ADDED_LINES_DIR" "$TARGET_LINES_REPORT_DIR" - -# 1) Lines added in COMMIT (same tree as --llvm-repo) -python -m added_lines \ - --llvm-repo "$LLVM_REPO" \ - --commit "$COMMIT" \ - --output-dir "$ADDED_LINES_DIR" - -# 2) Baseline suite coverage (required symcov under BASELINE_OUTPUT_DIR/processed_sancov/) -python -m coverage baseline \ - --output-dir "$BASELINE_OUTPUT_DIR" \ - --sancov "$LLVM_BIN/sancov" \ - --llvm-lit "$INSTRUMENTED_BIN_DIR/llvm-lit" \ - --llc "$INSTRUMENTED_BIN_DIR/llc" \ - --opt "$INSTRUMENTED_BIN_DIR/opt" \ - --lit-filter "$FILTER" - -# 3) Target lines not fully covered by the suite (target_lines_uncovered.csv) -python -m coverage target-lines \ - --output-dir "$TARGET_LINES_REPORT_DIR" \ - --line-coverage-uncovered-csv "$BASELINE_OUTPUT_DIR/line_coverage_uncovered.csv" \ - --llvm-repo "$LLVM_REPO" \ - --target-lines-csv "$ADDED_LINES_DIR/added-lines.csv" - -echo "Uncovered target lines: $TARGET_LINES_REPORT_DIR/target_lines_uncovered.csv" diff --git a/src/coverage/target_lines_check.py b/src/coverage/target_lines_check.py index 7b5be35b..7afa98b0 100644 --- a/src/coverage/target_lines_check.py +++ b/src/coverage/target_lines_check.py @@ -34,6 +34,11 @@ def run_target_lines_check( A line is listed only when its matched ``(file, line)`` is present in the baseline uncovered CSV produced by ``coverage baseline``. + + The report uses the same uncovered-lines contract as ``coverage baseline``: + columns ``file`` and ``line`` with absolute paths matching the baseline + summary / LLC address map. An optional ``text`` column preserves the source + line for review. """ uncovered_lines = load_uncovered_lines_csv(line_coverage_uncovered_csv) summary_files: set[str] = {file for file, _ in uncovered_lines} @@ -81,15 +86,15 @@ def run_target_lines_check( stats["reported"] += 1 uncovered_rows.append( { - "file": rel, - "line_no": str(line_no), + "file": sym_file, + "line": str(line_no), "text": text, } ) else: stats["not_in_uncovered_list"] += 1 - fieldnames = ["file", "line_no", "text"] + fieldnames = ["file", "line", "text"] with report_path.open("w", encoding="utf-8", newline="") as out: w = csv.DictWriter(out, fieldnames=fieldnames) w.writeheader() diff --git a/tests/test_target_lines_check.py b/tests/test_target_lines_check.py index d4562a50..296ca2a6 100644 --- a/tests/test_target_lines_check.py +++ b/tests/test_target_lines_check.py @@ -61,8 +61,9 @@ def test_only_uncovered_lines_written_to_report(self) -> None: with report.open(newline="", encoding="utf-8") as f: rows = list(csv.DictReader(f)) - reported = {(row["file"], int(row["line_no"])) for row in rows} - self.assertEqual(reported, {(REL, 30), (REL, 40)}) + reported = {(row["file"], int(row["line"])) for row in rows} + self.assertEqual(reported, {(ABS, 30), (ABS, 40)}) + self.assertEqual({row["text"] for row in rows}, {"uncovered line", "uncovered line 2"}) def test_missing_uncovered_csv_raises_systemexit(self) -> None: with tempfile.TemporaryDirectory() as tmp: