Docker workflow for gap finding and gap filling

Dockerfile

@@ -151,6 +151,7 @@ COPY --chown=${UID}:${GID} pyproject.toml LICENCE.txt README.md /work/fuzz-fill/
 COPY --chown=${UID}:${GID} src /work/fuzz-fill/src
 COPY --chown=${UID}:${GID} tests /work/fuzz-fill/tests
 COPY --chown=${UID}:${GID} integration-tests /work/fuzz-fill/integration-tests
+COPY --chown=${UID}:${GID} scripts /work/fuzz-fill/scripts
 
 RUN echo "=== fuzz-fill image LLVM source: $(cat /work/.llvm-source) ===" \
  && echo "=== fuzz-fill image SanitizerCoverage allowlist: $(cat /work/.sancov-allowlist) ===" \

integration-tests/coverage-pipeline.test

@@ -65,7 +65,7 @@
 # COM: The same coverage gaps reported by incremental (GAP_CSV) must be reported as
 # COM: all-points-uncovered by target-lines when fed as target lines.
 # RUN: %FileCheck %s --input-file=%t/target_lines/target_lines_uncovered.csv --check-prefix=TARGET_CSV
-# TARGET_CSV-DAG: file,line_no,text
+# TARGET_CSV-DAG: file,line,text
 # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,211,
 # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,222,
 # TARGET_CSV-DAG: {{.*}}AMDGPUAsmPrinter.cpp,581,

integration-tests/gap-filling.test

@@ -0,0 +1,41 @@
+# COM: Gap filling: candidate-test -> incremental using explicit profile CSVs
+# COM: (mirrors Docker gap-filling.sh). Scenario A uses baseline gap list; scenario B
+# COM: uses target_lines_uncovered.csv from target-lines as the PR gap list.
+
+# RUN: rm -rf %t && mkdir -p %t/profile %t/candidate_tests %t/incremental %t/pr_gaps %t/candidate_tests_pr %t/incremental_pr
+# RUN: %coverage baseline --output-dir %t/profile --sancov %sancov --llvm-lit %llvm-lit --llc %sancov-llc --opt %sancov-opt --lit-filter CodeGen/AMDGPU/loop -j2
+
+# COM: Scenario A — baseline gap fill
+# RUN: %coverage candidate-test --output-dir %t/candidate_tests --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1
+# RUN: %coverage incremental --output-dir %t/incremental --sancov %sancov --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests
+
+# RUN: test -f %t/incremental/new_coverage.csv
+# RUN: %FileCheck %s --input-file=%t/incremental/new_coverage.csv --check-prefix=GAP_CSV
+# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,211,
+# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,222,
+# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,581,
+# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,590,
+# GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,680,
+
+# RUN: %FileCheck %s --input-file=%t/incremental/new_coverage.csv --check-prefix=NO_BASELINE_GAIN
+# NO_BASELINE_GAIN-NOT: /AMDGPUAlwaysInlinePass.cpp,160,
+
+# COM: Scenario B — PR gap fill (target_lines_uncovered.csv as gap list)
+# RUN: %coverage target-lines --output-dir %t/pr_gaps --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llvm-repo %llvm-repo --target-lines-csv %S/fixtures/target-lines-sample.csv
+# RUN: test -f %t/pr_gaps/target_lines_uncovered.csv
+# RUN: %FileCheck %s --input-file=%t/pr_gaps/target_lines_uncovered.csv --check-prefix=PR_TARGET_CSV
+# PR_TARGET_CSV-DAG: file,line,text
+
+# RUN: %coverage candidate-test --output-dir %t/candidate_tests_pr --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1
+# RUN: %coverage incremental --output-dir %t/incremental_pr --sancov %sancov --line-coverage-uncovered-csv %t/pr_gaps/target_lines_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests_pr
+
+# RUN: test -f %t/incremental_pr/new_coverage.csv
+# RUN: %FileCheck %s --input-file=%t/incremental_pr/new_coverage.csv --check-prefix=PR_GAP_CSV
+# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,211,
+# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,222,
+# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUAsmPrinter.cpp,581,
+# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,590,
+# PR_GAP_CSV-DAG: standalone-empty-kernel.ll,{{.*}}/AMDGPUHSAMetadataStreamer.cpp,680,
+
+# RUN: %FileCheck %s --input-file=%t/incremental_pr/new_coverage.csv --check-prefix=PR_GAP_CSV_NOT
+# PR_GAP_CSV_NOT-NOT: /AMDGPUAliasAnalysis.cpp,103,

integration-tests/gap-reducing.test

@@ -0,0 +1,26 @@
+# COM: Gap reducing: build a single-row reduce harness from gap-filling output.
+# COM: Omits --llc/--llvm-reduce (prepare-only): config.json, interesting_ir.sh,
+# COM: and a copy of the candidate IR input.
+
+# RUN: rm -rf %t && mkdir -p %t/profile %t/candidate_tests %t/incremental %t/reduced
+# RUN: %coverage baseline --output-dir %t/profile --sancov %sancov --llvm-lit %llvm-lit --llc %sancov-llc --opt %sancov-opt --lit-filter CodeGen/AMDGPU/loop -j2
+# RUN: %coverage candidate-test --output-dir %t/candidate_tests --llc %sancov-llc --candidate-tests-dir %S/fixtures/coverage-new-tests --n 1
+# RUN: %coverage incremental --output-dir %t/incremental --sancov %sancov --line-coverage-uncovered-csv %t/profile/line_coverage_uncovered.csv --llc-address-line-map-csv %t/profile/llc_address_line_map.csv --candidate-tests-output-dir %t/candidate_tests
+
+# RUN: test -f %t/incremental/new_coverage.csv
+# RUN: %batch-reduce --csv %t/incremental/new_coverage.csv --candidate-tests %t/candidate_tests --output %t/reduced --n 1 --pipeline llvm_reduce_ir
+
+# RUN: test -f %t/reduced/t-00001-test_0_s/config.json
+# RUN: test -f %t/reduced/t-00001-test_0_s/interesting_ir.sh
+# RUN: test -f %t/reduced/t-00001-test_0_s/standalone-empty-kernel.ll
+# RUN: test -x %t/reduced/t-00001-test_0_s/interesting_ir.sh
+
+# RUN: %FileCheck %s --input-file=%t/reduced/t-00001-test_0_s/config.json --check-prefix=CONFIG
+# CONFIG: "input": "standalone-empty-kernel.ll",
+# CONFIG: "file": "llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp",
+# CONFIG: "line": 91,
+# CONFIG: "llvm_reduce_ir"
+
+# RUN: %FileCheck %s --input-file=%t/reduced/t-00001-test_0_s/interesting_ir.sh --check-prefix=INTERESTING
+# INTERESTING: LLC_FLAGS="-O0"
+# INTERESTING: COVERED="0x15a77d0"

integration-tests/lit.cfg.py

@@ -83,8 +83,12 @@ def _tool(bin_dir: str, name: str) -> str:
 
 
 _llvm_build_dir = os.path.dirname(_llvm_bin_dir)
+_repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 _venv_cmd = f". {shlex.quote(_venv_activate)}"
 _venv_python = shlex.quote(os.path.join(_venv_dir, "bin", "python"))
+_batch_reduce = shlex.quote(
+    os.path.join(_repo_root, "scripts", "batch_reduce_using_coverage.py")
+)
 
 config.name = "fuzz-fill-integration-tests"
 config.suffixes = [".test"]
@@ -103,6 +107,7 @@ def _tool(bin_dir: str, name: str) -> str:
         ("%venv", _venv_cmd),
         ("%coverage", f"{_venv_python} -m coverage"),
         ("%reduce", f"{_venv_python} -m reduce"),
+        ("%batch-reduce", f"{_venv_python} {_batch_reduce}"),
         ("%added-lines", f"{_venv_cmd} && python -m added_lines"),
     ]
 )

scripts/batch_reduce_using_coverage.py

@@ -6,7 +6,7 @@
 CSV columns: test_name, file, line, covered-points
 (test_name is a directory under --candidate-tests containing test.sh).
 
-Each test.sh records the instrumented llc invocation (binary, flags, .bc path).
+Each test.sh records the instrumented llc invocation (binary, flags, .bc or .ll path).
 Llvm flags on that command are copied into interesting_ir.sh and extract_*
 pipeline steps (config ``llc_O``). interesting_mir.sh uses the same COVERED
 sancov check as interesting_ir.sh. Use ``--mir-codegen-only`` when the pass
@@ -136,8 +136,30 @@ class TestShInfo:
     llc_flags: tuple[str, ...]
 
 
+def resolve_input_path(input_path: Path, test_sh: Path) -> Path:
+    """Resolve IR input from test.sh, including Docker/host path remapping."""
+    if input_path.is_file():
+        return input_path
+
+    repo = _repo_root()
+    posix = input_path.as_posix()
+    marker = "/fuzz-fill/"
+    if marker in posix:
+        candidate = repo / posix.split(marker, 1)[1]
+        if candidate.is_file():
+            return candidate
+
+    sibling = test_sh.parent / input_path.name
+    if sibling.is_file():
+        return sibling
+
+    raise FileNotFoundError(
+        f"Input IR file from test.sh does not exist: {input_path}"
+    )
+
+
 def parse_test_sh(test_sh: Path) -> TestShInfo:
-    """Parse candidate_tests/<test_name>/test.sh for llc binary, flags, and .bc path."""
+    """Parse candidate_tests/<test_name>/test.sh for llc binary, flags, and IR input."""
     if not test_sh.is_file():
         raise FileNotFoundError(f"test.sh not found: {test_sh}")
 
@@ -160,22 +182,28 @@ def parse_test_sh(test_sh: Path) -> TestShInfo:
     if len(parts) < 2:
         raise ValueError(f"Could not parse llc command in {test_sh}: {llc_line!r}")
 
-    llc_exe = Path(parts[0])
-    if llc_exe.name != "llc":
-        raise ValueError(f"Expected llc executable, got {llc_exe!r} in {test_sh}")
-
-    bc_indices = [i for i, p in enumerate(parts) if p.endswith(".bc")]
-    if len(bc_indices) != 1:
+    llc_indices = [i for i, p in enumerate(parts) if Path(p).name == "llc"]
+    if len(llc_indices) != 1:
+        raise ValueError(
+            f"Expected exactly one llc executable in {test_sh}, got {llc_indices!r}"
+        )
+    llc_idx = llc_indices[0]
+    llc_exe = Path(parts[llc_idx])
+
+    input_indices = [
+        i
+        for i, p in enumerate(parts)
+        if i > llc_idx and (p.endswith(".bc") or p.endswith(".ll"))
+    ]
+    if len(input_indices) != 1:
         raise ValueError(
-            f"Expected exactly one .bc argument in {test_sh}, got {bc_indices!r}"
+            f"Expected exactly one .bc or .ll argument in {test_sh}, got {input_indices!r}"
         )
-    bc_idx = bc_indices[0]
-    bc_path = Path(parts[bc_idx])
-    if not bc_path.is_file():
-        raise FileNotFoundError(f"Bitcode file from test.sh does not exist: {bc_path}")
+    input_idx = input_indices[0]
+    input_path = resolve_input_path(Path(parts[input_idx]), test_sh)
 
-    flags = tuple(parts[1:bc_idx])
-    return TestShInfo(llvm_bin=llc_exe.parent, bc_path=bc_path, llc_flags=flags)
+    flags = tuple(parts[llc_idx + 1 : input_idx])
+    return TestShInfo(llvm_bin=llc_exe.parent, bc_path=input_path, llc_flags=flags)
 
 
 def render_interesting_ir(
@@ -509,7 +537,7 @@ def validate_pipeline_cli(
 
 
 def copy_input_bc(test_info: TestShInfo, dest_dir: Path) -> str:
-    """Copy the .bc from test.sh into dest_dir; return config input basename."""
+    """Copy the IR input from test.sh into dest_dir; return config input basename."""
     dest_name = test_info.bc_path.name
     shutil.copy2(test_info.bc_path, dest_dir / dest_name)
     return dest_name

scripts/count_lit_failures.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Count LIT tests with failure-like result codes in a lit_failures.json report."""
+
+import argparse
+import json
+
+FAILURE_CODES = frozenset({"FAIL", "TIMEOUT", "UNRESOLVED", "XPASS"})
+
+
+def count_failures(path: str) -> int:
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return 0
+    return sum(1 for t in data.get("tests", []) if t.get("code") in FAILURE_CODES)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("lit_failures_json", help="Path to lit_failures.json from coverage baseline")
+    args = parser.parse_args()
+    print(count_failures(args.lit_failures_json))
+
+
+if __name__ == "__main__":
+    main()

scripts/docker/docker-image-cli.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Shared Docker image flag parsing and prepare helpers for gap docker runners.
+# Source after SCRIPT_DIR is set; requires ensure-image.sh.
+
+: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing docker-image-cli.sh}"
+
+docker_image_cli_init_vars() {
+    image_name="${IMAGE_NAME:-fuzz-fill-test}"
+    image_tag="${IMAGE_TAG:-latest}"
+    image_ref=""
+    pr_id=""
+    build_image=0
+    keep_image=0
+    force_build=0
+    llvm_repo=""
+    backend_tests=""
+    github_repo=""
+}
+
+# Parse one image-related flag. Returns 0 if consumed; sets DOCKER_IMAGE_CLI_SHIFT.
+docker_image_cli_try_parse() {
+    DOCKER_IMAGE_CLI_SHIFT=0
+
+    case "$1" in
+        --image)
+            [[ $# -ge 2 ]] || { echo "error: --image requires a value" >&2; exit 2; }
+            image_ref="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        --pr-id)
+            [[ $# -ge 2 ]] || { echo "error: --pr-id requires a value" >&2; exit 2; }
+            pr_id="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        --build-image)
+            build_image=1
+            DOCKER_IMAGE_CLI_SHIFT=1
+            return 0
+            ;;
+        --keep-image)
+            keep_image=1
+            DOCKER_IMAGE_CLI_SHIFT=1
+            return 0
+            ;;
+        --force-build)
+            force_build=1
+            DOCKER_IMAGE_CLI_SHIFT=1
+            return 0
+            ;;
+        --llvm-repo)
+            [[ $# -ge 2 ]] || { echo "error: --llvm-repo requires a value" >&2; exit 2; }
+            llvm_repo="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        --backend-tests)
+            [[ $# -ge 2 ]] || { echo "error: --backend-tests requires a value" >&2; exit 2; }
+            backend_tests="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        --github-repo)
+            [[ $# -ge 2 ]] || { echo "error: --github-repo requires a value" >&2; exit 2; }
+            github_repo="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        --image-name)
+            [[ $# -ge 2 ]] || { echo "error: --image-name requires a value" >&2; exit 2; }
+            image_name="$2"
+            DOCKER_IMAGE_CLI_SHIFT=2
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+docker_image_cli_prepare() {
+    docker_image_validate_build_flags
+    docker_image_normalize_backend_tests
+    docker_image_validate_pr_id
+    docker_image_resolve_ref
+    docker_image_ensure
+}

scripts/docker/docker-run.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Shared docker run helpers for gap docker runners.
+# Source after SCRIPT_DIR and REPO_ROOT are set.
+
+: "${REPO_ROOT:?REPO_ROOT must be set before sourcing docker-run.sh}"
+
+DOCKER_GAP_CONTAINER_WORKDIR="/work/fuzz-fill"
+
+docker_gap_container_script() {
+    printf '%s/scripts/docker/%s' "$DOCKER_GAP_CONTAINER_WORKDIR" "$(basename "$1")"
+}
+
+# Parse --bind-repo or -j/--jobs. Returns 0 if consumed; sets DOCKER_GAP_CLI_SHIFT.
+docker_gap_cli_try_parse_common() {
+    DOCKER_GAP_CLI_SHIFT=0
+
+    case "$1" in
+        --bind-repo)
+            bind_repo=1
+            DOCKER_GAP_CLI_SHIFT=1
+            return 0
+            ;;
+        -j|--jobs)
+            [[ $# -ge 2 ]] || { echo "error: $1 requires a value" >&2; exit 2; }
+            jobs="$2"
+            DOCKER_GAP_CLI_SHIFT=2
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+docker_gap_append_bind_repo_mount() {
+    local -n _mounts=$1
+    if [[ "${bind_repo:-0}" -eq 1 ]]; then
+        _mounts+=(-v "${REPO_ROOT}:${DOCKER_GAP_CONTAINER_WORKDIR}")
+        echo "Using local fuzz-fill checkout: ${REPO_ROOT}"
+    fi
+}
+
+docker_gap_append_jobs_env() {
+    local -n _env=$1
+    if [[ -n "${jobs:-}" ]]; then
+        _env+=(-e "JOBS=${jobs}")
+    fi
+}
+
+# Run the current docker gap workflow script inside a container.
+docker_gap_run() {
+    local container_script="$1"
+    local host_output_dir="$2"
+    local image="$3"
+    local -n _env=$4
+    local -n _mounts=$5
+
+    docker run --rm \
+        -v "${host_output_dir}:/mounted-output" \
+        "${_mounts[@]}" \
+        "${_env[@]}" \
+        -w "${DOCKER_GAP_CONTAINER_WORKDIR}" \
+        "${image}" \
+        bash "${container_script}"
+}