Regression CI

.buildkite/compare_regression.jl

@@ -0,0 +1,116 @@
+# Compare benchmark harness results from the base branch and this commit.
+#
+#     julia compare_regression.jl <base_root> <candidate_root> [--threshold=10] [--base=main] [--out=report.md]
+#
+# Each root holds the harness run directory for that side. Exits 1 on a
+# slowdown above the threshold or a failed candidate run; base failures are
+# only reported so a PR can fix them.
+
+using Printf
+using Statistics
+using TOML
+
+function load_runs(root)
+    runs = Dict{Tuple,Union{Float64,Nothing}}()
+    isdir(root) || return runs
+    for dir in readdir(root; join=true)
+        manifest_path = joinpath(dir, "manifest.toml")
+        isfile(manifest_path) || continue
+        for r in TOML.parsefile(manifest_path)["runs"]
+            key = (r["name"], r["T"], r["N"], r["M"])
+            runs[key] = r["status"] == "complete" ? median_time(dir, r) : nothing
+        end
+    end
+    return runs
+end
+
+# Median trial time in ms, or `nothing` when rows are missing or incorrect.
+function median_time(dir, r)
+    # The harness writes `<name>_<save_as>.csv`, where save_as extends the model
+    # (e.g. `cunumeric_nofusion`, `cunumeric_struct`).
+    results = joinpath(dir, r["results_subdir"])
+    isdir(results) || return nothing
+    prefix = "$(r["name"])_$(r["model"])"
+    files = filter(readdir(results)) do f
+        return f == "$prefix.csv" || (startswith(f, "$(prefix)_") && endswith(f, ".csv"))
+    end
+    times = Float64[]
+    for path in joinpath.(results, files), line in eachline(path)
+        f = split(strip(line), ',')
+        length(f) == 8 || continue
+        (parse(Int, f[3]), parse(Int, f[4])) == (r["N"], r["M"]) || continue
+        f[8] == "fail" && return nothing
+        push!(times, parse(Float64, f[6]))
+    end
+    return isempty(times) ? nothing : median(times)
+end
+
+label(key) = "$(key[1]) ($(key[2]), $(key[3])×$(key[4]))"
+
+function report(base_root, candidate_root, threshold, base_name)
+    base = load_runs(base_root)
+    candidate = load_runs(candidate_root)
+    isempty(candidate) && error("no candidate results in $candidate_root")
+
+    lines = [
+        "## Benchmark regression vs. `$base_name`", "",
+        "| Benchmark | `$base_name` (ms) | PR (ms) | Change |",
+        "| --- | ---: | ---: | ---: |",
+    ]
+    regressions, failed, uncompared = String[], String[], String[]
+    for key in sort!(collect(keys(candidate)))
+        after = candidate[key]
+        before = get(base, key, nothing)
+        if after === nothing
+            push!(failed, label(key))
+        elseif before === nothing
+            push!(uncompared, label(key))
+        else
+            change = 100 * (after / before - 1)
+            flag = change > threshold ? " ⚠️" : ""
+            push!(
+                lines,
+                @sprintf("| %s | %.3f | %.3f | %+.1f%%%s |",
+                    label(key), before, after, change, flag)
+            )
+            change > threshold &&
+                push!(regressions, @sprintf("%s: %.1f%% slower", label(key), change))
+        end
+    end
+    for (title, items) in (("Slower than the threshold", regressions),
+        ("Failed on this PR", failed),
+        ("Not compared (no base result)", uncompared))
+        isempty(items) && continue
+        append!(lines, ["", "**$title:**"], ["- $item" for item in items])
+    end
+    push!(lines, "", @sprintf("Threshold: more than %.0f%% slower (median of trials).", threshold))
+    return join(lines, '\n') * '\n', isempty(regressions) && isempty(failed)
+end
+
+function main(args)
+    threshold, out, base_name = 10.0, nothing, "base"
+    positional = String[]
+    for arg in args
+        if startswith(arg, "--threshold=")
+            threshold = parse(Float64, split(arg, '='; limit=2)[2])
+        elseif startswith(arg, "--base=")
+            base_name = split(arg, '='; limit=2)[2]
+        elseif startswith(arg, "--out=")
+            out = split(arg, '='; limit=2)[2]
+        else
+            push!(positional, arg)
+        end
+    end
+    length(positional) == 2 || error("usage: compare_regression.jl <base_root> <candidate_root>")
+    text, ok = report(positional..., threshold, base_name)
+    print(text)
+    out === nothing || write(out, text)
+    return ok ? 0 : 1
+end
+
+try
+    exit(main(ARGS))
+catch e
+    println(stderr, "comparison failed: ", sprint(showerror, e))
+    exit(2)
+end

.buildkite/install_cmake.sh

@@ -0,0 +1,11 @@
+# shellcheck shell=bash
+# Source to put a pinned CMake first on PATH (developer wrapper builds need it).
+CMAKE_VERSION="3.30.7"
+CMAKE_ROOT="$(mktemp -d)"
+CMAKE_INSTALLER="$CMAKE_ROOT/cmake-installer.sh"
+curl --fail --silent --show-error --location \
+    --output "$CMAKE_INSTALLER" \
+    "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-x86_64.sh"
+sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT"
+export PATH="$CMAKE_ROOT/bin:$PATH"
+cmake --version

.buildkite/regression.pipeline.yml

@@ -0,0 +1,21 @@
+steps:
+  - label: ":chart_with_upwards_trend: Benchmark regression vs. base branch"
+    key: "regression"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.12"
+          cache_dir: "${HOME}/.cache/julia-buildkite-plugin-regression"
+    command: ".buildkite/run_regression.sh"
+    artifact_paths:
+      - "regression/**/*"
+    agents:
+      queue: "cuda"
+    # One comparison at a time so runs do not share a GPU.
+    concurrency: 1
+    concurrency_group: "cunumeric/regression"
+    timeout_in_minutes: 180
+    env:
+      LD_LIBRARY_PATH: ""
+      LEGATE_AUTO_CONFIG: "0"
+      # nvidia-smi cannot read GPU memory in the CI container; cap the harness budget.
+      CUNUMERIC_BENCH_FBMEM_MB: "3072"

.buildkite/regression.toml

@@ -0,0 +1,86 @@
+# Opt-in performance regression suite ([regression-ci]): cuNumeric on the base
+# branch vs. this commit. Sizes are pinned so both sides run identical problems.
+# gemm, montecarlo(_naive), grayscott_plain and cg_plain avoid @accelerate and
+# newer APIs, so they are the entries comparable against main.
+[Global]
+models = ["cunumeric"]
+n_warmup = 2
+n_iter = 5
+n_trial = 5
+check_correctness = true
+auto_size = false
+# Legate's pool is capped at CUNUMERIC_BENCH_FBMEM_MB (3 GB) on CI.
+mem_frac = 0.9
+# cpus = 4: 8 CPU procs + GPU/util threads exceed the CI agent's cores.
+
+[[gemm]]
+T = "Float32"
+N = 4096
+M = 4096
+gpus = 1
+cpus = 4
+
+[[montecarlo]]
+T = "Float32"
+N = 50_000_000
+M = 1
+gpus = 1
+cpus = 4
+
+[[montecarlo_naive]]
+T = "Float32"
+N = 50_000_000
+M = 1
+gpus = 1
+cpus = 4
+
+[[grayscott]]
+T = "Float32"
+N = 1024
+M = 1024
+gpus = 1
+cpus = 4
+
+[[grayscott_plain]]
+T = "Float32"
+N = 1024
+M = 1024
+gpus = 1
+cpus = 4
+
+[[cg]]
+T = "Float64"
+N = 65536
+M = 1
+gpus = 1
+cpus = 4
+kwargs = { check_every = 10, max_iter = 1000 }
+
+[[cg_plain]]
+T = "Float64"
+N = 65536
+M = 1
+gpus = 1
+cpus = 4
+kwargs = { check_every = 10, max_iter = 1000 }
+
+[[nas_ep]]
+T = "Float64"
+n_iter = 1
+gpus = 1
+cpus = 4
+kwargs = { class = "A" }
+
+[[nas_mg]]
+T = "Float64"
+n_iter = 1
+gpus = 1
+cpus = 4
+kwargs = { class = "A" }
+
+[[nas_ft]]
+T = "Float64"
+n_iter = 1
+gpus = 1
+cpus = 4
+kwargs = { class = "A" }

.buildkite/regression_base_pins.toml

@@ -0,0 +1,6 @@
+# Extra version pins for the base side's harness environment, keyed by base
+# branch, for bases that no longer load against the current registry.
+
+[main]
+# cuNumeric 0.2.0 calls LegatePreferences.has_cuda_gpu, removed in 0.1.7.
+LegatePreferences = "0.1.6"

.buildkite/run_developer_ci.sh

@@ -10,15 +10,7 @@ case "${CUNUMERIC_FUSION:-}" in
         ;;
 esac
 
-CMAKE_VERSION="3.30.7"
-CMAKE_ROOT="$(mktemp -d)"
-CMAKE_INSTALLER="$CMAKE_ROOT/cmake-installer.sh"
-curl --fail --silent --show-error --location \
-    --output "$CMAKE_INSTALLER" \
-    "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-x86_64.sh"
-sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT"
-export PATH="$CMAKE_ROOT/bin:$PATH"
-cmake --version
+source .buildkite/install_cmake.sh
 
 # Exercise libcxxwrap cache validation separately from package tests, which run in
 # JLL jobs where no build toolchain is installed.

.buildkite/run_regression.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# Opt-in GPU performance comparison of this commit against a base branch:
+# [regression-ci <branch>], else REGRESSION_BASE_BRANCH, else the branch the PR
+# targets, else main. Both sides run this commit's harness and regression.toml.
+
+set -euo pipefail
+
+requested="$(buildkite-agent meta-data get regression-base-branch --default "" 2>/dev/null || true)"
+pr_base="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}"
+readonly BASE_BRANCH="${requested:-${REGRESSION_BASE_BRANCH:-${pr_base:-main}}}"
+readonly THRESHOLD="${REGRESSION_THRESHOLD:-10}"
+# Bounds each base harness pass so a hang on the base cannot use up the step.
+readonly BASE_TIMEOUT="${REGRESSION_BASE_TIMEOUT:-3600}"
+
+candidate="$PWD"
+harness="$candidate/benchmark"
+env_dir="$harness/environments/cunumeric"
+out="$candidate/regression"
+base="$(mktemp -d)/base"
+
+rm -rf "$out"
+mkdir -p "$out"
+
+git fetch --no-tags origin "+refs/heads/$BASE_BRANCH:refs/remotes/origin/$BASE_BRANCH"
+git worktree add --detach "$base" "origin/$BASE_BRANCH"
+trap 'git -C "$candidate" worktree remove --force "$base"' EXIT
+git submodule update --init benchmark
+
+julia --color=yes --project="$harness" -e 'using Pkg; Pkg.instantiate()'
+mkdir -p "$harness/results"
+
+# Point the harness's cuNumeric environment at one checkout. Wrapper overrides
+# and precompiled wrapper bindings from the other side must not leak across.
+bind_checkout() {
+    local source=$1 mode=$2 pins=${3:-}
+    local depot
+    depot="$(julia --startup-file=no -e 'print(DEPOT_PATH[1])')"
+    rm -rf "$depot"/packages/*/*/override \
+           "$depot"/compiled/v*/{cuNumeric,Legate,cunumeric_jl_wrapper_jll,legate_jl_wrapper_jll}
+    rm -f "$env_dir/Manifest.toml" "$env_dir/LocalPreferences.toml"
+    # The harness pins the current cuNumeric/CNPreferences; each side develops
+    # its own checkout, so drop those pins to let an older base resolve.
+    julia --color=yes --project="$env_dir" -e '
+        using Pkg, TOML
+        project = Base.active_project()
+        toml = TOML.parsefile(project)
+        foreach(p -> delete!(get(toml, "compat", Dict()), p), ("cuNumeric", "CNPreferences"))
+        open(io -> TOML.print(io, toml), project, "w")
+        Pkg.develop([PackageSpec(path = ARGS[1]), PackageSpec(path = ARGS[2])])
+        pins = isempty(ARGS[4]) ? Dict() : get(TOML.parsefile(ARGS[3]), ARGS[4], Dict())
+        isempty(pins) || Pkg.add([PackageSpec(name = k, version = v) for (k, v) in pins])
+        Pkg.instantiate()
+    ' "$source" "$source/lib/CNPreferences" "$candidate/.buildkite/regression_base_pins.toml" "$pins"
+    if [[ "$mode" == developer ]]; then
+        julia --color=yes --project="$env_dir" -e '
+            using CNPreferences, Pkg
+            CNPreferences.use_developer_mode()
+            Pkg.build("cuNumeric")
+        '
+    fi
+}
+
+result_dirs() { find "$harness/results" -mindepth 1 -maxdepth 1 -type d -printf '%f\n'; }
+
+run_side() {
+    local side=$1 source=$2 mode=$3
+    local limit=()
+    [[ "$side" == base ]] && limit=(timeout --signal=KILL "$BASE_TIMEOUT")
+    echo "--- :julia: $side ($mode wrapper)"
+    bind_checkout "$source" "$mode" "$([[ "$side" == base ]] && echo "$BASE_BRANCH")"
+    local before new
+    before="$(result_dirs)"
+    (cd "$harness" && "${limit[@]}" julia --color=yes --project=. run.jl \
+        --config="$candidate/.buildkite/regression.toml" --fusion=on) ||
+        echo "Harness reported failures ($side)."
+    new="$(comm -13 <(sort <<<"$before") <(result_dirs | sort) | head -1)"
+    if [[ -n "$new" ]]; then
+        mkdir -p "$out/$side"
+        mv "$harness/results/$new" "$out/$side/results"
+    fi
+}
+
+# Build a side's wrapper from source when it differs from the release its own
+# checkout records. A base without RELEASED_COMMIT is checked against ours.
+wrapper_mode() {
+    local dir=$1
+    [[ -f "$dir/scripts/wrapper_changed.sh" ]] || dir="$candidate"
+    if (cd "$dir" && scripts/wrapper_changed.sh "$2" >&2); then
+        echo jll
+    else
+        local status=$?
+        ((status == 1)) || exit "$status"
+        echo developer
+    fi
+}
+
+base_mode="$(wrapper_mode "$base" "$(git -C "$base" rev-parse HEAD)")" || base_mode=developer
+candidate_mode="$(wrapper_mode "$candidate" HEAD)"
+if [[ "$base_mode" == developer || "$candidate_mode" == developer ]]; then
+    source .buildkite/install_cmake.sh
+fi
+
+echo "Comparing against $BASE_BRANCH."
+# Base failures only leave its results uncompared; the candidate must pass.
+run_side base "$base" "$base_mode" || echo "Base side failed; its results are not compared."
+run_side candidate "$candidate" "$candidate_mode"
+
+echo "--- :bar_chart: Compare"
+status=0
+julia --startup-file=no "$candidate/.buildkite/compare_regression.jl" \
+    "$out/base" "$out/candidate" --threshold="$THRESHOLD" --base="$BASE_BRANCH" \
+    --out="$out/report.md" ||
+    status=$?
+
+if [[ -f "$out/report.md" ]] && command -v buildkite-agent >/dev/null; then
+    style=$([[ $status == 0 ]] && echo success || echo error)
+    buildkite-agent annotate --context regression --style "$style" < "$out/report.md"
+fi
+exit "$status"