diff --git a/.github/workflows/preflight-selftest.yml b/.github/workflows/preflight-selftest.yml new file mode 100644 index 000000000..f165b8955 --- /dev/null +++ b/.github/workflows/preflight-selftest.yml @@ -0,0 +1,29 @@ +# Offline self-test for the STRICT mode of scripts/merge-preflight.sh. +# +# merge-preflight.yml only ever runs the script with --ci, and the +# required-checks leg runs only in strict mode, so before this job CI had zero +# coverage of the code that decides whether a merge is safe (DSR on #452). +# The suite replaces `gh` with a fixture server and never touches the network. +# +# No paths filter on purpose: a filtered workflow does not appear at all on PRs +# it skips, and "did not run" must not be mistaken for "passed". +name: preflight-selftest + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + preflight-strict-fixtures: + name: preflight strict-mode fixtures + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Strict-mode fixture suite (offline) + run: | + jq --version + python3 scripts/test_merge_preflight_strict.py diff --git a/scripts/merge-preflight.sh b/scripts/merge-preflight.sh index 6a9b72c65..9e7a36554 100755 --- a/scripts/merge-preflight.sh +++ b/scripts/merge-preflight.sh @@ -335,12 +335,21 @@ api total '.total_count' "repos/$REPO/commits/$head/check-runs" \ if [ "${total:-0}" -eq 0 ]; then echo "FAIL no check runs for $head — 'all green' and 'never ran' are not the same reading"; fail=1 else - api allcheck '[.check_runs[].name]|join("\n")' \ + # A name containing a newline would forge a boundary in the newline-joined + # set the required-contexts leg compares against (Codex round 3): refuse it. + api allcheck '[.check_runs[].name]|if any(.[]; type!="string" or test("[[:cntrl:]]")) then error("control character in check name") else join("\n") end' \ "repos/$REPO/commits/$head/check-runs" --paginate \ || { echo "PREFLIGHT FAIL — do not merge $PR"; exit 4; } - base_for_req=$(gh pr view "$PR" --repo "$REPO" --json baseRefName -q '.baseRefName' 2>/dev/null | tr -d ' \n') + base_for_req=$(gh pr view "$PR" --repo "$REPO" --json baseRefName -q '.baseRefName' 2>/dev/null | tr -d ' \n') \ + || base_for_req="" # a failed lookup that printed something is still unread - api bad '[.check_runs[]|select(.conclusion=="failure" or .conclusion=="timed_out" or .conclusion=="cancelled" or .conclusion=="action_required")|.name]|join("\n")' \ + # Run records for the required-checks allowlist (strict leg below). + api runsj '[.check_runs[]|{n:.name,a:.app.id,s:.status,c:.conclusion,i:.id}]|tojson' \ + "repos/$REPO/commits/$head/check-runs" --paginate \ + || { echo "PREFLIGHT FAIL — do not merge $PR"; exit 4; } + # `stale` is a legal GitHub conclusion for a run abandoned before completing; + # it is not success and must not read as quiet (DSR on #452). + api bad '[.check_runs[]|select(.conclusion=="failure" or .conclusion=="timed_out" or .conclusion=="cancelled" or .conclusion=="action_required" or .conclusion=="stale")|.name]|join("\n")' \ "repos/$REPO/commits/$head/check-runs" --paginate \ || { echo "PREFLIGHT FAIL — do not merge $PR"; exit 4; } api pend '[.check_runs[]|select(.status!="completed")|.name]|join("\n")' \ @@ -466,7 +475,17 @@ else # return value and not the side effect, so an unreadable base still failed the # run while printing "not claiming". Same shape as every other mismatch today: # the verdict arriving by a path other than the one being read. - if [ -z "$base_for_req" ]; then + # Branch names are path segments in every endpoint below and must be + # URL-encoded: `feat/x#y` is a legal ref, and unencoded the request goes to + # `feat/x` (the fragment is dropped) - a different branch whose protection + # would then be reported as this PR's. Found by Codex. Encoding failure + # leaves base_enc empty, which the first branch below refuses - including a + # jq that printed the value and then exited non-zero (Codex round 2). + base_enc="" + if [ -n "$base_for_req" ]; then + base_enc=$(jq -rn --arg b "$base_for_req" '$b|@uri' 2>/dev/null) || base_enc="" + fi + if [ -z "$base_for_req" ] || [ -z "$base_enc" ]; then # No `:-main` fallback. That silent substitution was removed three commits # ago and the reasoning is still on line ~143 of this file; reinstating it # here would report a different branch's configuration as this PR's. @@ -476,49 +495,108 @@ else # strictest. Found by Codex. echo "FAIL base branch unreadable; cannot check required contexts" fail=1 - elif prot=$(gh api "repos/$REPO/branches/$base_for_req/protection" 2>/dev/null) \ - && [ "$(printf '%s' "$prot" | jq -r 'has("enforce_admins")' 2>/dev/null)" = "true" ]; then - # TWO steps, because `// []` alone cannot tell "this branch requires - # nothing" from "I could not see what it requires". A partial response, a - # permission problem, or an error object all leave the field absent, and - # `// []` turned every one of them into "requires nothing" — which PASSES. - # That silently converted a fail-closed path into a fail-open one, in the - # commit that was fixing a misdiagnosis. Found by Codex. + else + # The required set is the UNION of what classic branch protection and + # repository rulesets require; either can require checks without the + # other (Codex round 3). Each source is read fail-closed and VALIDATED, + # not normalized: `// []` on a malformed field reads exactly like + # "requires nothing". + # + # A requirement is {context, app_id}; app_id null = any source. A + # requirement is satisfied only by the LATEST run with that name (and, + # when app_id is set, from that App) being completed with conclusion + # "success" - an allowlist, not a failure blacklist. Presence by name + # alone let skipped/neutral/stale runs and same-named runs from another + # App count as "reported" (DSR + pr-daemon on #452, rounds 2-4). # - # So: first prove a real protection object was read (`enforce_admins` is present on every - # protection object and on nothing else; `.url` was the first choice and did - # NOT work — the required_signatures sub-endpoint returns {url, enabled} and - # sailed straight through the guard), and only - # then treat a missing required_status_checks as a genuine zero. - reqctx=$(printf '%s' "$prot" | jq -r '(.required_status_checks.contexts // [])|join("\n")' 2>/dev/null) - # `// []` above matters. Without it, a branch WITH protection but NO required - # status checks makes jq error on null ("Cannot iterate over null", rc 5), gh - # returns 1, and this lands in the unreadable branch — reporting "could not - # read" for a branch that simply requires nothing, so strict could never pass - # there. Verified with jq directly, both ways. Raised by pr-daemon. - if [ -z "$reqctx" ]; then - echo "INFO $base_for_req requires no status checks; nothing to verify reported" + # Names must contain a non-space and no control character: a NUL is + # stripped by bash command substitution, and the old newline-joined + # comparison let a newline forge a boundary (Codex rounds 3-4). + JQ_DEFS='def name_ok: type=="string" and test("\\S") and (test("[[:cntrl:]]")|not); + def app_ok: . == null or (type=="number" and . > 0 and . == floor); + def req($c; $a): if ($c|name_ok) and ($a|app_ok) then {context:$c, app_id:$a} else error("unread") end;' + src_ok=1; classic="[]"; classic_desc=""; rsets="[]" + if prot=$(gh api "repos/$REPO/branches/$base_enc/protection" 2>/dev/null); then + # A real protection object: enforce_admins is an object and url a + # string (`.url` alone is not enough - the required_signatures + # sub-endpoint returns {url, enabled}). required_status_checks null is + # a genuine zero; present but malformed - contexts missing/null/false, + # checks not an array of objects, a bad name or app_id - is unread. + classic=$(printf '%s' "$prot" | jq -ec "$JQ_DEFS"' + if type!="object" or (.enforce_admins|type)!="object" or (.url|type)!="string" then error("unread") else + .required_status_checks as $r + | if $r == null then [] + elif ($r|type)!="object" or ($r.contexts|type)!="array" then error("unread") + else [ $r.contexts[] | req(.; null) ] + + ( if ($r|has("checks")) then $r.checks else [] end + | if type=="array" and all(.[]; type=="object") then [ .[] | req(.context; .app_id) ] else error("unread") end ) + end end' 2>/dev/null) && classic_desc="protected" || src_ok=0 else - missing="" - while IFS= read -r c; do - [ -z "$c" ] && continue - printf '%s\n' "$allcheck" | grep -qxF "$c" || missing="${missing:+$missing, }$c" - done <<< "$reqctx" - if [ -n "$missing" ]; then - echo "FAIL required context(s) never reported on this head: $missing" - echo " A required check that did not run is ABSENT, not passing." - echo " Usually a paths filter: the workflow was not triggered by these" - echo " files. GitHub blocks on it; re-push or widen the filter." + # 404 "Branch not protected" has the same shape as "no permission". + # Accept "no classic protection" only on a positive read of the branch + # object: a boolean protected == false. tostring because jq -e exits 1 + # when the result IS false (caught by the positive controls). + unprot=$(gh api "repos/$REPO/branches/$base_enc" 2>/dev/null) \ + && unprot=$(printf '%s' "$unprot" | jq -er 'if type=="object" and (.protected|type)=="boolean" then (.protected|tostring) else error("unread") end' 2>/dev/null) \ + && [ "$unprot" = "false" ] && classic_desc="unprotected" || src_ok=0 + fi + if [ "$src_ok" -eq 1 ]; then + # Rulesets, every page (>= 1). Each rule must be an object with a + # non-empty string type; each required_status_checks rule must carry a + # readable list of {context, integration_id?}. + rsets=$(gh api --paginate --slurp "repos/$REPO/rules/branches/$base_enc" 2>/dev/null) \ + && rsets=$(printf '%s' "$rsets" | jq -ec "$JQ_DEFS"' + if type=="array" and length>0 and all(.[]; type=="array") then add else error("unread") end + | if all(.[]; type=="object" and (.type|type)=="string" and (.type|length)>0) then . else error("unread") end + | [ .[] | select(.type=="required_status_checks") + | .parameters.required_status_checks + | if type=="array" and all(.[]; type=="object") then .[] else error("unread") end + | req(.context; .integration_id) ]' 2>/dev/null) || src_ok=0 + fi + # Every check run on this head, for the allowlist below. Pages arrive + # as one JSON array per line; each record must be well-formed. + if [ "$src_ok" -eq 1 ]; then + runs=$(printf '%s\n' "$runsj" | jq -ecs ' + if all(.[]; type=="array") then add // [] else error("unread") end + | if all(.[]; type=="object" and (.n|type)=="string" and (.i|type)=="number" and (.s|type)=="string" + and ((.a|type)=="number" or .a==null) and ((.c|type)=="string" or .c==null)) + then . else error("unread") end' 2>/dev/null) || src_ok=0 + fi + if [ "$src_ok" -ne 1 ]; then + echo "FAIL could not read $base_for_req's required checks (classic protection, rulesets or check runs)." + echo " Not proceeding on an unread value: whether every required check" + echo " succeeded is exactly what this leg exists to establish." + fail=1 + elif ! verdict=$(jq -ern --argjson a "$classic" --argjson b "$rsets" --argjson runs "$runs" ' + # union; a specific-App requirement supersedes an any-App one for the same context + ($a + $b | unique) as $all + | [ $all[] as $r + | select($r.app_id != null or ([ $all[] | select(.context == $r.context and .app_id != null) ] | length) == 0) + | $r ] as $req + | [ $req[] as $r + | ([ $runs[] | select(.n == $r.context and ($r.app_id == null or .a == $r.app_id)) ] | max_by(.i)) as $l + | if $l == null then "\($r.context) [absent\(if $r.app_id != null then " from app \($r.app_id)" else "" end)]" + elif $l.s != "completed" then "\($r.context) [status=\($l.s)]" + elif $l.c != "success" then "\($r.context) [conclusion=\($l.c)]" + else empty end ] as $bad + | "\($req|length)\t\($bad|join("; "))"' 2>/dev/null); then + echo "FAIL could not evaluate $base_for_req's required checks; refusing on an unread value" fail=1 else - echo "OK all $(printf '%s\n' "$reqctx" | grep -c .) required contexts reported" + nreq=${verdict%% *}; unmet=${verdict#* } + if [ "$nreq" = "0" ]; then + echo "INFO $base_for_req ($classic_desc, no ruleset check rule) requires no status checks;" + echo " nothing to verify reported" + elif [ -n "$unmet" ]; then + echo "FAIL required check(s) not satisfied on this head: $unmet" + echo " Only the latest run of a required check, from the required App," + echo " completed with conclusion=success counts. A required check that" + echo " did not run is ABSENT, not passing; skipped/neutral/stale are not success." + fail=1 + else + echo "OK all $nreq required checks succeeded ($classic_desc + rulesets)" + fi fi - fi - else - echo "FAIL could not read $base_for_req's required contexts." - echo " Not proceeding on an unread value: whether every required check" - echo " reported is exactly what this leg exists to establish." - fail=1 fi fi fi diff --git a/scripts/test_merge_preflight_strict.py b/scripts/test_merge_preflight_strict.py new file mode 100644 index 000000000..639ed1d66 --- /dev/null +++ b/scripts/test_merge_preflight_strict.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Offline fixture tests for the STRICT (non --ci) mode of scripts/merge-preflight.sh. + +The workflow that runs merge-preflight.sh only ever uses --ci, and the +required-checks leg runs only in strict mode, so without this suite CI had zero +coverage of the code that decides whether a merge is safe (DSR on #452). + +A fake `gh` is put first on PATH. It serves JSON fixtures per endpoint and +emulates the three gh behaviours the script depends on: `--jq` (applied per +page with real jq, raw output), `--paginate` (one output per page) and +`--slurp` (all pages as one JSON array). Nothing touches the network. + +Every scenario asserts BOTH the overall exit status AND the exact line the +required-checks leg printed, so a scenario cannot pass for an unrelated reason +(e.g. failing early on the approval leg). + +Usage: python3 scripts/test_merge_preflight_strict.py (exit 0 = all pass) +""" +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.environ.get("PREFLIGHT_SCRIPT") or os.path.join(ROOT, "scripts", "merge-preflight.sh") +REPO = "o/r" +PR = "7" +HEAD = "a" * 40 +APP = 15368 + +FAKE_GH = r'''#!/usr/bin/env python3 +import json, os, subprocess, sys +fx = json.load(open(os.environ["FAKE_GH_FIXTURE"])) +a = sys.argv[1:] +with open(os.environ["FAKE_GH_LOG"], "a") as f: + f.write(json.dumps(a) + "\n") + +def jq(expr, doc): + p = subprocess.run(["jq", "-r", expr], input=json.dumps(doc), capture_output=True, text=True) + sys.stdout.write(p.stdout) + return p.returncode + +if a[:2] == ["pr", "view"]: + expr = a[a.index("-q") + 1] + sys.exit(1 if jq(expr, fx["pr"]) else 0) + +if a and a[0] == "api": + rest, flags, expr, i = a[1:], set(), None, 0 + paths = [] + while i < len(rest): + t = rest[i] + if t in ("--paginate", "--slurp"): + flags.add(t) + elif t in ("--jq", "-q"): + expr = rest[i + 1]; i += 1 + else: + paths.append(t) + i += 1 + ep = fx["api"].get(paths[0]) + if ep is None: + print(json.dumps({"message": "Not Found"})) + sys.exit(1) + if "raw" in ep: + sys.stdout.write(ep["raw"]) + sys.exit(ep.get("rc", 0)) + pages = ep["pages"] + if ep.get("rc", 0): + print(json.dumps(pages[0] if pages else {})) + sys.exit(ep["rc"]) + if "--slurp" in flags: + print(json.dumps(pages)); sys.exit(0) + for pg in pages: + if expr is not None: + if jq(expr, pg): + sys.exit(1) + else: + print(json.dumps(pg)) + sys.exit(0) +sys.exit(2) +''' + +# jq wrapper: makes one specific jq invocation exit non-zero AFTER printing, +# to prove the script checks jq's status and not only its output. +FAKE_JQ = r'''#!/bin/bash +real=$(PATH="${FAKE_REAL_PATH}" command -v jq) +if [ -n "${FAKE_JQ_FAIL_ON:-}" ] && [[ "$*" == *"${FAKE_JQ_FAIL_ON}"* ]]; then + "$real" "$@"; exit 7 +fi +exec "$real" "$@" +''' + + +def run_rec(name, conclusion="success", status="completed", app=APP, rid=1): + return {"name": name, "status": status, "conclusion": conclusion, "id": rid, + "app": {"id": app}, "details_url": f"https://x/runs/1/job/{rid}"} + + +def base_fixture(base): + runs = [run_rec("build", rid=11), run_rec("test", rid=12), + run_rec("cla-check", rid=13), run_rec("preflight-report", rid=14)] + enc = base.replace("/", "%2F").replace("#", "%23") + api = { + f"repos/{REPO}/pulls/{PR}/reviews": {"pages": [[{ + "state": "APPROVED", "commit_id": HEAD, "submitted_at": "2026-10-01T00:00:00Z", + "body": f"APPROVE at {HEAD}"}]]}, + f"repos/{REPO}/issues/{PR}/timeline": {"pages": [[]]}, + f"repos/{REPO}/pulls/{PR}/commits": {"pages": [[ + {"commit": {"committer": {"date": "2026-09-30T00:00:00Z"}}}]]}, + f"repos/{REPO}/commits/{HEAD}/check-runs": {"pages": [ + {"total_count": len(runs), "check_runs": runs}]}, + f"repos/{REPO}/commits/{HEAD}/status": {"pages": [{"state": "pending", "statuses": []}]}, + f"repos/{REPO}/rules/branches/{enc}": {"pages": [[]]}, + } + if base == "main": + api[f"repos/{REPO}/branches/main/protection"] = {"pages": [{ + "url": f"https://api.github.com/repos/{REPO}/branches/main/protection", + "enforce_admins": {"enabled": False}, + "required_pull_request_reviews": {"dismiss_stale_reviews": True}, + "required_status_checks": { + "contexts": ["build", "test"], + "checks": [{"context": "build", "app_id": APP}, {"context": "test", "app_id": APP}]}}]} + api[f"repos/{REPO}/branches/main"] = {"pages": [{"name": "main", "protected": True}]} + else: + api[f"repos/{REPO}/branches/{enc}/protection"] = { + "rc": 1, "pages": [{"message": "Branch not protected"}]} + api[f"repos/{REPO}/branches/{enc}"] = {"pages": [{"name": base, "protected": False}]} + return {"pr": {"headRefOid": HEAD, "baseRefName": base, "reviewDecision": "APPROVED"}, "api": api} + + +U, P = "feat/x", "main" +UE = "feat%2Fx" +PROT = f"repos/{REPO}/branches/main/protection" +CR = f"repos/{REPO}/commits/{HEAD}/check-runs" + + +def set_prot(fx, rsc=None, raw=None, **top): + if raw is not None: + fx["api"][PROT] = {"raw": raw} + return + body = {"url": "https://u", "enforce_admins": {"enabled": False}} + body.update(top) + if rsc is not ...: + body["required_status_checks"] = rsc + fx["api"][PROT] = {"pages": [body]} + + +def set_rules(fx, base, pages=None, rc=0, raw=None): + k = f"repos/{REPO}/rules/branches/{base}" + fx["api"][k] = {"raw": raw, "rc": rc} if raw is not None else {"pages": pages, "rc": rc} + + +def set_branch(fx, body=None, rc=0, raw=None): + k = f"repos/{REPO}/branches/{UE}" + fx["api"][k] = {"raw": raw, "rc": rc} if raw is not None else {"pages": [body], "rc": rc} + + +def set_runs(fx, runs): + fx["api"][CR] = {"pages": [{"total_count": len(runs), "check_runs": runs}]} + + +def rs_rule(*ctx): + return {"type": "required_status_checks", "parameters": { + "required_status_checks": [dict(c) for c in ctx]}} + + +OK_ALL = r"^OK all \d+ required checks succeeded" +INFO_NONE = r"^INFO .* requires no status checks" +UNREAD = r"^FAIL could not read .* required checks" +UNMET = r"^FAIL required check\(s\) not satisfied on this head: " +BASE_UNREAD = r"^FAIL base branch unreadable" + +# (name, base, mutate, expect_exit_zero, leg_regex, env) +S = [] +def sc(name, base, mut, ok, leg, env=None): + S.append((name, base, mut, ok, leg, env or {})) + +# --- positive controls ------------------------------------------------------- +sc("unprotected, no rulesets", U, lambda f: None, True, INFO_NONE) +sc("protected, contexts+checks all succeeded", P, lambda f: None, True, OK_ALL) +sc("unprotected, ruleset requires present cla-check", U, + lambda f: set_rules(f, UE, [[rs_rule({"context": "cla-check"})]]), True, OK_ALL) +sc("unprotected, ruleset integration matches", U, + lambda f: set_rules(f, UE, [[rs_rule({"context": "cla-check", "integration_id": APP})]]), True, OK_ALL) +sc("unprotected, non-check rule only", U, lambda f: set_rules(f, UE, [[{"type": "deletion"}]]), True, INFO_NONE) +sc("protected, required_status_checks null = genuine zero", P, lambda f: set_prot(f, rsc=None), True, INFO_NONE) +sc("protected, latest run of a check is success after an older one", P, + lambda f: set_runs(f, [run_rec("build", "neutral", rid=1), run_rec("build", rid=11), + run_rec("test", rid=12), run_rec("preflight-report", rid=14)]), True, OK_ALL) + +# --- unprotected base: classic-protection read -------------------------------- +sc("branch body has no protected field", U, lambda f: set_branch(f, {"name": "x"}), False, UNREAD) +sc("protected is the string 'false'", U, lambda f: set_branch(f, {"protected": "false"}), False, UNREAD) +sc("gh exits 1 but prints protected:false", U, lambda f: set_branch(f, {"protected": False}, rc=1), False, UNREAD) +sc("branch body not JSON", U, lambda f: set_branch(f, raw="not json"), False, UNREAD) +sc("base name with # is URL-encoded (unknown branch -> unread)", U, + lambda f: f["pr"].update(baseRefName="feat/x#y"), False, UNREAD) +sc("@uri encoding: jq prints then exits 7", U, lambda f: None, False, BASE_UNREAD, + {"FAKE_JQ_FAIL_ON": "@uri"}) + +# --- rulesets ----------------------------------------------------------------- +sc("rules: gh exits 1 printing [[]]", U, lambda f: set_rules(f, UE, [[]], rc=1), False, UNREAD) +sc("rules: object instead of pages", U, lambda f: set_rules(f, UE, raw='{"message":"x"}'), False, UNREAD) +sc("rules: not JSON", U, lambda f: set_rules(f, UE, raw="[[ broken"), False, UNREAD) +sc("rules: zero pages []", U, lambda f: set_rules(f, UE, []), False, UNREAD) +sc("rules: entry {}", U, lambda f: set_rules(f, UE, [[{}]]), False, UNREAD) +sc("rules: entry null", U, lambda f: set_rules(f, UE, [[None]]), False, UNREAD) +sc("rules: type 42", U, lambda f: set_rules(f, UE, [[{"type": 42}]]), False, UNREAD) +sc("rules: type empty string", U, lambda f: set_rules(f, UE, [[{"type": ""}]]), False, UNREAD) +sc("rules: required_status_checks without parameters", U, + lambda f: set_rules(f, UE, [[{"type": "required_status_checks", "parameters": {}}]]), False, UNREAD) +sc("rules: requirement on page 2 is absent", U, + lambda f: set_rules(f, UE, [[{"type": "deletion"}], [rs_rule({"context": "security"})]]), False, UNMET) +sc("rules: required check from the wrong integration", U, + lambda f: set_rules(f, UE, [[rs_rule({"context": "cla-check", "integration_id": 999})]]), False, UNMET) +sc("protected base also honours rulesets", P, + lambda f: set_rules(f, "main", [[rs_rule({"context": "security"})]]), False, UNMET) + +# --- protected base: protection object shape ----------------------------------- +sc("protection: garbage after object", P, lambda f: set_prot(f, raw='{"enforce_admins":{},"url":"u"} garbage'), False, UNREAD) +sc("protection: enforce_admins null", P, lambda f: set_prot(f, rsc=None, enforce_admins=None), False, UNREAD) +sc("protection: no url", P, lambda f: f["api"][PROT]["pages"][0].pop("url"), False, UNREAD) +sc("protection: required_status_checks {}", P, lambda f: set_prot(f, rsc={}), False, UNREAD) +sc("protection: contexts null", P, lambda f: set_prot(f, rsc={"contexts": None}), False, UNREAD) +sc("protection: contexts 42", P, lambda f: set_prot(f, rsc={"contexts": 42}), False, UNREAD) +sc("protection: contexts false with checks", P, + lambda f: set_prot(f, rsc={"contexts": False, "checks": [{"context": "build"}]}), False, UNREAD) +sc("protection: checks false", P, lambda f: set_prot(f, rsc={"contexts": [], "checks": False}), False, UNREAD) +sc("protection: empty name", P, lambda f: set_prot(f, rsc={"contexts": [""]}), False, UNREAD) +sc("protection: whitespace-only name", P, lambda f: set_prot(f, rsc={"contexts": [" \t"]}), False, UNREAD) +sc("protection: NUL inside name", P, lambda f: set_prot(f, rsc={"contexts": ["bu\u0000ild"]}), False, UNREAD) +sc("protection: newline inside name", P, lambda f: set_prot(f, rsc={"contexts": ["build\ntest"]}), False, UNREAD) +sc("protection: app_id is a string", P, + lambda f: set_prot(f, rsc={"contexts": [], "checks": [{"context": "build", "app_id": "15368"}]}), False, UNREAD) +sc("protection: checks-only requirement absent", P, + lambda f: set_prot(f, rsc={"contexts": [], "checks": [{"context": "never-ran", "app_id": APP}]}), False, UNMET) +sc("protection: '-ebuild' must not act as a grep option", P, + lambda f: set_prot(f, rsc={"contexts": ["-ebuild"]}), False, UNMET) + +# --- conclusion allowlist and App identity -------------------------------------- +def runs_with(build): + return lambda f: set_runs(f, [build, run_rec("test", rid=12), run_rec("preflight-report", rid=14)]) +sc("required check skipped", P, runs_with(run_rec("build", "skipped", rid=11)), False, UNMET) +sc("required check neutral", P, runs_with(run_rec("build", "neutral", rid=11)), False, UNMET) +sc("required check stale", P, runs_with(run_rec("build", "stale", rid=11)), False, UNMET) +sc("required check still running", P, runs_with(run_rec("build", None, "in_progress", rid=11)), False, UNMET) +sc("required check from another App (same name)", P, runs_with(run_rec("build", app=999, rid=11)), False, UNMET) +sc("latest run failed after an older success", P, + lambda f: set_runs(f, [run_rec("build", rid=1), run_rec("build", "failure", rid=11), + run_rec("test", rid=12), run_rec("preflight-report", rid=14)]), False, UNMET) +sc("check-run record malformed (id missing)", P, + lambda f: set_runs(f, [{k: v for k, v in run_rec("build", rid=11).items() if k != "id"}, + run_rec("test", rid=12), run_rec("preflight-report", rid=14)]), False, UNREAD) +sc("required-set union: jq prints then exits 7", P, lambda f: None, False, + r"^FAIL could not evaluate", {"FAKE_JQ_FAIL_ON": "--argjson runs"}) + +LEG = re.compile(r"^(OK all \d+ required checks|INFO .*requires no status checks|" + r"FAIL (could not (read|evaluate) .*required checks|required check\(s\) not satisfied|base branch unreadable))") + + +def main(): + tmp = tempfile.mkdtemp(prefix="preflight-strict-") + try: + bindir = os.path.join(tmp, "bin") + os.mkdir(bindir) + for name, body in (("gh", FAKE_GH), ("jq", FAKE_JQ)): + p = os.path.join(bindir, name) + open(p, "w").write(body) + os.chmod(p, 0o755) + failures = 0 + for i, (name, base, mut, ok, leg, env) in enumerate(S): + fx = base_fixture(base) + mut(fx) + fpath = os.path.join(tmp, f"fx{i}.json") + json.dump(fx, open(fpath, "w")) + e = dict(os.environ, FAKE_GH_FIXTURE=fpath, FAKE_GH_LOG=os.path.join(tmp, "calls.log"), + FAKE_REAL_PATH=os.environ["PATH"], PATH=bindir + os.pathsep + os.environ["PATH"], + REPO=REPO, GH_TOKEN="offline") + # The script derives SELF_NAME from GITHUB_JOB / GITHUB_RUN_ID; inside + # Actions those name THIS test job, not the fixture's run. Strip every + # GITHUB_* and pin SELF_NAME so local and CI runs see the same input + # (the first CI run failed all 7 positive controls on exactly this). + for k in [k for k in e if k.startswith("GITHUB_")]: + e.pop(k) + e["SELF_NAME"] = "preflight-report" + e.update(env) + p = subprocess.run(["bash", SCRIPT, PR], capture_output=True, text=True, env=e, timeout=120) + out = p.stdout + p.stderr + legs = [l for l in out.splitlines() if LEG.match(l)] + got_leg = legs[0] if legs else "" + exit_ok = (p.returncode == 0) == ok + leg_ok = re.search(leg, got_leg) is not None + mark = "ok " if exit_ok and leg_ok else "FAIL" + if not (exit_ok and leg_ok): + failures += 1 + print(f"{mark} [{'PASS' if ok else 'FAIL'} expected, exit {p.returncode}] {name}") + if not (exit_ok and leg_ok): + print(f" leg: {got_leg}") + print(" output:\n" + "\n".join(" " + l for l in out.splitlines())) + print(f"\n{len(S) - failures}/{len(S)} scenarios as expected") + return 1 if failures else 0 + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main())