HYP-S: delegate language-ban + sha-pinning criteria to live scanners (coverage 66→73%)

lib/hypatia/cli.ex

@@ -297,7 +297,16 @@ defmodule Hypatia.CLI do
 
   # ─── Finding collection across rule modules ──────────────────────────
 
-  defp collect_findings(repo_path, rules) do
+  @doc """
+  Run the named rule modules against `repo_path` and return normalized findings
+  (`%{rule_module, type, severity, file, reason, action}`). Public so the RSR
+  conformance oracle can delegate content-scan criteria to the live scanners
+  rather than reimplement per-file detection. `rules` is a list of module atoms
+  (e.g. `[:cicd_rules, :structural_drift]`); GitHub-API modules
+  (`:dependabot_alerts`, `:secret_scanning_alerts`, `:code_scanning_alerts`,
+  `:scorecard`) require network + token and return nothing offline.
+  """
+  def collect_findings(repo_path, rules) do
     results = []
 
     # Root Hygiene

lib/rules/rsr_conformance.ex

@@ -20,16 +20,26 @@ defmodule Hypatia.Rules.RsrConformance do
     * score = passed weight / applicable **verified** weight;
     * tier from the catalogue's thresholds.
 
-  Detection, however, is deliberately partial in this increment: a built-in
-  table of ~30 file-presence/parse checks covers the mechanically-checkable
-  criteria (community-health files, the descriptile substrate, workflows).
-  Every other criterion — including all `detect = "manual"` ones and those
-  whose `detect` names a scanner rule this oracle does not yet execute — is
-  returned as `:unverified`, counted in `automatable_coverage`, and **never**
-  silently passed. Because coverage < 100%, every scorecard this increment
-  produces carries `provisional = true`: per the no-overclaim doctrine a firm
-  tier claim requires full detection coverage (and the spec's ratification
-  bar requires exactly that before RSR v2.0 leaves Draft).
+  Detection is layered:
+
+    1. **Delegated** — criteria whose detection is *authoritative* only via a
+       live scanner run once per score (`delegated_index/1` →
+       `Hypatia.CLI.collect_findings/2`): the language bans (carve-out-aware,
+       via `cicd_rules/banned_language_file`) and SHA-pinning (via
+       `workflow_audit/{unpinned_action,wrong_sha_pin}`). Mapping is keyed on
+       *observed* `(module,type)` finding keys, which deliberately do NOT match
+       the SSOT's aspirational `detect` strings.
+    2. **File-presence/parse tranche** — a built-in table for community-health
+       files, the descriptile substrate, workflows, REUSE dir, guix stub.
+    3. Everything else — `detect = "manual"`, intrinsically external criteria
+       (OpenSSF Scorecard `4.2.x`), and flag-only licence (`7.1.1`) — is
+       returned `:unverified`, counted in `automatable_coverage`, and **never**
+       silently passed.
+
+  Because some criteria are intrinsically external or flag-only, coverage < 100%
+  is expected, so every scorecard carries `provisional = true`: per the
+  no-overclaim doctrine a firm tier claim requires either full offline coverage
+  or an accepted external attestation, a spec decision for RSR v2.0.x (§10).
 
   ## Weight interpretation
 
@@ -69,10 +79,11 @@ defmodule Hypatia.Rules.RsrConformance do
   def score(%{criteria: criteria} = catalogue, repo_path) do
     caps = declared_capabilities(repo_path)
     weights = criterion_weights(catalogue)
+    delegated = delegated_index(repo_path)
 
     results =
       Enum.map(criteria, fn %Criterion{} = c ->
-        verdict = evaluate(c, repo_path, caps)
+        verdict = evaluate(c, repo_path, caps, delegated)
 
         %{
           id: c.id,
@@ -238,17 +249,80 @@ defmodule Hypatia.Rules.RsrConformance do
 
   # --- evaluation ------------------------------------------------------------
 
-  defp evaluate(%Criterion{} = c, repo, caps) do
+  defp evaluate(%Criterion{} = c, repo, caps, delegated) do
     if applicable?(c, caps) do
-      case Map.fetch(detectors(), c.id) do
-        {:ok, fun} -> fun.(repo)
-        :error -> :unverified
+      # Criteria whose detection is DELEGATED to a live scanner take priority
+      # over the file-presence tranche (they are authoritative — e.g. the
+      # language bans respect the estate carve-outs, which a raw extension
+      # check cannot). `:skip` falls through to the built-in detector table,
+      # then to :unverified.
+      case delegated_verdict(c.id, delegated) do
+        :skip ->
+          case Map.fetch(detectors(), c.id) do
+            {:ok, fun} -> fun.(repo)
+            :error -> :unverified
+          end
+
+        verdict ->
+          verdict
       end
     else
       :na
     end
   end
 
+  # --- delegation to live scanner rules --------------------------------------
+
+  # Run the offline content-scanners ONCE per score and index the findings the
+  # oracle can map to a criterion. Observed (module,type) keys — not the SSOT's
+  # aspirational `detect` strings — drive the mapping (the two disagree):
+  #   * language bans  -> cicd_rules/banned_language_file, keyed by file ext,
+  #     already carve-out-filtered by the scanner;
+  #   * SHA pinning    -> workflow_audit/{unpinned_action,wrong_sha_pin}.
+  # A scan failure degrades to %{} (every delegated criterion then :skip's to
+  # the file-presence tranche / :unverified) — never a false pass, never a crash.
+  defp delegated_index(repo) do
+    findings = safe_scan(repo, [:cicd_rules, :workflow_audit])
+
+    banned_exts =
+      for f <- findings,
+          f.rule_module == "cicd_rules",
+          f.type == "banned_language_file",
+          into: MapSet.new(),
+          do: f.file |> to_string() |> Path.extname()
+
+    unpinned? =
+      Enum.any?(findings, fn f ->
+        f.rule_module == "workflow_audit" and f.type in ["unpinned_action", "wrong_sha_pin"]
+      end)
+
+    %{banned_exts: banned_exts, unpinned: unpinned?, scanned: findings != [] or true}
+  end
+
+  defp safe_scan(repo, rules) do
+    Hypatia.CLI.collect_findings(repo, rules)
+  rescue
+    _ -> []
+  catch
+    _, _ -> []
+  end
+
+  # Verdict for a delegated criterion, or `:skip` if this criterion is not
+  # delegated. A language ban PASSES iff the scanner emitted no
+  # banned_language_file finding for that extension (no such file, or every
+  # such file is inside an approved carve-out — both are compliant).
+  defp delegated_verdict("5.1.1", d), do: ban_verdict(d, [".py"])
+  defp delegated_verdict("5.1.2", d), do: ban_verdict(d, [".ts", ".tsx"])
+  defp delegated_verdict("5.1.3", d), do: ban_verdict(d, [".res", ".resi"])
+  defp delegated_verdict("5.1.5", d), do: ban_verdict(d, [".go"])
+  defp delegated_verdict("4.1.3", %{unpinned: true}), do: :fail
+  defp delegated_verdict("4.1.3", %{unpinned: false}), do: :pass
+  defp delegated_verdict(_, _), do: :skip
+
+  defp ban_verdict(%{banned_exts: exts}, wanted) do
+    if Enum.any?(wanted, &MapSet.member?(exts, &1)), do: :fail, else: :pass
+  end
+
   # Built-in detector tranche: file-presence and record-dialect-parse checks,
   # keyed by criterion id from the SSOT. Criteria not listed here are
   # :unverified — reported, never assumed. Extending this table (or delegating
@@ -293,14 +367,12 @@ defmodule Hypatia.Rules.RsrConformance do
       "6.1.2" => present(".github/workflows/hypatia-scan.yml"),
       "6.1.3" => present(".github/workflows/governance.yml"),
       "6.2.1" => present(".github/workflows/dogfood-gate.yml"),
-      # Language-policy bans that are unambiguous file-presence checks (no
-      # estate carve-outs of consequence). TS/ReScript (5.1.2/5.1.3) are NOT
-      # here: their carve-out logic belongs to the live cicd_rules scanner, so
-      # they stay :unverified rather than risk a false positive.
+      # Language bans 5.1.1/5.1.2/5.1.3/5.1.5 and SHA-pinning 4.1.3 are handled
+      # by delegated_verdict/2 (live scanner, carve-out-aware) — not here.
+      # v.mod / package-lock.json have no carve-out subtlety, so a file-presence
+      # check is authoritative for them.
       "1.2.1" => any_of(["guix.scm", "build/guix.scm"]),
-      "5.1.1" => no_ext([".py"]),
       "5.1.4" => no_file_named("v.mod"),
-      "5.1.5" => no_ext([".go"]),
       "5.1.6" => no_file_named("package-lock.json"),
       "7.1.2" => present("LICENSES"),
       "8.1.4" => &guix_not_stub/1,
@@ -387,13 +459,6 @@ defmodule Hypatia.Rules.RsrConformance do
     if yml ++ yaml == [], do: :fail, else: :pass
   end
 
-  # :pass iff NO tracked file with one of `exts` exists anywhere in the tree
-  # (vendored/build/.git dirs excluded). Used only for unambiguous language
-  # bans — Python, Go — where the estate has no carve-out.
-  defp no_ext(exts) do
-    fn repo -> if tree_hits(repo, Enum.map(exts, &("*" <> &1))) == [], do: :pass, else: :fail end
-  end
-
   # :pass iff NO file with exactly `name` exists anywhere in the tree.
   defp no_file_named(name) do
     fn repo -> if tree_hits(repo, [name]) == [], do: :pass, else: :fail end

test/rules/rsr_conformance_test.exs

@@ -241,6 +241,46 @@ defmodule Hypatia.Rules.RsrConformanceTest do
     end
   end
 
+  describe "delegation to live scanner rules" do
+    @describetag :tmp_dir
+
+    test "a planted .py fails 5.1.1 via the carve-out-aware scanner", %{tmp_dir: tmp} do
+      mk!(tmp, "src/bad.py", "print('x')\n")
+      {:ok, catalogue} = RsrCriteria.load(@ssot)
+      {:ok, sc} = RsrConformance.score(catalogue, tmp)
+      verdicts = Map.new(sc.results, &{&1.id, &1.verdict})
+      assert verdicts["5.1.1"] == :fail
+    end
+
+    test "an unpinned action fails 4.1.3 (sha-pinning delegated to workflow_audit)", %{
+      tmp_dir: tmp
+    } do
+      mk!(tmp, ".github/workflows/w.yml", """
+      name: x
+      jobs:
+        a:
+          steps:
+            - uses: actions/checkout@v4
+      """)
+
+      {:ok, catalogue} = RsrCriteria.load(@ssot)
+      {:ok, sc} = RsrConformance.score(catalogue, tmp)
+      verdicts = Map.new(sc.results, &{&1.id, &1.verdict})
+      assert verdicts["4.1.3"] == :fail
+    end
+
+    test "a clean tree passes the delegated language bans", %{tmp_dir: tmp} do
+      mk!(tmp, "README.adoc")
+      {:ok, catalogue} = RsrCriteria.load(@ssot)
+      {:ok, sc} = RsrConformance.score(catalogue, tmp)
+      verdicts = Map.new(sc.results, &{&1.id, &1.verdict})
+      # no banned files -> the scanner emits nothing -> bans pass
+      assert verdicts["5.1.1"] == :pass
+      assert verdicts["5.1.2"] == :pass
+      assert verdicts["5.1.5"] == :pass
+    end
+  end
+
   describe "self-application smoke" do
     test "scoring the hypatia repo itself returns a scorecard without raising" do
       {:ok, catalogue} = RsrCriteria.load(@ssot)