fix: report lost baseline coverage as worse [skip ci]

Author: richiemcilroy

scripts/verify/diff.py

@@ -151,6 +151,10 @@ def coverage_gained(b, m):
     return sorted(b["unobserved"] - m["unobserved"]) if m["ran"] else []
 
 
+def coverage_lost(b, m):
+    return sorted(m["unobserved"] - b["unobserved"]) if b["ran"] and m["ran"] else []
+
+
 def distribution(summary, cls, metric):
     values = []
     for sample in summary["samples"]:
@@ -177,12 +181,14 @@ def compare_case(b, m, rel, abs_):
         return "N/A", ["unsupported on baseline: " + "; ".join(b.get("why", [])[:1])]
     if not b["ran"] and not m["ran"]:
         return "EQUAL", []
+    lost = coverage_lost(b, m)
+    worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost)
     bc, mc = b["classes"], m["classes"]
     stochastic = "missing-after-hold"
     distribution_classes = (set(bc) | set(mc)) & {stochastic}
     if b["case"].startswith("recording/") or m["case"].startswith("recording/"):
         distribution_classes.add(stochastic)
-    for cls in sorted(distribution_classes):
+    for cls in sorted(distribution_classes - set(lost)):
         metrics = {"count"} | set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {}))
         for metric in sorted(metrics):
             bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)
@@ -252,8 +258,8 @@ def main():
     out = {}
     print(f"# {args.title}\n")
     print(f"tolerance: rel {args.rel_tol}, abs {args.abs_tol}\n")
-    print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")
-    print("|---|---|---|---|---|---|---|---|---|---|---|---|---|")
+    print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | coverage lost | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")
+    print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")
     detail = []
     for lane in lanes:
         b, m = brec.get(lane, {}), mrec.get(lane, {})
@@ -278,6 +284,9 @@ def main():
             gained = coverage_gained(bs, ms)
             if gained and verdict != "COVERAGE-GAINED":
                 res["COVERAGE-GAINED"].append((key, [f"{cls}: baseline unobserved -> main observed" for cls in gained]))
+            lost = coverage_lost(bs, ms)
+            if lost:
+                res["COVERAGE-LOST"].append((key, [f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost]))
         bj, mj = bjunit.get(lane, {}), mjunit.get(lane, {})
         jworse, jbetter = [], []
         bad = {"fail", "error"}
@@ -298,12 +307,12 @@ def main():
                 jworse.append((test, ["fail -> error (crash/timeout)"]))
         print(
             f"| {lane} | {len(b)}/{len(m)} | {len(res['WORSE'])} | {len(res['BETTER'])} | {len(res['EQUAL'])}"
-            f" | {len(res['COVERAGE-GAINED'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"
+            f" | {len(res['COVERAGE-GAINED'])} | {len(res['COVERAGE-LOST'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"
         )
         out[lane] = {k: v for k, v in res.items()}
         out[lane]["JUNIT-WORSE"] = jworse
         out[lane]["JUNIT-BETTER"] = jbetter
-        for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):
+        for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "COVERAGE-LOST", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):
             items = out[lane].get(label) or []
             if not items:
                 continue

scripts/verify/test_diff.py

@@ -1,4 +1,8 @@
 import importlib.util
+import json
+import subprocess
+import sys
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -24,6 +28,60 @@ def compare(base, main):
 
 
 class ComparisonTests(unittest.TestCase):
+    def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self):
+        main = record(unobserved={"sync-screen-mic": "track absent"})
+        for failed in [False, True]:
+            with self.subTest(failed=failed):
+                base = record()
+                if failed:
+                    base["failures"] = {"sync-screen-mic": []}
+                verdict, reasons = compare([base], [main])
+                self.assertEqual(verdict, "WORSE")
+                self.assertEqual(reasons, ["sync-screen-mic: baseline observed -> main unobserved (lost coverage)"])
+
+    def test_coverage_gain_does_not_hide_simultaneous_loss(self):
+        base = record(unobserved={"sync-screen-mic": "track absent"})
+        main = record(unobserved={"sync-screen-system": "track absent"})
+        bs, ms = diff.summarize([base]), diff.summarize([main])
+        self.assertEqual(diff.compare_case(bs, ms, 0, 0)[0], "WORSE")
+        self.assertEqual(diff.coverage_gained(bs, ms), ["sync-screen-mic"])
+        self.assertEqual(diff.coverage_lost(bs, ms), ["sync-screen-system"])
+
+    def test_partial_baseline_observation_still_detects_total_loss(self):
+        unseen = record(unobserved={"sync-screen-mic": "track absent"})
+        self.assertEqual(compare([unseen, record()], [unseen, unseen])[0], "WORSE")
+
+    def test_partial_main_observation_is_not_total_loss(self):
+        unseen = record(unobserved={"sync-screen-mic": "track absent"})
+        self.assertEqual(diff.coverage_lost(diff.summarize([record()]), diff.summarize([unseen, record()])), [])
+
+    def test_unobserved_on_both_sides_is_not_lost_coverage(self):
+        unseen = record(unobserved={"sync-screen-mic": "track absent"})
+        self.assertEqual(compare([unseen], [unseen]), ("EQUAL", []))
+
+    def test_lost_distribution_coverage_is_worse_instead_of_insufficient(self):
+        unseen = record(unobserved={"missing-after-hold": "track absent"})
+        self.assertEqual(compare([record(1)], [unseen])[0], "WORSE")
+
+    def test_cli_reports_coverage_lost_and_gained_separately(self):
+        with tempfile.TemporaryDirectory() as root:
+            root = Path(root)
+            base = record(unobserved={"sync-screen-mic": "track absent"})
+            main = record(unobserved={"sync-screen-system": "track absent"})
+            for side, rec in [("base", base), ("main", main)]:
+                path = root / side / "verify-report-macos-fast"
+                path.mkdir(parents=True)
+                rec.update({"suite": "export", "case": "a"})
+                (path / "records.jsonl").write_text(json.dumps(rec) + "\n")
+            result = subprocess.run(
+                [sys.executable, "-B", str(Path(__file__).with_name("diff.py")), str(root / "base"), str(root / "main"), "--json", str(root / "diff.json")],
+                check=True, capture_output=True, text=True,
+            )
+            self.assertIn("| coverage gained | coverage lost |", result.stdout)
+            output = json.loads((root / "diff.json").read_text())["macos"]
+            for category in ["WORSE", "COVERAGE-LOST", "COVERAGE-GAINED"]:
+                self.assertEqual(output[category][0][0], "export::a")
+
     def test_absent_track_is_coverage_gained_even_when_main_fails(self):
         base = record(unobserved={"sync-screen-mic": "track absent"})
         main = record()