test(coverage): close the macOS live bucket from its scenario sources too

The commit before this one recorded macOS as gated one way only, on the finding that its provider
scenarios execute `open`, `appstate`, `snapshot`, `screenshot`, `wait` and `click` without claiming
them live. That reading was wrong: those commands are claimed live, attributed to the replay scenario,
so the mismatch is scenario attribution and not a missing claim. Compared at platform level, the
commands the live scenarios execute and the commands the manifest calls live are the same 18.

`assertLiveCoverageMatchesEvidence` states that tie for the two platforms whose evidence enumerates its
own commands — macOS through `parseReplayScriptDetailed` and the provider call shapes, web through the
argv heads its smoke scenario passes — and Linux and Android already read the same kind of enumeration
directly. Appending `callCommand('hover', …)` to a macOS live scenario fails the gate with the platform
named, and the rollup assertion's doc comment now points here for membership instead of implying it
covers classification.

Author: thymikee

test/integration/smoke-macos-coverage.test.ts

@@ -1,11 +1,16 @@
 import assert from 'node:assert/strict';
+import { parseReplayScriptDetailed } from '@agent-device/ad-script';
 import fs from 'node:fs';
 import path from 'node:path';
 import test from 'node:test';
 
 import { mkdtempForTest } from '../../src/__tests__/test-utils/tmp-dir.ts';
 import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';
-import { assertCoverageClassificationSummaryWiredToManifest } from './support/coverage-classification.ts';
+import {
+  assertCoverageClassificationSummaryWiredToManifest,
+  assertLiveCoverageMatchesEvidence,
+} from './support/coverage-classification.ts';
+import type { MacOsLiveScenario } from './macos-e2e/live-scenarios.ts';
 import {
   MACOS_COVERAGE_GAP_ISSUE,
   MACOS_LIVE_SCENARIOS,
@@ -48,6 +53,17 @@ test('macOS coverage report counts every manifest classification', () => {
   );
 });
 
+// The scenario sources say which commands macOS actually runs, independent of what the manifest
+// labels them. Provider scenarios run commands whose live claim is attributed to the replay
+// scenario, so this closes the platform's live bucket rather than each scenario's partition of it.
+test('macOS live claims are exactly the commands its live scenarios execute', () => {
+  assertLiveCoverageMatchesEvidence(
+    'macOS',
+    MACOS_PLATFORM_COVERAGE,
+    commandsExecutedByMacOsLiveScenarios(),
+  );
+});
+
 test('macOS live claims reference commands in existing executable scenarios', () => {
   const scenariosById = new Map(MACOS_LIVE_SCENARIOS.map((scenario) => [scenario.id, scenario]));
   const claimedCommands: string[] = [];
@@ -119,6 +135,40 @@ test('macOS coverage report persists the manifest rollup and live command list',
   );
 });
 
+function commandsExecutedByMacOsLiveScenarios(): string[] {
+  const catalogCommands: ReadonlySet<string> = new Set(publicCommands);
+  const executed = new Set<string>();
+  for (const scenario of MACOS_LIVE_SCENARIOS) {
+    for (const command of commandsExecutedInScenarioSource(
+      readEvidence(scenario.owner),
+      scenario.syntax,
+    )) {
+      if (catalogCommands.has(command)) {
+        executed.add(command);
+      }
+    }
+  }
+  return [...executed];
+}
+
+function commandsExecutedInScenarioSource(
+  source: string,
+  syntax: MacOsLiveScenario['syntax'],
+): string[] {
+  const group = (pattern: RegExp): string[] =>
+    [...source.matchAll(pattern)]
+      .map((match) => match[1])
+      .filter((command): command is string => command !== undefined);
+  switch (syntax) {
+    case 'replay':
+      return parseReplayScriptDetailed(source).actions.map((action) => action.command);
+    case 'provider-command':
+      return group(/command:\s*'([a-z][a-z-]*)'/g);
+    case 'provider-call-command':
+      return group(/callCommand\(\s*'([a-z][a-z-]*)'/g);
+  }
+}
+
 function readEvidence(owner: { path: string; test: string }): string {
   const evidencePath = path.resolve(owner.path);
   assert.equal(fs.existsSync(evidencePath), true, `${owner.path} does not exist`);

test/integration/smoke-web-platform-coverage.test.ts

@@ -5,7 +5,10 @@ import test from 'node:test';
 
 import { mkdtempForTest } from '../../src/__tests__/test-utils/tmp-dir.ts';
 import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';
-import { assertCoverageClassificationSummaryWiredToManifest } from './support/coverage-classification.ts';
+import {
+  assertCoverageClassificationSummaryWiredToManifest,
+  assertLiveCoverageMatchesEvidence,
+} from './support/coverage-classification.ts';
 import {
   WEB_COVERAGE_GAP_ISSUE,
   WEB_PLATFORM_COVERAGE,
@@ -54,7 +57,11 @@ test('web coverage report counts every manifest classification', () => {
 test('web live claims are exactly the commands the smoke scenario invokes', () => {
   const smokeSource = fs.readFileSync(path.resolve(WEB_SMOKE_EVIDENCE.path), 'utf8');
   assert.ok(smokeSource.includes(WEB_SMOKE_TEST_NAME));
-  assert.deepEqual(invokedWebSmokeCommands(smokeSource), liveCommandsForWebSmoke().sort());
+  assertLiveCoverageMatchesEvidence(
+    'web',
+    WEB_PLATFORM_COVERAGE,
+    invokedWebSmokeCommands(smokeSource),
+  );
 });
 
 test('web contract claims name existing executable evidence', () => {

test/integration/support/coverage-classification.ts

@@ -23,12 +23,10 @@ const BUCKET_BY_LEVEL: Readonly<Record<CoverageClassificationLevel, CoverageBuck
  * mis-bucketed rollup, and a denominator that stopped matching the catalog.
  *
  * What it cannot catch is a row classified under the wrong `level`: the recount and the published
- * summary read the same rows, so a row moved between buckets moves both and stays consistent. Row
- * membership is gated where an independent enumeration of it exists — the Linux lane compares its
- * live set against the parsed replay script, the Android lane against each scenario's own command
- * declaration, the web lane against the commands its smoke scenario invokes. A platform whose live
- * claims have no enumeration behind them, macOS, is gated one way only: each claim must be executed
- * by the scenario that owns it, and adding a live claim therefore needs evidence.
+ * summary read the same rows, so a row moved between buckets moves both and stays consistent. That
+ * membership is gated by {@link assertLiveCoverageMatchesEvidence}, or by the equivalent enumeration
+ * a platform already reads — Linux parses its replay script, Android reads each scenario's own
+ * command declaration.
  */
 export function assertCoverageClassificationSummaryWiredToManifest(
   platform: string,
@@ -59,6 +57,29 @@ export function assertCoverageClassificationSummaryWiredToManifest(
   );
 }
 
+/**
+ * Ties a platform's live bucket to an enumeration of the commands its own evidence executes, read
+ * from the scenario sources rather than from the manifest's `level` fields. Both directions matter:
+ * an executed command that is not claimed live means the platform is understating what it proves,
+ * and a live claim no scenario executes any more means the platform is overstating it. A platform
+ * with no independent enumeration behind a bucket can only assert the rollup above.
+ */
+export function assertLiveCoverageMatchesEvidence(
+  platform: string,
+  manifest: Readonly<Record<string, { level: CoverageClassificationLevel }>>,
+  evidenceCommands: Iterable<string>,
+): void {
+  const live = Object.entries(manifest)
+    .filter(([, entry]) => entry.level === 'live')
+    .map(([command]) => command)
+    .sort();
+  assert.deepEqual(
+    [...new Set(evidenceCommands)].sort(),
+    live,
+    `${platform} live claims are not the commands its own evidence executes`,
+  );
+}
+
 export function buildCoverageClassificationSummary(
   entries: readonly { level: CoverageClassificationLevel }[],
 ): CoverageClassificationSummary {