Validate latest PR40 assembly discovery batch

.gitattributes

@@ -0,0 +1,4 @@
+# Discovery records are generated, diffable source data. Keep manifests visible
+# by default so reviewers can inspect counts, cursors and evidence provenance.
+testdata/discovery/ledger/records/*.jsonl linguist-generated
+testdata/discovery/assembly-ledger/records/*.jsonl linguist-generated

.github/scripts/priority-completion.test.cjs

@@ -0,0 +1,45 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { test } = require('node:test');
+
+const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8');
+
+function jobSection(name) {
+  const start = workflow.indexOf(`\n  ${name}:\n`);
+  assert.notEqual(start, -1, `missing job ${name}`);
+  const rest = workflow.slice(start + 1);
+  const next = rest.slice(1).search(/^  [a-zA-Z0-9_-]+:\s*$/m);
+  return next < 0 ? rest : rest.slice(0, next + 1);
+}
+
+test('all other entry jobs require successful priority completion', () => {
+  for (const name of [
+    'fmt', 'build', 'benchmark', 'reported_library_list', 'discovered_library_corpus',
+    'stdlib-corpus', 'cross-runtime', 'test', 'arm-scan', 'race', 'coverage',
+  ]) {
+    const section = jobSection(name);
+    assert.match(section, /^    needs: discovered_library_priority$/m, name);
+    // Native needs success gating must not be bypassed by always()/!cancelled().
+    assert.doesNotMatch(section, /^    if:/m, name);
+  }
+  assert.doesNotMatch(workflow, /^  priority_started:/m);
+});
+
+test('all failed shards run and no priority failure can become success', () => {
+  const priority = jobSection('discovered_library_priority');
+  assert.match(priority, /^      fail-fast: false$/m);
+  assert.doesNotMatch(priority, /continue-on-error:/);
+  assert.match(priority, /^    needs: ci_policy$/m);
+  assert.doesNotMatch(priority, /^    if:/m);
+});
+
+test('cheap CI policy checks pass before expensive corpus or test jobs start', () => {
+  const policy = jobSection('ci_policy');
+  assert.match(policy, /bash scripts\/install-ci-llvm22-ubuntu\.sh/);
+  assert.match(policy, /node --test \.github\/scripts\/priority-\*\.test\.cjs/);
+  assert.match(policy, /go test \. -run '\^TestCI' -count=1/);
+  assert.doesNotMatch(policy, /^    (needs|if):/m);
+  assert.ok(workflow.indexOf('\n  ci_policy:\n') <
+    workflow.indexOf('\n  discovered_library_priority:\n'));
+});

.github/scripts/priority-discovery-cache.test.cjs

@@ -0,0 +1,103 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const test = require('node:test');
+
+const source = fs.readFileSync(
+  path.resolve(__dirname, '../../scripts/check-discovered-library-corpus.sh'),
+  'utf8',
+);
+
+function runCorpusFixture(t, failedShard = '', priorityShards = '') {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan9asm-cache-test-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const scripts = path.join(root, 'scripts');
+  const bin = path.join(root, 'bin');
+  fs.mkdirSync(scripts);
+  fs.mkdirSync(bin);
+  const calls = path.join(root, 'calls');
+  const verified = path.join(root, 'verified');
+  const caches = path.join(root, 'caches');
+  fs.mkdirSync(caches);
+  const runner = path.join(scripts, 'check-discovered-library-corpus.sh');
+  fs.writeFileSync(runner, source, { mode: 0o755 });
+  fs.writeFileSync(path.join(bin, 'xargs'), `#!/usr/bin/env bash
+set -euo pipefail
+status=0
+while IFS= read -r shard; do
+  test -d "$PLAN9ASM_DISCOVERY_BUILD_CACHE"
+  printf '%s\t%s\n' "$shard" "$PLAN9ASM_DISCOVERY_BUILD_CACHE" >> "$FIXTURE_CALLS"
+  if [[ "$shard" == "$FIXTURE_FAILED_SHARD" ]]; then
+    status=123
+  fi
+done
+exit "$status"
+`, { mode: 0o755 });
+  fs.writeFileSync(path.join(scripts, 'verify-discovered-library-corpus.sh'), `#!/usr/bin/env bash
+set -euo pipefail
+printf 'verified\n' >> "$FIXTURE_VERIFIED"
+`, { mode: 0o755 });
+  const result = spawnSync('bash', [runner, 'all', '5'], {
+    encoding: 'utf8',
+    timeout: 10000,
+    env: {
+      ...process.env,
+      PATH: `${bin}${path.delimiter}${process.env.PATH}`,
+      TMPDIR: caches,
+      PLAN9ASM_DISCOVERY_PARALLELISM: '2',
+      PLAN9ASM_DISCOVERY_PRIORITY_SHARDS: priorityShards,
+      FIXTURE_CALLS: calls,
+      FIXTURE_VERIFIED: verified,
+      FIXTURE_FAILED_SHARD: failedShard,
+    },
+  });
+  assert.ifError(result.error);
+  const records = fs.existsSync(calls)
+    ? fs.readFileSync(calls, 'utf8').trim().split('\n')
+      .map((line) => line.split('\t'))
+    : [];
+  return { result, records, caches, verified };
+}
+
+test('discovery shares a cache only within each bounded parallel batch', (t) => {
+  const { result, records, caches, verified } = runCorpusFixture(t);
+  assert.equal(result.status, 0, result.stderr);
+  assert.deepEqual(records.map(([shard]) => shard), ['0', '1', '2', '3', '4']);
+  assert.equal(records[0][1], records[1][1]);
+  assert.equal(records[2][1], records[3][1]);
+  assert.equal(new Set(records.map(([, cache]) => cache)).size, 3);
+  assert.deepEqual(fs.readdirSync(caches), []);
+  assert.equal(fs.readFileSync(verified, 'utf8'), 'verified\n');
+});
+
+test('failed discovery batches keep failures, clean caches and run the final gate', (t) => {
+  const { result, records, caches, verified } = runCorpusFixture(t, '2');
+  assert.notEqual(result.status, 0);
+  assert.deepEqual(records.map(([shard]) => shard), ['0', '1', '2', '3', '4']);
+  assert.deepEqual(fs.readdirSync(caches), []);
+  assert.equal(fs.readFileSync(verified, 'utf8'), 'verified\n');
+});
+
+test('priority shards run first without omitting or repeating other shards', (t) => {
+  const { result, records, caches, verified } = runCorpusFixture(t, '', '4,2');
+  assert.equal(result.status, 0, result.stderr);
+  assert.deepEqual(records.map(([shard]) => shard), ['4', '2', '0', '1', '3']);
+  assert.equal(records[0][1], records[1][1]);
+  assert.equal(records[2][1], records[3][1]);
+  assert.equal(new Set(records.map(([, cache]) => cache)).size, 3);
+  assert.deepEqual(fs.readdirSync(caches), []);
+  assert.equal(fs.readFileSync(verified, 'utf8'), 'verified\n');
+});
+
+test('invalid priority lists fail before starting corpus work', (t) => {
+  for (const priority of ['5', '2,2', '-1', '1,,2', ',1', '1,', '01', '1 x',
+    '18446744073709551616']) {
+    const { result, records, caches, verified } = runCorpusFixture(t, '', priority);
+    assert.notEqual(result.status, 0, `accepted invalid priority ${priority}`);
+    assert.deepEqual(records, []);
+    assert.deepEqual(fs.readdirSync(caches), []);
+    assert.equal(fs.existsSync(verified), false);
+  }
+});

.github/scripts/priority-jobs.test.cjs

@@ -0,0 +1,89 @@
+const assert = require('node:assert/strict');
+const { createHash } = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { test } = require('node:test');
+
+const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8');
+
+function jobSection(name) {
+  const start = workflow.indexOf(`\n  ${name}:\n`);
+  assert.notEqual(start, -1, `missing job ${name}`);
+  const rest = workflow.slice(start + 1);
+  const next = rest.slice(1).search(/^  [a-zA-Z0-9_-]+:\s*$/m);
+  return next < 0 ? rest : rest.slice(0, next + 1);
+}
+
+function shards(section) {
+  const match = section.match(/^        shard:\n((?:          - \d+\n)+)/m);
+  assert.ok(match, 'missing shard matrix');
+  return Array.from(match[1].matchAll(/^          - (\d+)$/gm),
+    ([, value]) => Number(value));
+}
+
+test('priority and remaining matrices partition all 64 shards exactly once', () => {
+  const priority = shards(jobSection('discovered_library_priority'));
+  const remaining = shards(jobSection('discovered_library_corpus'));
+  // Retain instruction/ABI regressions and the cache cleanup failure from
+  // fork run 36509240233.
+  assert.deepEqual(priority, [5, 29, 36, 38, 39, 42]);
+  assert.deepEqual([...priority, ...remaining].sort((a, b) => a - b),
+    Array.from({ length: 64 }, (_, index) => index));
+  for (const name of ['discovered_library_priority', 'discovered_library_corpus']) {
+    const section = jobSection(name);
+    assert.match(section, /discovered-corpus \(shard \$\{\{ matrix\.shard \}\}\/64\)/);
+    assert.match(section, /check-discovered-library-corpus\.sh "\$\{\{ matrix\.shard \}\}" 64/);
+  }
+});
+
+test('failed exact versions rehash into priority shards', () => {
+  const priority = shards(jobSection('discovered_library_priority'));
+  for (const exactVersion of [
+    'github.com/Qitmeer/go-ethereum@v1.10.12',
+    'github.com/RookieCoderrr/neo3fura-ctrverification@v0.0.0-20221201045318-9878de6dbeed',
+    'github.com/SysVerification/gokv@v0.0.0-20250508184610-d007325b6ee8',
+    'github.com/reddit/milvus/pkg/v3@v3.0.0-20260702082229-182134e29ebf',
+    'github.com/apernet/hysteria/app/v2@v2.12.3',
+    'github.com/HyNetwork/hysteria/app@v1.3.5',
+    'github.com/intel/ixl-go@v0.0.0-20260223204051-3e809238f558',
+  ]) {
+    const hash = createHash('sha256').update(exactVersion).digest();
+    const shard = Number(hash.readBigUInt64BE(0) % 64n);
+    assert.ok(priority.includes(shard), `${exactVersion} belongs to shard ${shard}`);
+  }
+});
+
+test('priority and remaining shards use identical test and artifact steps', () => {
+  function steps(name) {
+    const section = jobSection(name);
+    return section.slice(section.indexOf('    steps:\n')).trim();
+  }
+  assert.equal(steps('discovered_library_priority'), steps('discovered_library_corpus'));
+  assert.match(jobSection('discovered_library_corpus_verify'),
+    /needs: \[discovered_library_priority, discovered_library_corpus\]/);
+});
+
+test('discovery matrices have no explicit concurrency cap', () => {
+  for (const name of ['discovered_library_priority', 'discovered_library_corpus']) {
+    assert.doesNotMatch(jobSection(name), /^      max-parallel:/m, name);
+  }
+});
+
+test('local discovery helpers use the CI shard count by default', () => {
+  for (const name of [
+    'check-discovered-library-corpus.sh',
+    'discovery-status.sh',
+    'update-assembly-ledger.sh',
+  ]) {
+    const script = fs.readFileSync(path.join(__dirname, '../../scripts', name), 'utf8');
+    assert.match(script, /shard_count=\$\{2:-64\}/, name);
+  }
+});
+
+test('discovery matrices verify checksums against direct signed databases', () => {
+  for (const name of ['discovered_library_priority', 'discovered_library_corpus']) {
+    const section = jobSection(name);
+    assert.match(section, /^          GOSUMDB:.*sum\.golang\.google\.cn.*sum\.golang\.org https:\/\/sum\.golang\.org/m, name);
+    assert.doesNotMatch(section, /^          GOSUMDB: ['"]?off/m, name);
+  }
+});