diff --git a/.github/scripts/timed-run.mjs b/.github/scripts/timed-run.mjs index 9487bf901..10fe9c43d 100644 --- a/.github/scripts/timed-run.mjs +++ b/.github/scripts/timed-run.mjs @@ -96,6 +96,9 @@ export function parseInputs(env) { if (!RUNNERS.includes(runner)) bad('runner', runner, RUNNERS.join(' or ')); if (runner === 'hosted' && command !== 'counted') bad('runner', runner, `${command} is a timing run and runs on souls-timed only`); const args = checkArgs(command, env.INPUT_ARGS), inp = { ref, command, repeats: Number(repeats), args }; + const cores = String(env.INPUT_CORES ?? '').trim(); // #2507: a core plan, for the split commands only + if (cores && !SPLIT.includes(command)) bad('cores', cores, `a core plan is for ${SPLIT.join(', ')} only`); + if (cores) inp.cores = parseCorePlan(cores) ?? bad('cores', cores, 'world=N or world=N,launcher=M (physical cores, 1 to 64); the load drivers get the rest'); if (command === 'clip' && !args.includes('--kit')) bad('args', env.INPUT_ARGS, 'clip needs --kit (a folder of tools/proof/clips)'); if (command !== 'counted') return { ...inp, runner }; // counted: a base ref, a scenario, and the one cell this job runs (base-k or cand-k, k up to repeats) @@ -151,16 +154,32 @@ export function frameTiming(pts) { return { frames: t.length, spanSec: r3(span), meanFps: span > 0 ? +((t.length - 1) / span).toFixed(2) : null, maxGapSec: gaps.length ? r3(Math.max(...gaps)) : null, gapsOver100ms: gaps.filter((g) => g > 0.1).length }; } +/** #2507: `world=N[,launcher=M]` → { world, launcher } physical cores (launcher 0: not split off), or null. */ +export function parseCorePlan(s) { + const m = /^world=([1-9]\d?)(?:,launcher=([1-9]\d?))?$/.exec(String(s)), w = Number(m?.[1]), l = Number(m?.[2] ?? 0); + return m && w <= 64 && l <= 64 ? { world: w, launcher: l } : null; +} /** #1970 A2: the runner's own logical CPUs grouped into physical cores (`siblingsOf(cpu)`: that CPU's SMT siblings); the * first two cores for the world under test, the rest for the load generator: 2+2 on four cores, 2+1 on three (the timed - * runners give each job three). Any other count is refused. */ -export function splitCores(own, siblingsOf) { + * runners give each job three). Any other count is refused. #2507: a core plan gives the world its first N cores, the + * launcher the next M and the load drivers the rest (at least one); a plan that does not fit is refused. */ +export function splitCores(own, siblingsOf, plan = null) { const cores = new Map(); for (const c of own) { const k = siblingsOf(c).filter((x) => own.includes(x)).sort((a, b) => a - b).join(',') || String(c); cores.set(k, k.split(',').map(Number)); } const byFirst = [...cores.values()].sort((a, b) => a[0] - b[0]); - const n = byFirst.length; + const n = byFirst.length, take = (a, b) => byFirst.slice(a, b).flat().join(','); + if (plan) { + const { world: w, launcher: l } = plan, name = `world=${w}${l ? `,launcher=${l}` : ''}`; + if (w + l + 1 > n) throw new Error(`the core plan ${name} needs ${w + l + 1} physical cores (at least one for the load drivers); this runner has ${n} of its own`); + return { physicalCores: n, plan: name, layout: `${w}+${l ? `${l}+` : ''}${n - w - l}`, world: take(0, w), ...(l ? { launcher: take(w, w + l) } : {}), load: take(w + l) }; + } if (n !== 3 && n !== 4) throw new Error(`the runner has ${n} physical core(s) of its own; the world/load split needs 3 or 4`); - return { physicalCores: n, layout: `2+${n - 2}`, world: byFirst.slice(0, 2).flat().join(','), load: byFirst.slice(2).flat().join(',') }; + return { physicalCores: n, layout: `2+${n - 2}`, world: take(0, 2), load: take(2) }; +} +/** #2507: each group's mean busy fraction of its own CPUs between two /proc/stat samples (groups: name → "1,2,3"). */ +export function groupBusy(a, b, groups) { + const busy = (c) => { const d = b[c] && a[c] ? b[c][1] - a[c][1] : 0; return d > 0 ? (b[c][0] - a[c][0]) / d : 0; }; + return Object.fromEntries(Object.entries(groups).map(([k, v]) => { const cs = parseCpuList(v); return [k, +(cs.reduce((s, c) => s + busy(c), 0) / cs.length).toFixed(3)]; })); } /** [[pid, ppid], …] → every descendant pid of root. */ export function descendants(table, root) { @@ -170,6 +189,8 @@ export function descendants(table, root) { } /** A world server process (a direct boot, or a child of the launcher), by its NUL-separated command line. */ export const isWorld = (cmdline) => cmdline.split('\0').some((a) => a.endsWith('apps/server/src/main.ts')); +/** #2507: the world launcher process (it carries every viewer byte and every act), by its NUL-separated command line. */ +export const isLauncher = (cmdline) => cmdline.split('\0').some((a) => a.endsWith('tools/world-launcher/launcher.mjs')); /** "0-3,8,10-11" → [0,1,2,3,8,10,11]. */ export function parseCpuList(s) { @@ -298,18 +319,28 @@ const siblingsOf = (c) => { try { return parseCpuList(readFileSync(`/sys/devices const procTable = () => readdirSync('/proc').filter((d) => /^\d+$/.test(d)).flatMap((d) => { try { return [[Number(d), Number(readFileSync(`/proc/${d}/stat`, 'utf8').split(') ').at(-1).split(' ')[1])]]; } catch { return []; } }); const allowedOf = (pid) => { try { return /^Cpus_allowed_list:\s*(\S+)/m.exec(readFileSync(`/proc/${pid}/status`, 'utf8'))?.[1] ?? null; } catch { return null; } }; /** #1970 A2: each world process under root, with all its threads, onto the world's cores as soon as it appears (threads - * made later inherit the mask). Returns what was pinned: a count and the allowed lists read back, no pids. */ -function pinWorlds(root, cpus) { - const pinned = new Map(), tick = () => { + * made later inherit the mask). #2507: with a launcher in the plan, each launcher process too, onto its own cores (a + * world it starts is moved on to the world's). Every 5 s each group's busy share is sampled. Returns what was pinned (a + * count and the allowed lists read back, no pids) and each group's mean and peak busy share. */ +function pinWorlds(root, split) { + const stat = () => parseProcStat(readFileSync('/proc/stat', 'utf8')), groups = { world: split.world, ...(split.launcher ? { launcher: split.launcher } : {}), load: split.load }; + const first = stat(), peak = {}, pinned = { world: new Map(), launcher: new Map() }; + let last = first, n = 0; + const tick = () => { for (const pid of descendants(procTable(), root)) { - if (pinned.has(pid)) continue; + if (pinned.world.has(pid) || pinned.launcher.has(pid)) continue; let cmd; try { cmd = readFileSync(`/proc/${pid}/cmdline`, 'utf8'); } catch { continue; } - if (!isWorld(cmd)) continue; - try { execFileSync('taskset', ['-a', '-p', '-c', cpus, String(pid)], { stdio: 'ignore' }); pinned.set(pid, allowedOf(pid)); } catch { /* exited */ } + const g = isWorld(cmd) ? 'world' : split.launcher && isLauncher(cmd) ? 'launcher' : null; if (!g) continue; + try { execFileSync('taskset', ['-a', '-p', '-c', split[g], String(pid)], { stdio: 'ignore' }); pinned[g].set(pid, allowedOf(pid)); } catch { /* exited */ } } + if (++n % 20) return; + const now = stat(); for (const [k, v] of Object.entries(groupBusy(last, now, groups))) peak[k] = Math.max(peak[k] ?? 0, v); last = now; }; const iv = setInterval(tick, 250); - return { stop: () => { clearInterval(iv); return { worldProcesses: pinned.size, worldAllowed: [...new Set(pinned.values())] }; } }; + return { stop: () => { clearInterval(iv); const mean = groupBusy(first, stat(), groups); + return { worldProcesses: pinned.world.size, worldAllowed: [...new Set(pinned.world.values())], + ...(split.launcher ? { launcherProcesses: pinned.launcher.size, launcherAllowed: [...new Set(pinned.launcher.values())] } : {}), + busy: Object.fromEntries(Object.keys(groups).map((k) => [k, { mean: mean[k], peak5s: peak[k] ?? null }])) }; } }; } const la = () => loadavg().slice(0, 2).map((x) => +x.toFixed(2)); const sh = (cmd, args) => { try { execFileSync(cmd, args, { stdio: 'ignore' }); return true; } catch { return false; } }; @@ -419,10 +450,11 @@ async function main(cmd) { console.log('no entries written outside the per-run directory'); return; } const inp = parseInputs(process.env), rf = arg('record'); - if (cmd === 'inputs') { output('target', target(inp)); console.log(`inputs ok: ${inp.ref}, ${inp.command} x${inp.repeats}${inp.args.length ? ` ${inp.args.join(' ')}` : ''}${inp.cell ? `, cell ${inp.cell} of base ${inp.base}` : ''} on ${inp.runner}`); return; } + if (cmd === 'inputs') { output('target', target(inp)); console.log(`inputs ok: ${inp.ref}, ${inp.command} x${inp.repeats}${inp.args.length ? ` ${inp.args.join(' ')}` : ''}${inp.cores ? `, cores world=${inp.cores.world},launcher=${inp.cores.launcher}` : ''}${inp.cell ? `, cell ${inp.cell} of base ${inp.base}` : ''} on ${inp.runner}`); return; } if (cmd === 'load') { // A3: own cores and whole machine; wait a bounded time for own cores to settle, else refuse // counted scores pass/fail per item: its starting load is recorded, never gated const own = ownCpus(), stat = () => parseProcStat(readFileSync('/proc/stat', 'utf8')), gated = inp.command !== 'counted'; + if (inp.cores) console.log(`cores: plan ${JSON.stringify(splitCores(own, siblingsOf, inp.cores))}`); // #2507: throws if it does not fit let f; for (let t = 0; ; t += 15) { const a = stat(); await sleep(SAMPLE_S * 1000); f = loadFacts(a, stat(), own); @@ -443,18 +475,18 @@ async function main(cmd) { if (inp.command === 'clip') return runClip(inp, rec, out, env); // #1970: the world on two physical cores, the load generator (the driver and all it starts but the world) on the rest // (one or two). HOME goes inside the per-run directory: the capacity bench moves finished world data under ~/.Trash. - const split = SPLIT.includes(inp.command) ? splitCores(ownCpus(), siblingsOf) : null; + const split = SPLIT.includes(inp.command) ? splitCores(ownCpus(), siblingsOf, inp.cores) : null; if (split) { execFileSync('taskset', ['-V'], { stdio: 'ignore' }); env.HOME = join(env.PER_RUN, 'home'); mkdirSync(env.HOME, { recursive: true }); - console.log(`cores: split ${split.layout} of ${split.physicalCores} physical, world ${split.world}, load generator ${split.load}`); + console.log(`cores: split ${split.layout} of ${split.physicalCores} physical, world ${split.world}${split.launcher ? `, launcher ${split.launcher}` : ''}, load generator ${split.load}`); } const n = inp.command === 'walk' ? inp.repeats : 1, runs = []; for (let k = 1; k <= n; k++) { const argv = commandLine(inp, { out, port: await freePorts(PORT_SPAN[inp.command]) }), log = join(out, logName(inp.command, k)), t0 = Date.now(); console.log(`${inp.command} ${k}/${n}: ${argv.join(' ')}`); const [bin, args] = split ? ['taskset', ['-c', split.load, process.execPath, ...argv]] : [process.execPath, argv]; - const ws = createWriteStream(log), child = spawn(bin, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }), pins = split ? pinWorlds(child.pid, split.world) : null; + const ws = createWriteStream(log), child = spawn(bin, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }), pins = split ? pinWorlds(child.pid, split) : null; for (const s of [child.stdout, child.stderr]) s.on('data', (b) => { ws.write(b); process.stdout.write(b); }); for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig)); const code = await new Promise((r) => child.on('close', (c) => r(c ?? 1))); ws.end(); @@ -464,6 +496,7 @@ async function main(cmd) { const failed = runs.filter((r) => r.exitCode !== 0); if (failed.length) throw new Error(`${failed.length} of ${runs.length} run(s) exited non-zero`); if (runs.some((r) => r.cpuSplit && !r.cpuSplit.worldProcesses)) throw new Error('no world process was seen to pin to the world\'s cores'); + if (runs.some((r) => r.cpuSplit?.launcher && !r.cpuSplit.launcherProcesses)) throw new Error('no launcher process was seen to pin to the launcher\'s cores'); return; } if (cmd === 'stage') { diff --git a/.github/scripts/timed-run.test.mjs b/.github/scripts/timed-run.test.mjs index fe9a38e14..5d3afcf5f 100644 --- a/.github/scripts/timed-run.test.mjs +++ b/.github/scripts/timed-run.test.mjs @@ -6,7 +6,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createHash } from 'node:crypto'; -import { parseInputs, checkArgs, commandLine, parseCpuList, parseProcStat, loadFacts, stage, snapshot, newEntries, sortStrays, globTest, splitCores, descendants, isWorld, SPLIT, LOAD_GATE, COUNTED, target, countedSteps } from './timed-run.mjs'; +import { parseInputs, checkArgs, commandLine, parseCpuList, parseProcStat, loadFacts, stage, snapshot, newEntries, sortStrays, globTest, splitCores, descendants, isWorld, isLauncher, parseCorePlan, groupBusy, SPLIT, LOAD_GATE, COUNTED, target, countedSteps } from './timed-run.mjs'; const ENV = { INPUT_REF: 'main', INPUT_COMMAND: 'walk', INPUT_REPEATS: '1', INPUT_ARGS: '' }; @@ -178,6 +178,28 @@ test('core split: two physical cores for the world, the rest (two of four, one o assert.throws(() => splitCores([0, 1, 2, 3, 4, 5], (c) => [c]), /6 physical core/); }); +test('core plan (#2507): cores is accepted for the split commands, validated, recorded in the inputs, and refused at the start gate when it does not fit', () => { + const V = { ...ENV, INPUT_COMMAND: 'viewers' }; + assert.deepEqual(parseInputs({ ...V, INPUT_CORES: 'world=2,launcher=1' }).cores, { world: 2, launcher: 1 }); + assert.deepEqual(parseInputs({ ...V, INPUT_CORES: ' world=6 ' }).cores, { world: 6, launcher: 0 }); + assert.ok(!('cores' in parseInputs({ ...V, INPUT_CORES: '' })), 'no plan: the inputs are as before'); + for (const v of ['world=0', 'world=2,launcher=0', 'launcher=1', 'world=2,launcher=1,load=1', 'world=65', 'world=2 launcher=1', 'world=x', 'world=2;id']) + assert.throws(() => parseInputs({ ...V, INPUT_CORES: v }), /input cores=.*world=N or world=N,launcher=M/, v); + assert.throws(() => parseInputs({ ...ENV, INPUT_CORES: 'world=2' }), /input cores=.*for eval-load, .*viewers only/, 'walk takes no plan'); + assert.equal(parseCorePlan('world=2,launcher=1').launcher, 1); + const smt = (c) => [c % 8, (c % 8) + 8], three = [5, 6, 7, 13, 14, 15]; + assert.deepEqual(splitCores(three, smt, { world: 1, launcher: 1 }), { physicalCores: 3, plan: 'world=1,launcher=1', layout: '1+1+1', world: '5,13', launcher: '6,14', load: '7,15' }); + assert.deepEqual(splitCores([0, 1, 2, 3, 4, 5, 6, 7], (c) => [c], { world: 6, launcher: 1 }), { physicalCores: 8, plan: 'world=6,launcher=1', layout: '6+1+1', world: '0,1,2,3,4,5', launcher: '6', load: '7' }); + assert.deepEqual(splitCores([0, 1, 2, 3, 4, 5], (c) => [c], { world: 2, launcher: 0 }), { physicalCores: 6, plan: 'world=2', layout: '2+4', world: '0,1', load: '2,3,4,5' }, 'a plan lifts the 3-or-4 rule'); + assert.throws(() => splitCores(three, smt, { world: 2, launcher: 1 }), /the core plan world=2,launcher=1 needs 4 physical cores \(at least one for the load drivers\); this runner has 3 of its own/); + assert.throws(() => splitCores(three, smt, { world: 3, launcher: 0 }), /world=3 needs 4/); + assert.deepEqual(splitCores(three, smt, null), splitCores(three, smt), 'no plan: the split of today'); + assert.ok(isLauncher(['/usr/bin/node', '/w/src/tools/world-launcher/launcher.mjs', '--port', '1'].join('\0'))); + for (const c of ['node\0apps/server/src/main.ts', 'node\0tools/world-launcher/launcher.test.mjs', 'node\0tools/bench/capacity/viewers/viewers.mjs']) assert.ok(!isLauncher(c), c); + const a = parseProcStat('cpu0 10 0 0 90 0 0 0 0\ncpu1 0 0 0 100 0 0 0 0\ncpu2 50 0 0 50 0 0 0 0\n'), b = parseProcStat('cpu0 60 0 0 140 0 0 0 0\ncpu1 10 0 0 190 0 0 0 0\ncpu2 150 0 0 50 0 0 0 0\n'); + assert.deepEqual(groupBusy(a, b, { world: '0,1', load: '2' }), { world: 0.3, load: 1 }); +}); + test('world pinning: only world servers under the run\'s own process tree', () => { assert.deepEqual(descendants([[10, 1], [11, 10], [12, 11], [13, 1], [14, 12]], 10).sort(), [11, 12, 14]); assert.deepEqual(descendants([[10, 1]], 99), []); diff --git a/.github/workflows/timed-run.yml b/.github/workflows/timed-run.yml index 4de76a3b4..5fb18e9c5 100644 --- a/.github/workflows/timed-run.yml +++ b/.github/workflows/timed-run.yml @@ -46,6 +46,9 @@ on: args: description: 'Allowlisted: walk --world; eval-scripted --battery --seat scripted --items --warm-sec; eval-load --souls --duration-sec --card-sec; bench-client --arms --profile --raw-from --short; meetup --profile --active-min --short; slopes-profile none; load --world --boots --souls --profile --short; settled --world --souls --settle-min --short; hundred --small --eden --short --fixture-only --active-min --gap-min --settle-min --build-parallel; colocate --forms --settle-s --walk-s --busy --short; viewers --viewers --window-sec --short; counted --scenario --loop-sec; clip --kit --viewport' default: '' + cores: + description: 'eval-load, bench-client, meetup, slopes-profile, viewers only: a core plan, world=N or world=N,launcher=M (physical cores); the load drivers get the rest. Blank: 2 for the world, the rest for the load' + default: '' permissions: contents: read jobs: @@ -67,6 +70,7 @@ jobs: INPUT_BASE: ${{ inputs.base }} INPUT_RUNNER: ${{ inputs.runner }} INPUT_CELL: ${{ matrix.cell }} + INPUT_CORES: ${{ inputs.cores }} CTL: ../ctl/.github/scripts/timed-run.mjs # The runners share one account home, so A2's check diffs a snapshot of the home and /tmp. Not counted as the job's: # - any runner install (a dir holding `.runner`) and the tool cache: the runner service's own _work, _temp, _tool and diff --git a/docs/testing/playtests.md b/docs/testing/playtests.md index a7d16f263..4fe16b36c 100644 --- a/docs/testing/playtests.md +++ b/docs/testing/playtests.md @@ -232,7 +232,11 @@ load, the meetup workload, the slopes tick profile) go to the `souls-timed` self `bench-client`, `meetup` or `slopes-profile`), `repeats` (1 to 3) and allowlisted `args`. Each runner has cores of its own and takes one job; a run records its starting load and does not start while its own cores are busy. The last four put the world on two of the runner's physical cores and the load generator on the rest (one on a three-core runner, two -on a four-core one), the split recorded in `timed-run.json`. An `eval-load` artifact also carries each world's own log +on a four-core one), the split recorded in `timed-run.json`. For those, `cores` sets a core plan instead: +`world=N` or `world=N,launcher=M` physical cores, with the load drivers on the rest. The launcher processes are then +pinned apart, as the world's are. A plan that does not fit the runner's own cores is refused before the run starts. +Each run's `cpuSplit` records the plan and each group's busy share of its own CPUs (`busy..mean` over the run, +`peak5s` its busiest 5 s). An `eval-load` artifact also carries each world's own log (`eval-load/n-r/server/world.log`), and `eval-load.log` names a world that exited on its own (exit code, signal) and its resident memory every 30 s. The runners use a dedicated, unprivileged account: the args allowlist is not a security boundary, since a branch's own scripts run. Live-model seats and GPU work stay on a