Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 48 additions & 15 deletions .github/scripts/timed-run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ export function parseInputs(env) {
if (!RUNNERS.includes(runner)) bad('runner', runner, RUNNERS.join(' or '));
if (runner === 'hosted' && command !== 'counted') bad('runner', runner, `${command} is a timing run and runs on souls-timed only`);
const args = checkArgs(command, env.INPUT_ARGS), inp = { ref, command, repeats: Number(repeats), args };
const cores = String(env.INPUT_CORES ?? '').trim(); // #2507: a core plan, for the split commands only
if (cores && !SPLIT.includes(command)) bad('cores', cores, `a core plan is for ${SPLIT.join(', ')} only`);
if (cores) inp.cores = parseCorePlan(cores) ?? bad('cores', cores, 'world=N or world=N,launcher=M (physical cores, 1 to 64); the load drivers get the rest');
if (command === 'clip' && !args.includes('--kit')) bad('args', env.INPUT_ARGS, 'clip needs --kit <name> (a folder of tools/proof/clips)');
if (command !== 'counted') return { ...inp, runner };
// counted: a base ref, a scenario, and the one cell this job runs (base-k or cand-k, k up to repeats)
Expand Down Expand Up @@ -151,16 +154,32 @@ export function frameTiming(pts) {
return { frames: t.length, spanSec: r3(span), meanFps: span > 0 ? +((t.length - 1) / span).toFixed(2) : null, maxGapSec: gaps.length ? r3(Math.max(...gaps)) : null, gapsOver100ms: gaps.filter((g) => g > 0.1).length };
}

/** #2507: `world=N[,launcher=M]` → { world, launcher } physical cores (launcher 0: not split off), or null. */
export function parseCorePlan(s) {
const m = /^world=([1-9]\d?)(?:,launcher=([1-9]\d?))?$/.exec(String(s)), w = Number(m?.[1]), l = Number(m?.[2] ?? 0);
return m && w <= 64 && l <= 64 ? { world: w, launcher: l } : null;
}
/** #1970 A2: the runner's own logical CPUs grouped into physical cores (`siblingsOf(cpu)`: that CPU's SMT siblings); the
* first two cores for the world under test, the rest for the load generator: 2+2 on four cores, 2+1 on three (the timed
* runners give each job three). Any other count is refused. */
export function splitCores(own, siblingsOf) {
* runners give each job three). Any other count is refused. #2507: a core plan gives the world its first N cores, the
* launcher the next M and the load drivers the rest (at least one); a plan that does not fit is refused. */
export function splitCores(own, siblingsOf, plan = null) {
const cores = new Map();
for (const c of own) { const k = siblingsOf(c).filter((x) => own.includes(x)).sort((a, b) => a - b).join(',') || String(c); cores.set(k, k.split(',').map(Number)); }
const byFirst = [...cores.values()].sort((a, b) => a[0] - b[0]);
const n = byFirst.length;
const n = byFirst.length, take = (a, b) => byFirst.slice(a, b).flat().join(',');
if (plan) {
const { world: w, launcher: l } = plan, name = `world=${w}${l ? `,launcher=${l}` : ''}`;
if (w + l + 1 > n) throw new Error(`the core plan ${name} needs ${w + l + 1} physical cores (at least one for the load drivers); this runner has ${n} of its own`);
return { physicalCores: n, plan: name, layout: `${w}+${l ? `${l}+` : ''}${n - w - l}`, world: take(0, w), ...(l ? { launcher: take(w, w + l) } : {}), load: take(w + l) };
}
if (n !== 3 && n !== 4) throw new Error(`the runner has ${n} physical core(s) of its own; the world/load split needs 3 or 4`);
return { physicalCores: n, layout: `2+${n - 2}`, world: byFirst.slice(0, 2).flat().join(','), load: byFirst.slice(2).flat().join(',') };
return { physicalCores: n, layout: `2+${n - 2}`, world: take(0, 2), load: take(2) };
}
/** #2507: each group's mean busy fraction of its own CPUs between two /proc/stat samples (groups: name → "1,2,3"). */
export function groupBusy(a, b, groups) {
const busy = (c) => { const d = b[c] && a[c] ? b[c][1] - a[c][1] : 0; return d > 0 ? (b[c][0] - a[c][0]) / d : 0; };
return Object.fromEntries(Object.entries(groups).map(([k, v]) => { const cs = parseCpuList(v); return [k, +(cs.reduce((s, c) => s + busy(c), 0) / cs.length).toFixed(3)]; }));
}
/** [[pid, ppid], …] → every descendant pid of root. */
export function descendants(table, root) {
Expand All @@ -170,6 +189,8 @@ export function descendants(table, root) {
}
/** A world server process (a direct boot, or a child of the launcher), by its NUL-separated command line. */
export const isWorld = (cmdline) => cmdline.split('\0').some((a) => a.endsWith('apps/server/src/main.ts'));
/** #2507: the world launcher process (it carries every viewer byte and every act), by its NUL-separated command line. */
export const isLauncher = (cmdline) => cmdline.split('\0').some((a) => a.endsWith('tools/world-launcher/launcher.mjs'));

/** "0-3,8,10-11" → [0,1,2,3,8,10,11]. */
export function parseCpuList(s) {
Expand Down Expand Up @@ -298,18 +319,28 @@ const siblingsOf = (c) => { try { return parseCpuList(readFileSync(`/sys/devices
const procTable = () => readdirSync('/proc').filter((d) => /^\d+$/.test(d)).flatMap((d) => { try { return [[Number(d), Number(readFileSync(`/proc/${d}/stat`, 'utf8').split(') ').at(-1).split(' ')[1])]]; } catch { return []; } });
const allowedOf = (pid) => { try { return /^Cpus_allowed_list:\s*(\S+)/m.exec(readFileSync(`/proc/${pid}/status`, 'utf8'))?.[1] ?? null; } catch { return null; } };
/** #1970 A2: each world process under root, with all its threads, onto the world's cores as soon as it appears (threads
* made later inherit the mask). Returns what was pinned: a count and the allowed lists read back, no pids. */
function pinWorlds(root, cpus) {
const pinned = new Map(), tick = () => {
* made later inherit the mask). #2507: with a launcher in the plan, each launcher process too, onto its own cores (a
* world it starts is moved on to the world's). Every 5 s each group's busy share is sampled. Returns what was pinned (a
* count and the allowed lists read back, no pids) and each group's mean and peak busy share. */
function pinWorlds(root, split) {
const stat = () => parseProcStat(readFileSync('/proc/stat', 'utf8')), groups = { world: split.world, ...(split.launcher ? { launcher: split.launcher } : {}), load: split.load };
const first = stat(), peak = {}, pinned = { world: new Map(), launcher: new Map() };
let last = first, n = 0;
const tick = () => {
for (const pid of descendants(procTable(), root)) {
if (pinned.has(pid)) continue;
if (pinned.world.has(pid) || pinned.launcher.has(pid)) continue;
let cmd; try { cmd = readFileSync(`/proc/${pid}/cmdline`, 'utf8'); } catch { continue; }
if (!isWorld(cmd)) continue;
try { execFileSync('taskset', ['-a', '-p', '-c', cpus, String(pid)], { stdio: 'ignore' }); pinned.set(pid, allowedOf(pid)); } catch { /* exited */ }
const g = isWorld(cmd) ? 'world' : split.launcher && isLauncher(cmd) ? 'launcher' : null; if (!g) continue;
try { execFileSync('taskset', ['-a', '-p', '-c', split[g], String(pid)], { stdio: 'ignore' }); pinned[g].set(pid, allowedOf(pid)); } catch { /* exited */ }
}
if (++n % 20) return;
const now = stat(); for (const [k, v] of Object.entries(groupBusy(last, now, groups))) peak[k] = Math.max(peak[k] ?? 0, v); last = now;
};
const iv = setInterval(tick, 250);
return { stop: () => { clearInterval(iv); return { worldProcesses: pinned.size, worldAllowed: [...new Set(pinned.values())] }; } };
return { stop: () => { clearInterval(iv); const mean = groupBusy(first, stat(), groups);
return { worldProcesses: pinned.world.size, worldAllowed: [...new Set(pinned.world.values())],
...(split.launcher ? { launcherProcesses: pinned.launcher.size, launcherAllowed: [...new Set(pinned.launcher.values())] } : {}),
busy: Object.fromEntries(Object.keys(groups).map((k) => [k, { mean: mean[k], peak5s: peak[k] ?? null }])) }; } };
}
const la = () => loadavg().slice(0, 2).map((x) => +x.toFixed(2));
const sh = (cmd, args) => { try { execFileSync(cmd, args, { stdio: 'ignore' }); return true; } catch { return false; } };
Expand Down Expand Up @@ -419,10 +450,11 @@ async function main(cmd) {
console.log('no entries written outside the per-run directory'); return;
}
const inp = parseInputs(process.env), rf = arg('record');
if (cmd === 'inputs') { output('target', target(inp)); console.log(`inputs ok: ${inp.ref}, ${inp.command} x${inp.repeats}${inp.args.length ? ` ${inp.args.join(' ')}` : ''}${inp.cell ? `, cell ${inp.cell} of base ${inp.base}` : ''} on ${inp.runner}`); return; }
if (cmd === 'inputs') { output('target', target(inp)); console.log(`inputs ok: ${inp.ref}, ${inp.command} x${inp.repeats}${inp.args.length ? ` ${inp.args.join(' ')}` : ''}${inp.cores ? `, cores world=${inp.cores.world},launcher=${inp.cores.launcher}` : ''}${inp.cell ? `, cell ${inp.cell} of base ${inp.base}` : ''} on ${inp.runner}`); return; }
if (cmd === 'load') { // A3: own cores and whole machine; wait a bounded time for own cores to settle, else refuse
// counted scores pass/fail per item: its starting load is recorded, never gated
const own = ownCpus(), stat = () => parseProcStat(readFileSync('/proc/stat', 'utf8')), gated = inp.command !== 'counted';
if (inp.cores) console.log(`cores: plan ${JSON.stringify(splitCores(own, siblingsOf, inp.cores))}`); // #2507: throws if it does not fit
let f;
for (let t = 0; ; t += 15) {
const a = stat(); await sleep(SAMPLE_S * 1000); f = loadFacts(a, stat(), own);
Expand All @@ -443,18 +475,18 @@ async function main(cmd) {
if (inp.command === 'clip') return runClip(inp, rec, out, env);
// #1970: the world on two physical cores, the load generator (the driver and all it starts but the world) on the rest
// (one or two). HOME goes inside the per-run directory: the capacity bench moves finished world data under ~/.Trash.
const split = SPLIT.includes(inp.command) ? splitCores(ownCpus(), siblingsOf) : null;
const split = SPLIT.includes(inp.command) ? splitCores(ownCpus(), siblingsOf, inp.cores) : null;
if (split) {
execFileSync('taskset', ['-V'], { stdio: 'ignore' });
env.HOME = join(env.PER_RUN, 'home'); mkdirSync(env.HOME, { recursive: true });
console.log(`cores: split ${split.layout} of ${split.physicalCores} physical, world ${split.world}, load generator ${split.load}`);
console.log(`cores: split ${split.layout} of ${split.physicalCores} physical, world ${split.world}${split.launcher ? `, launcher ${split.launcher}` : ''}, load generator ${split.load}`);
}
const n = inp.command === 'walk' ? inp.repeats : 1, runs = [];
for (let k = 1; k <= n; k++) {
const argv = commandLine(inp, { out, port: await freePorts(PORT_SPAN[inp.command]) }), log = join(out, logName(inp.command, k)), t0 = Date.now();
console.log(`${inp.command} ${k}/${n}: ${argv.join(' ')}`);
const [bin, args] = split ? ['taskset', ['-c', split.load, process.execPath, ...argv]] : [process.execPath, argv];
const ws = createWriteStream(log), child = spawn(bin, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }), pins = split ? pinWorlds(child.pid, split.world) : null;
const ws = createWriteStream(log), child = spawn(bin, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }), pins = split ? pinWorlds(child.pid, split) : null;
for (const s of [child.stdout, child.stderr]) s.on('data', (b) => { ws.write(b); process.stdout.write(b); });
for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig));
const code = await new Promise((r) => child.on('close', (c) => r(c ?? 1))); ws.end();
Expand All @@ -464,6 +496,7 @@ async function main(cmd) {
const failed = runs.filter((r) => r.exitCode !== 0);
if (failed.length) throw new Error(`${failed.length} of ${runs.length} run(s) exited non-zero`);
if (runs.some((r) => r.cpuSplit && !r.cpuSplit.worldProcesses)) throw new Error('no world process was seen to pin to the world\'s cores');
if (runs.some((r) => r.cpuSplit?.launcher && !r.cpuSplit.launcherProcesses)) throw new Error('no launcher process was seen to pin to the launcher\'s cores');
return;
}
if (cmd === 'stage') {
Expand Down
24 changes: 23 additions & 1 deletion .github/scripts/timed-run.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createHash } from 'node:crypto';
import { parseInputs, checkArgs, commandLine, parseCpuList, parseProcStat, loadFacts, stage, snapshot, newEntries, sortStrays, globTest, splitCores, descendants, isWorld, SPLIT, LOAD_GATE, COUNTED, target, countedSteps } from './timed-run.mjs';
import { parseInputs, checkArgs, commandLine, parseCpuList, parseProcStat, loadFacts, stage, snapshot, newEntries, sortStrays, globTest, splitCores, descendants, isWorld, isLauncher, parseCorePlan, groupBusy, SPLIT, LOAD_GATE, COUNTED, target, countedSteps } from './timed-run.mjs';

const ENV = { INPUT_REF: 'main', INPUT_COMMAND: 'walk', INPUT_REPEATS: '1', INPUT_ARGS: '' };

Expand Down Expand Up @@ -178,6 +178,28 @@ test('core split: two physical cores for the world, the rest (two of four, one o
assert.throws(() => splitCores([0, 1, 2, 3, 4, 5], (c) => [c]), /6 physical core/);
});

test('core plan (#2507): cores is accepted for the split commands, validated, recorded in the inputs, and refused at the start gate when it does not fit', () => {
const V = { ...ENV, INPUT_COMMAND: 'viewers' };
assert.deepEqual(parseInputs({ ...V, INPUT_CORES: 'world=2,launcher=1' }).cores, { world: 2, launcher: 1 });
assert.deepEqual(parseInputs({ ...V, INPUT_CORES: ' world=6 ' }).cores, { world: 6, launcher: 0 });
assert.ok(!('cores' in parseInputs({ ...V, INPUT_CORES: '' })), 'no plan: the inputs are as before');
for (const v of ['world=0', 'world=2,launcher=0', 'launcher=1', 'world=2,launcher=1,load=1', 'world=65', 'world=2 launcher=1', 'world=x', 'world=2;id'])
assert.throws(() => parseInputs({ ...V, INPUT_CORES: v }), /input cores=.*world=N or world=N,launcher=M/, v);
assert.throws(() => parseInputs({ ...ENV, INPUT_CORES: 'world=2' }), /input cores=.*for eval-load, .*viewers only/, 'walk takes no plan');
assert.equal(parseCorePlan('world=2,launcher=1').launcher, 1);
const smt = (c) => [c % 8, (c % 8) + 8], three = [5, 6, 7, 13, 14, 15];
assert.deepEqual(splitCores(three, smt, { world: 1, launcher: 1 }), { physicalCores: 3, plan: 'world=1,launcher=1', layout: '1+1+1', world: '5,13', launcher: '6,14', load: '7,15' });
assert.deepEqual(splitCores([0, 1, 2, 3, 4, 5, 6, 7], (c) => [c], { world: 6, launcher: 1 }), { physicalCores: 8, plan: 'world=6,launcher=1', layout: '6+1+1', world: '0,1,2,3,4,5', launcher: '6', load: '7' });
assert.deepEqual(splitCores([0, 1, 2, 3, 4, 5], (c) => [c], { world: 2, launcher: 0 }), { physicalCores: 6, plan: 'world=2', layout: '2+4', world: '0,1', load: '2,3,4,5' }, 'a plan lifts the 3-or-4 rule');
assert.throws(() => splitCores(three, smt, { world: 2, launcher: 1 }), /the core plan world=2,launcher=1 needs 4 physical cores \(at least one for the load drivers\); this runner has 3 of its own/);
assert.throws(() => splitCores(three, smt, { world: 3, launcher: 0 }), /world=3 needs 4/);
assert.deepEqual(splitCores(three, smt, null), splitCores(three, smt), 'no plan: the split of today');
assert.ok(isLauncher(['/usr/bin/node', '/w/src/tools/world-launcher/launcher.mjs', '--port', '1'].join('\0')));
for (const c of ['node\0apps/server/src/main.ts', 'node\0tools/world-launcher/launcher.test.mjs', 'node\0tools/bench/capacity/viewers/viewers.mjs']) assert.ok(!isLauncher(c), c);
const a = parseProcStat('cpu0 10 0 0 90 0 0 0 0\ncpu1 0 0 0 100 0 0 0 0\ncpu2 50 0 0 50 0 0 0 0\n'), b = parseProcStat('cpu0 60 0 0 140 0 0 0 0\ncpu1 10 0 0 190 0 0 0 0\ncpu2 150 0 0 50 0 0 0 0\n');
assert.deepEqual(groupBusy(a, b, { world: '0,1', load: '2' }), { world: 0.3, load: 1 });
});

test('world pinning: only world servers under the run\'s own process tree', () => {
assert.deepEqual(descendants([[10, 1], [11, 10], [12, 11], [13, 1], [14, 12]], 10).sort(), [11, 12, 14]);
assert.deepEqual(descendants([[10, 1]], 99), []);
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/timed-run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ on:
args:
description: 'Allowlisted: walk --world; eval-scripted --battery <name> --seat scripted --items --warm-sec; eval-load --souls --duration-sec --card-sec; bench-client --arms --profile --raw-from --short; meetup --profile --active-min --short; slopes-profile none; load --world --boots --souls --profile --short; settled --world --souls --settle-min --short; hundred --small --eden --short --fixture-only --active-min --gap-min --settle-min --build-parallel; colocate --forms --settle-s --walk-s --busy --short; viewers --viewers --window-sec --short; counted --scenario --loop-sec; clip --kit --viewport'
default: ''
cores:
description: 'eval-load, bench-client, meetup, slopes-profile, viewers only: a core plan, world=N or world=N,launcher=M (physical cores); the load drivers get the rest. Blank: 2 for the world, the rest for the load'
default: ''
permissions:
contents: read
jobs:
Expand All @@ -67,6 +70,7 @@ jobs:
INPUT_BASE: ${{ inputs.base }}
INPUT_RUNNER: ${{ inputs.runner }}
INPUT_CELL: ${{ matrix.cell }}
INPUT_CORES: ${{ inputs.cores }}
CTL: ../ctl/.github/scripts/timed-run.mjs
# The runners share one account home, so A2's check diffs a snapshot of the home and /tmp. Not counted as the job's:
# - any runner install (a dir holding `.runner`) and the tool cache: the runner service's own _work, _temp, _tool and
Expand Down
6 changes: 5 additions & 1 deletion docs/testing/playtests.md
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,11 @@ load, the meetup workload, the slopes tick profile) go to the `souls-timed` self
`bench-client`, `meetup` or `slopes-profile`), `repeats` (1 to 3) and allowlisted `args`. Each runner has cores of its
own and takes one job; a run records its starting load and does not start while its own cores are busy. The last four
put the world on two of the runner's physical cores and the load generator on the rest (one on a three-core runner, two
on a four-core one), the split recorded in `timed-run.json`. An `eval-load` artifact also carries each world's own log
on a four-core one), the split recorded in `timed-run.json`. For those, `cores` sets a core plan instead:
`world=N` or `world=N,launcher=M` physical cores, with the load drivers on the rest. The launcher processes are then
pinned apart, as the world's are. A plan that does not fit the runner's own cores is refused before the run starts.
Each run's `cpuSplit` records the plan and each group's busy share of its own CPUs (`busy.<group>.mean` over the run,
`peak5s` its busiest 5 s). An `eval-load` artifact also carries each world's own log
(`eval-load/n<souls>-r<repeat>/server/world.log`), and `eval-load.log` names a world that exited on its own (exit code,
signal) and its resident memory every 30 s. The runners use a dedicated, unprivileged account: the
args allowlist is not a security boundary, since a branch's own scripts run. Live-model seats and GPU work stay on a
Expand Down
Loading