diff --git a/docs/architecture/load.md b/docs/architecture/load.md index 86540a6d6..743df73d8 100644 --- a/docs/architecture/load.md +++ b/docs/architecture/load.md @@ -146,10 +146,10 @@ As built (L1, `tools/world-launcher/guard.mjs` + `children.mjs`): liveness asks - the act's handler: `act.bind`, `act.check`, `act.handler-other`; - the tick's route searches: `route.first` (a walk's first plan), `route.people` (`retryPeople`), `route.detour` (a re-plan from inside a walk's step), `route.chunk` (a paged walk's next chunk) and `route.other`; together they are #2470's `act.route(tick)`; - `act.commit(tick)` and `tick`; - - the planner: `nav.sync` (`syncPaged`, `navScan`, `navChanged`) and `nav.audit` (`navWarmV1`, `fineTileSteps`, `labelComponents`); + - the planner: `nav.sync` (`syncPaged`, `navScan`, `navChanged`); `nav.warm`, the connectivity audit (`navWarmV1`, or a tile or labels built in connectivity's `update`/`compile`); `nav.tile`, a fine tile built on demand (`fineTileSteps` outside the audit); `nav.label`, a tile's labels built on demand (`labelComponents` outside it). Until #2592 these three were one group, `nav.audit`, and they still sum to it; - `gc`, `snapshot` and requests by route. - The report gives each act group's clock ms per act, the planner's ms per second of clock, and long holds (busy runs of 100 ms or more) by their main group. Planner work inside an act stays `nav.*`. + The report gives each act group's clock ms per act, the planner's ms per second of clock, and long holds (busy runs of 100 ms or more) by their main group. Planner work inside an act stays `nav.*`. Each JS task of 100 ms or more gets a cause (`step0.causes`, the top 10 per group, and `step0.longTasks`, each task with its heaviest path), so the report needs no `.cpuprofile`: for `nav.*` the caller outside the spatial service, otherwise the frame just under the group's entry, followed down while one child holds at least half the task. - **`--menu-stats`.** The world runs with `EDGE_MENU_STATS=1`, which logs `[menu-stats]` every 10 s. The report folds the counters' change over the load window, with builds per act (`one.built` plus `one.exact`, the exact-walk projections). The window runs between the first lines logged after the load's start and after its end, so each end is up to 10 s late. - **Without these flags** the world and the report are as before, apart from the intended and achieved line. - **Via timed-run.** `timed-run eval-load` allows `--profile` and `--menu-stats`. diff --git a/tools/playtest/eval/prof-buckets.mjs b/tools/playtest/eval/prof-buckets.mjs index fe8298d32..07df970a4 100644 --- a/tools/playtest/eval/prof-buckets.mjs +++ b/tools/playtest/eval/prof-buckets.mjs @@ -7,19 +7,36 @@ // walk's step (a detour, stepping aside, a wait's re-plan), a paged walk's next // chunk, and any other // act.commit(tick), tick the tick applying inputs (drainInputs), and the rest of the tick -// nav.sync, nav.audit the planner's page sync and scans (syncPaged, navScan, navChanged), and the -// connectivity audit (navWarmV1, fineTileSteps, labelComponents) +// nav.sync the planner's page sync and scans (syncPaged, navScan, navChanged) +// nav.warm, nav.tile, nav.label the connectivity audit (navWarmV1, with the sync it starts with, or a fine +// tile or labels built inside connectivity.ts update/compile); a fine tile built +// on demand outside it (fineTileSteps, e.g. fineTile -> drain); a tile's labels +// built on demand outside it (labelComponents, e.g. a label-cache miss in check) +// Until the split (M4's and A9's tables, #2592) these three were one group, `nav.audit`: the samples with navWarmV1, +// fineTileSteps or labelComponents on the stack. nav.audit = nav.warm + nav.tile + nav.label, sample for sample, and +// perAct's navAuditMsPerSec stays that sum. // gc, snapshot, req , req (middleware/unknown), other // #2533: nav.sync's samples also by who asked (`navSyncBy`): the action step or pass a tick ran the sync in, or the server // frame outside a tick, and the first frame outside the spatial service. // Long holds: runs of busy samples (no idle between) of at least `holdMs`, each named by the group that took most of it. // #2592 `step0`: each hold of 1 s or more split into JS tasks (one long task, or back-to-back work), tasks of 50, 100, 250 // and 1000 ms or more by the group that took most of each, and the 25 longest tasks with their app frames (inclusive ms). +// #2592 attribution: each task of 100 ms or more gets a cause, from its main group's samples only: +// nav.* who asked: the first app frame outside the spatial service (and the engine's nav*V1 wrappers), going out +// from the outermost frame of the group (navWarmV1 or connectivity update, fineTileSteps, labelComponents, +// the sync's frame); +// others the frame just under the group's entry (the innermost tick frame for tick, act.commit(tick) and route.*; +// the outermost bind or check frame for act.bind and act.check; the act's function for act.handler-other; +// the root for the rest), then down the heaviest child while it holds at least 50 % of the task; the deepest +// app frame on that path. +// `causes` lists the top 10 causes per group by total ms, with counts. `longTasks` keeps every such task compactly (its +// cause, groups and the heaviest path's app frames), so later runs need no .cpuprofile (load.mjs deletes it). import { readFileSync } from 'node:fs'; const ROUTE_RE = /app\.(get|post|put|delete|patch|all)\(\s*['"`]([^'"`]+)/i; const SNAP = new Set(['captureCheckpoint', 'captureSnapshot', 'schedulePeriodicSnapshot', 'snapshotView', 'writeSnapshot', 'persistSnapshot']); -const AUDIT = new Set(['navWarmV1', 'fineTileSteps', 'labelComponents']); +/** The connectivity audit's build (connectivity.ts): a fine tile or labels built under it are the audit's own. */ +const AUDIT_BUILD = new Set(['update', 'compile']); const SYNC = new Set(['navScan', 'syncPaged', 'navChanged']); const BIND = new Set(['bindOne', 'bindsCore', 'bindsTail', 'binds']); const CHECK = new Set(['canPerform', 'routeProblem', 'previewAdmission', 'offerGate']); @@ -52,13 +69,14 @@ export function bucketProfile(p, repo, { holdMs = 100 } = {}) { if (memo.has(leafId)) return memo.get(leafId); const ids = []; for (let x = leafId; x != null; x = parent.get(x)) ids.push(x); const frames = ids.map((id) => byId.get(id).callFrame), names = new Set(frames.map((f) => f.functionName)); - let route = null, req = false, tick = false, drain = false, outer = null; + let route = null, req = false, tick = false, drain = false, outer = null, build = false; for (let j = frames.length - 1; j >= 0; j--) { const f = frames[j], rel = relOf(f.url); const r = routeAt(f.url, f.lineNumber); if (r) route = r; if (/hono/.test(rel) || /^apps\/server\/src\/(http|load\/admission)\.ts$/.test(rel)) req = true; if (TICK.has(f.functionName)) tick = true; if (f.functionName === 'drainInputs') drain = true; + if (AUDIT_BUILD.has(f.functionName) && rel.endsWith('spatial/connectivity.ts')) build = true; if (!outer && /^(apps|packages|mars)\//.test(rel)) outer = `${f.functionName || '(anon)'} ${rel}:${f.lineNumber + 1}`; } const any = (s) => { for (const n of s) if (names.has(n)) return true; return false; }; @@ -66,7 +84,9 @@ export function bucketProfile(p, repo, { holdMs = 100 } = {}) { let g; if (frames[0].functionName === '(garbage collector)') g = 'gc'; else if (any(SNAP)) g = 'snapshot'; - else if (any(AUDIT)) g = 'nav.audit'; + else if (names.has('navWarmV1') || (build && (names.has('fineTileSteps') || names.has('labelComponents')))) g = 'nav.warm'; + else if (names.has('fineTileSteps')) g = 'nav.tile'; + else if (names.has('labelComponents')) g = 'nav.label'; else if (any(SYNC)) g = 'nav.sync'; else if (route === 'POST /v1/edge/act' || (!tick && !route && any(ACT_FN))) g = any(BIND) ? 'act.bind' : any(CHECK) ? 'act.check' : 'act.handler-other'; else if (tick) g = any(ROUTE) ? routeKind() : drain ? 'act.commit(tick)' : 'tick'; @@ -125,6 +145,48 @@ export function bucketProfile(p, repo, { holdMs = 100 } = {}) { holdsOver1s: holds.filter((h) => h.ms >= 1000).map((h) => ({ atS: r1(h.at / 1000), ms: r1(h.ms), tasks: h.tasks, longestTaskMs: r1(h.longestTask), groups: top(h.by, 5) })), longestTasks: [...tasks].sort((a, b) => b.ms - a.ms).slice(0, 25).map((tk) => ({ atS: r1(tk.at / 1000), ms: r1(tk.ms), groups: top(tk.by, 4), frames: framesOf(tk, 12) })), }; + // #2592 attribution (see the header): each task of 100 ms or more, its cause from its main group's samples + const rootId = p.nodes.find((n) => !parent.has(n.id)).id; + const pathOf = (leaf) => { const out = []; for (let x = leaf; x != null; x = parent.get(x)) out.push(x); return out.reverse(); }; + const fnOf = (id) => byId.get(id).callFrame.functionName, relId = (id) => relOf(byId.get(id).callFrame.url); + const isApp = (id) => /^(apps|packages|mars)\//.test(relId(id)); + const label = (id) => { const f = byId.get(id).callFrame; return `${f.functionName || '(anon)'} ${relId(id).split('/').slice(-2).join('/')}:${f.lineNumber + 1}`; }; + const NAV_OF = { 'nav.warm': (id) => fnOf(id) === 'navWarmV1' || (AUDIT_BUILD.has(fnOf(id)) && relId(id).endsWith('spatial/connectivity.ts')), + 'nav.tile': (id) => fnOf(id) === 'fineTileSteps', 'nav.label': (id) => fnOf(id) === 'labelComponents', 'nav.sync': (id) => SYNC.has(fnOf(id)) }; + const ENTRY = { tick: [TICK, 'in'], 'act.commit(tick)': [TICK, 'in'], 'act.bind': [BIND, 'out'], 'act.check': [CHECK, 'out'], 'act.handler-other': [ACT_FN, 'out'], snapshot: [SNAP, 'out'] }; + const entryIn = (g, P) => { + const [set, way] = ENTRY[g] ?? (g.startsWith('route.') ? [TICK, 'in'] : [null]); if (!set) return 0; + let e = -1; for (let k = 0; k < P.length; k++) if (set.has(fnOf(P[k]))) { e = k; if (way === 'out') break; } + return Math.max(e, 0); + }; + const askedBy = (g, P) => { + const o = P.findIndex(NAV_OF[g]); + for (let k = o - 1; k >= 0; k--) if (isApp(P[k]) && !relId(P[k]).includes('/engine/spatial/') && !/^(nav\w*V1|connectivityV1)$/.test(fnOf(P[k]))) return label(P[k]); + return '-'; + }; + const heaviest = (inc, from, keep) => { const out = []; for (let cur = from; ;) { let best = null; for (const c of byId.get(cur).children ?? []) if ((inc.get(c) ?? 0) > (inc.get(best) ?? 0)) best = c; if (best == null || !keep(best, out.length)) return out; out.push(best); cur = best; } }; + const causeOf = (tk, g) => { + const leaves = [...tk.leaf].filter(([id]) => g === 'gc' || classify(id) === g), all = new Map(), by = new Map(), inc = new Map(), entries = new Map(); + for (const [leaf, ms] of leaves) { + const P = pathOf(leaf); for (const id of P) all.set(id, (all.get(id) ?? 0) + ms); + if (NAV_OF[g]) { const k = askedBy(g, P); by.set(k, (by.get(k) ?? 0) + ms); continue; } + const e = entryIn(g, P); entries.set(P[e], (entries.get(P[e]) ?? 0) + ms); for (let k = e + 1; k < P.length; k++) inc.set(P[k], (inc.get(P[k]) ?? 0) + ms); + } + const path = heaviest(all, rootId, () => true).filter(isApp).slice(-12).map((id) => [label(id), r1(all.get(id))]); + if (NAV_OF[g]) return { cause: top(by, 1)[0]?.[0] ?? '-', path }; + const E = top(entries, 1)[0]?.[0]; if (E == null) return { cause: '-', path }; + const down = heaviest(inc, E, (id, n) => n === 0 || inc.get(id) >= tk.ms / 2), app = down.filter(isApp); + return { cause: label(app.at(-1) ?? down[0] ?? E), path }; + }; + const longTasks = [], causes = {}; + for (const tk of tasks) { + if (tk.ms < 100) continue; + const g = mainOf(tk.by), { cause, path } = causeOf(tk, g), c = (causes[g] ??= new Map()), e = c.get(cause) ?? { n: 0, ms: 0 }; + e.n++; e.ms += tk.ms; c.set(cause, e); + longTasks.push({ atS: r1(tk.at / 1000), ms: r1(tk.ms), group: g, cause, groups: top(tk.by, 3), path }); + } + step0.causes = Object.fromEntries(Object.entries(causes).map(([g, c]) => [g, [...c].sort((a, b) => b[1].ms - a[1].ms).slice(0, 10).map(([k, e]) => [k, e.n, r1(e.ms)])])); + step0.longTasks = longTasks; const bySource = new Map(); for (const h of holds) { const [src] = [...h.by].sort((a, b) => b[1] - a[1])[0]; const e = bySource.get(src) ?? { n: 0, ms: 0, maxMs: 0 }; e.n++; e.ms += h.ms; e.maxMs = Math.max(e.maxMs, h.ms); bySource.set(src, e); } return { @@ -144,7 +206,7 @@ export function perAct(b, acts) { const route = ['route.first', 'route.people', 'route.detour', 'route.chunk', 'route.other'].reduce((s, k) => s + g(k), 0); const sec = b.totalMs / 1000; return { acts, ...per, 'act.route(tick)': acts ? r1(route / acts) : null, handlerSide: acts ? r1((g('act.bind') + g('act.check') + g('act.handler-other')) / acts) : null, - navSyncMsPerSec: sec ? r1(g('nav.sync') / sec) : null, navAuditMsPerSec: sec ? r1(g('nav.audit') / sec) : null, busyShare: b.totalMs ? Math.round(1000 * b.busyMs / b.totalMs) / 1000 : null }; + navSyncMsPerSec: sec ? r1(g('nav.sync') / sec) : null, navAuditMsPerSec: sec ? r1((g('nav.warm') + g('nav.tile') + g('nav.label')) / sec) : null, busyShare: b.totalMs ? Math.round(1000 * b.busyMs / b.totalMs) / 1000 : null }; } export const readProfile = (file) => JSON.parse(readFileSync(file, 'utf8')); diff --git a/tools/playtest/eval/prof-buckets.test.mjs b/tools/playtest/eval/prof-buckets.test.mjs index ea4adc573..457047953 100644 --- a/tools/playtest/eval/prof-buckets.test.mjs +++ b/tools/playtest/eval/prof-buckets.test.mjs @@ -26,7 +26,7 @@ test('#2508: every sample goes to one group; the tick\'s route searches by kind; [['tickOnce', 'advance'], 50], [['(idle)'], 5], [['tickOnce', 'syncPaged'], 9], [['tickOnce', 'navWarmV1'], 3], [['(garbage collector)'], 2], ]); const b = bucketProfile(p, '/repo'); - assert.deepEqual(b.groups, { 'act.bind': 30, 'act.check': 10, 'act.handler-other': 8, 'route.first': 40, 'route.people': 20, 'route.detour': 12, 'route.chunk': 6, 'act.commit(tick)': 4, tick: 50, 'nav.sync': 9, 'nav.audit': 3, gc: 2 }); + assert.deepEqual(b.groups, { 'act.bind': 30, 'act.check': 10, 'act.handler-other': 8, 'route.first': 40, 'route.people': 20, 'route.detour': 12, 'route.chunk': 6, 'act.commit(tick)': 4, tick: 50, 'nav.sync': 9, 'nav.warm': 3, gc: 2 }); assert.equal(b.busyMs, 194); assert.equal(b.totalMs, 199); // one hold before the idle (180 ms, mostly the tick) and none after (14 ms) assert.deepEqual(b.holds, { overMs: 100, n: 1, ms: 180, bySource: { tick: { n: 1, ms: 180, maxMs: 180 } } }); @@ -62,8 +62,44 @@ test('#2592: step0 splits a hold into tasks, counts long tasks by group and name ]); const s = bucketProfile(p, '/repo').step0; assert.deepEqual(s.tasksOver, { 50: 2, 100: 2, 250: 2, 1000: 0 }); - assert.deepEqual(s.taskHist['nav.audit'], { n: 1, ms: 820, maxMs: 820, over50: 1, over100: 1, over250: 1, over1000: 0 }); + assert.deepEqual(s.taskHist['nav.warm'], { n: 1, ms: 820, maxMs: 820, over50: 1, over100: 1, over250: 1, over1000: 0 }); assert.deepEqual(s.taskHist['act.bind'], { n: 1, ms: 400, maxMs: 400, over50: 1, over100: 1, over250: 1, over1000: 0 }); - assert.deepEqual(s.holdsOver1s, [{ atS: 0, ms: 1220, tasks: 2, longestTaskMs: 820, groups: [['nav.audit', 800], ['act.bind', 400], ['gc', 20]] }]); + assert.deepEqual(s.holdsOver1s, [{ atS: 0, ms: 1220, tasks: 2, longestTaskMs: 820, groups: [['nav.warm', 800], ['act.bind', 400], ['gc', 20]] }]); assert.deepEqual(s.longestTasks[0].frames, [['tickOnce src/world.ts:1', 800], ['navWarmV1 mars/a.ts:1', 800]]); }); + +test('#2592: nav.audit split into nav.warm, nav.tile and nav.label; each task of 100 ms or more attributed to its cause', () => { + const W = 'apps/server/src/world.ts', SP = 'mars/viewer/engine/spatial', T = `tickOnceInner@${W}`, R = 'stepInner@mars/viewer/engine/runtime/runtime.ts'; + const P = 'mars/viewer/engine/people.ts', idle = [['(idle)'], 5]; + const p = profileOf([ + // the budgeted audit: a tile built inside connectivity's compile, under navWarmV1 (asked by the bridge's navStep) + [['processTimers', T, 'navStep@apps/server/src/god/engine-bridge.ts', 'navWarmV1@packages/engine-mars/src/engine.ts', `update@${SP}/connectivity.ts`, `compile@${SP}/connectivity.ts`, `fineTileSteps@${SP}/tiled.ts`], 120], idle, + // a fine tile built on demand by a walker's page sync (inside syncPaged, still a tile) + [['processTimers', T, 'stepWalk@mars/viewer/engine/locomotion/walk.ts', 'enterLeg@mars/viewer/engine/locomotion/walk.ts', `syncPaged@${SP}/service.ts`, `fineTile@${SP}/tiled.ts`, `drain@${SP}/tiled.ts`, `fineTileSteps@${SP}/tiled.ts`], 150], idle, + // a tile's labels on demand: a label-cache miss in an act's connectivity read (the engine's wrapper is skipped) + [['processImmediate', 'actWork@apps/server/src/edge/ledger/ledger.ts', 'goal@apps/server/src/god/nav.ts', 'navCheckV1@packages/engine-mars/src/engine.ts', `check@${SP}/connectivity.ts`, `labelComponents@${SP}/connectivity.ts`], 110], idle, + // the rest of the tick, one dominant path: tickOnceInner > stepInner > stepPeople (130 of 170) > crowd (100 of 170) + [['processTimers', T, R, `stepPeople@${P}`, `crowd@${P}`], 100], [['processTimers', T, R, `stepPeople@${P}`, `other@${P}`], 30], + [['processTimers', T, R, 'stepDoors@mars/viewer/engine/doors.ts'], 40], idle, + // the rest of the tick, no dominant child: the frame just under the tick + [['processTimers', T, R, `a@${P}`], 40], [['processTimers', T, R, `b@${P}`], 30], [['processTimers', T, R, `c@${P}`], 30], idle, + // under 100 ms: not attributed + [['processTimers', T, 'navStep@apps/server/src/god/engine-bridge.ts', 'navWarmV1@packages/engine-mars/src/engine.ts'], 60], + ]); + const b = bucketProfile(p, '/repo'); + assert.deepEqual([b.groups['nav.warm'], b.groups['nav.tile'], b.groups['nav.label'], b.groups['nav.sync']], [180, 150, 110, undefined]); + assert.equal(b.groups['nav.audit'], undefined); + assert.equal(perAct(b, 1).navAuditMsPerSec, Math.round(10 * 440 / (b.totalMs / 1000)) / 10); // the old group: the three summed + const s = b.step0; + assert.deepEqual(s.longTasks.map((t) => [t.group, t.cause, t.ms]), [ + ['nav.warm', 'navStep god/engine-bridge.ts:1', 120], + ['nav.tile', 'enterLeg locomotion/walk.ts:1', 150], + ['nav.label', 'goal god/nav.ts:1', 110], + ['tick', 'crowd engine/people.ts:1', 170], + ['tick', 'stepInner runtime/runtime.ts:1', 100], + ]); + assert.deepEqual(s.causes.tick, [['crowd engine/people.ts:1', 1, 170], ['stepInner runtime/runtime.ts:1', 1, 100]]); + assert.deepEqual(s.causes['nav.tile'], [['enterLeg locomotion/walk.ts:1', 1, 150]]); + // the compact record keeps the task's heaviest path (app frames), so a later read needs no .cpuprofile + assert.deepEqual(s.longTasks[3].path, [['tickOnceInner src/world.ts:1', 170], ['stepInner runtime/runtime.ts:1', 170], ['stepPeople engine/people.ts:1', 130], ['crowd engine/people.ts:1', 100]]); +});