From 8e3a76fb2367a239f99bb8a58d80e82d9b476670 Mon Sep 17 00:00:00 2001 From: Gene Kogan <1335251+genekogan@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:14:14 -0500 Subject: [PATCH] feat(telemetry): the numbers a spike needs are exported, alerted on, and reach the public health check (#2541) Four series (viewers per world, the launcher's own loop delay p99, worlds created, disk free/size), four rules (launcher-loop-stall, worlds-created, disk-free, and viewers-high as information only), and a problems file the collector rewrites after each scrape: every firing rule that pages is an alert: host code in /__sim/healthz at once (hostOk false), cleared when it resolves; a dead collector is alert:stale. sim-worlds-telemetry@.service is installed by install.sh and never enabled. A failed hourly backup (timeout or a FAILED database) raises backup-hourly: and shows on the health page, like the nightly one. Co-Authored-By: Claude Opus 5.5 --- docs/operations/telemetry.md | 52 +++++- tools/world-launcher/deploy/host-health.mjs | 24 ++- tools/world-launcher/deploy/install.sh | 5 +- .../deploy/sim-worlds-telemetry@.service | 27 ++++ .../health-dash-backup.test.mjs | 14 ++ tools/world-launcher/health-dash.mjs | 6 +- tools/world-launcher/host-health.test.mjs | 15 ++ tools/world-launcher/launcher.mjs | 25 ++- tools/world-launcher/metrics.mjs | 10 +- .../world-launcher/telemetry-alerts.test.mjs | 4 +- tools/world-launcher/telemetry-spike.test.mjs | 148 ++++++++++++++++++ tools/world-launcher/telemetry.mjs | 24 ++- tools/world-launcher/telemetry/alerts.mjs | 57 ++++++- 13 files changed, 377 insertions(+), 34 deletions(-) create mode 100644 tools/world-launcher/deploy/sim-worlds-telemetry@.service create mode 100644 tools/world-launcher/telemetry-spike.test.mjs diff --git a/docs/operations/telemetry.md b/docs/operations/telemetry.md index c77942877..fd4dd0ba4 100644 --- a/docs/operations/telemetry.md +++ b/docs/operations/telemetry.md @@ -21,6 +21,8 @@ node tools/world-launcher/telemetry.mjs report --dir /telemetry - node tools/world-launcher/telemetry.mjs close --dir /telemetry # roll up and prune now ``` +On the host, the `sim-worlds-telemetry@` unit runs the collector with alerts and the problems file, once the maintainer enables it ([below](#the-unit)). + `--url` must be a loopback address. The launcher answers `/__sim/metrics` to direct loopback requests only. The collector stops cleanly on SIGINT/SIGTERM, and on restart it continues the current hour, recording the restart as a boundary. @@ -48,6 +50,19 @@ boundary. - Only files inside `--dir` are ever removed. A failed write is counted and the collector carries on; the game never waits on telemetry. +### The series a spike needs + +The launcher exports four series for a spike (#2541), next to the world series from `live.json`: + +| Series | Type | What | +|---|---|---| +| `sim_world_viewers{world}` | gauge | viewer sockets open on the world at the launcher (its `live` map); 0 is a reading | +| `sim_launcher_event_loop_delay_p99_seconds` | gauge | the launcher process's own event-loop delay p99 since the previous scrape | +| `sim_site_worlds_created_total` | counter | worlds created on the site since the launcher started (sign-ups and operator creations) | +| `sim_host_disk_free_bytes`, `sim_host_disk_size_bytes` | gauge | free and total bytes of the disk the launcher data is on | + +Labels are the world slug only: never a Soul id, account id or address. + Size, measured with a synthetic scrape of 400 registered worlds (20 running): about 7 KB per scrape before gzip. At 15 s that is about 43 MB per day before gzip, well inside the cap. @@ -118,7 +133,7 @@ Each rule has three windows, which give it hysteresis: - **Clear window:** the condition must stay clear for this long before the rule resolves. The default is 60 s. - **Clear threshold** (optional): a firing rule only counts as clear once its value is back past this threshold. -A running world without a rule's series becomes `unknown` after the hold window, never healthy. A world that is not running is not judged, except for crash loops. Nothing is sent anywhere: delivery and dead-man alerting belong to #461, and a monitor on the same host cannot detect that host failing. +A running world without a rule's series becomes `unknown` after the hold window, never healthy. A world that is not running is not judged, except for crash loops. A firing rule reaches a person through the public health check ([below](#reaching-a-person)). Nothing is sent from the host itself: a monitor on the same host cannot report that host failing, and the outside check can. ### Rules @@ -134,9 +149,42 @@ A running world without a rule's series becomes `unknown` after the hold window, | `refusals` | world | any increase of `sim_world_refused_total` or `sim_world_shed_total` (fires at once; clears after 5 quiet minutes) | — | | `stale-live` | world | `live.json` older than 30 s (fires at once) | — | | `crash-loop` | any world | at least 3 non-planned restarts within 10 min (fires at once; clears after 10 min) | — | +| `viewers-high` | world | more than 1,000 viewer sockets on one world (fires at once). Information only: it never pages. | — | +| `launcher-loop-stall` | fleet | the launcher's own event-loop delay p99 > 0.25 s | 0.15 s | +| `worlds-created` | fleet | more than 10 worlds created in the last hour (fires at once). Counted across launcher restarts; creations before the collector's first scrape are not counted. | — | +| `disk-free` | fleet | under 20 % of the data disk free | — | | `host-memory` | fleet | host memory available < 1 GiB | 1.5 GiB | | `telemetry-gap` | collector | no answered scrape for more than 2.5 × the cadence. This covers failed scrapes, a silent collector seen by a later `alerts` run, and an empty store. | — | +Every rule pages except `refusals` and `viewers-high`, which are recorded in `alerts.jsonl` only. `refusals` fires on any refused request, which is the ceilings doing their job, not an incident. + +### Reaching a person + +With `--problems /alert-problems.json`, the collector rewrites that file after every scrape. It holds `{at, everyMs, problems}`, with one problem per firing rule that pages, coded `alert:` (for example `alert:disk-free`). + +- The launcher reads the file on every `/__sim/healthz`. A firing rule makes `hostOk` false and puts its code in `hostCodes` at once, and the code goes when the rule resolves. +- `deploy/host-health.mjs` (every 10 min) folds the same codes into `host-health.json` and logs an ALERT line in `host-health.log`. For an alert, healthz trusts the file it reads itself over this timer copy, so a resolved alert clears at once. +- A file older than 4 scrapes and at least 2 min means the collector died, and it shows as `alert:stale`. A clean stop (SIGTERM, `systemctl stop`) removes the file, so a stopped collector is never mistaken for a dead one. +- With no file (the collector not enabled), nothing changes. + +The outside check watches `hostOk` on the public healthz, and that is how the alert leaves the machine. Setting it up is a hands-on step outside this repository. + +The same health check also covers the hourly off-site backup. While `sim-worlds-backup-hourly@.timer` is enabled, a last run that did not succeed (a timeout, or a FAILED database, which exits 1) raises `backup-hourly:`, as the nightly run raises `backup:`. The owners' health page shows the failure, with the failed database from `backup-hourly-last.json` when the run wrote one. + +### The unit + +`deploy/sim-worlds-telemetry@.service` runs `telemetry.mjs collect --launcher-args --alerts --problems ...` as `simworlds`. It writes to `/srv/sim-worlds-data//telemetry` and to `alert-problems.json` beside it. `--launcher-args` takes the launcher's port and base from `SIM_WORLDS_ARGS` in `/etc/sim-worlds/.conf`, so the unit needs no port of its own. + +`install.sh` installs the unit on every release and never enables it. Enabling it is the maintainer's step, on the host: + +```bash +systemctl enable --now sim-worlds-telemetry@production +systemctl status sim-worlds-telemetry@production +cat /srv/sim-worlds-data/production/alert-problems.json # {"at": ..., "problems": []} when all is well +``` + +To stop it, run `systemctl disable --now sim-worlds-telemetry@production`. The clean stop removes `alert-problems.json`. + ### Changing the rules `rules.json` overrides rules by id: `{"rules": {"loop-stall": {"threshold": 0.5, "holdMs": 60000}, "host-memory": {"off": true}}}`. The keys an override can set are `threshold`, `clearAt`, `holdMs`, `clearMs` and `off`. An unknown id is refused. @@ -147,7 +195,7 @@ The launcher does not export process CPU, process RSS, cgroup CPU or memory pres ### Cost -With 400 registered worlds (20 running), evaluation after each scrape takes 40 ms at the median and at most 80 ms at the end of an hour. That is under 0.6 % of one core at the 15 s cadence. The state file is about 40 KB. +With 400 registered worlds (20 running), evaluation after each scrape takes 40 ms at the median and at most 80 ms at the end of an hour. That is under 0.6 % of one core at the 15 s cadence. The state file is about 40 KB. With the four spike rules (#2541), measured back to back against the rules before them over one synthetic hour (240 scrapes, two repeats each), the median was 40.8 ms against 39.0 ms and the maximum 76–79 ms against 73–80 ms. ## Cost and the context packet diff --git a/tools/world-launcher/deploy/host-health.mjs b/tools/world-launcher/deploy/host-health.mjs index 653645074..f9e28455c 100644 --- a/tools/world-launcher/deploy/host-health.mjs +++ b/tools/world-launcher/deploy/host-health.mjs @@ -6,13 +6,17 @@ // short problem `codes` for everyone, the numbers for direct (loopback) requests. Alert when disk free < 15% OR < 20 GB, // or MemAvailable < 1.5 GB; (t4-o1) when a sim-worlds@ unit's working set (cgroup memory.current minus inactive // file cache) reaches 90% of its MemoryMax, or its oom_kill count rose since the last run; and, only while a -// sim-worlds-backup@.timer is enabled, when that backup's last run did not succeed. -// No dependencies, no network. The off-box watcher (team/ops/sim-watch, Modal) reads `hostOk`/`hostCodes` from healthz and +// sim-worlds-backup@.timer is enabled, when that backup's last run did not succeed (#2541: likewise the hourly +// sim-worlds-backup-hourly@, code `backup-hourly:`); and (#2541) for each telemetry alert rule firing that +// pages, `alert:`, from /alert-problems.json (telemetry.mjs collect --problems), or `alert:stale` when +// the collector stopped writing that file without a clean stop. +// No dependencies (but telemetry/alerts.mjs, for the alerts file), no network. The off-box watcher (team/ops/sim-watch, Modal) reads `hostOk`/`hostCodes` from healthz and // DMs Gene. Exit 0 always (a failed check must not flap the unit). import { execFileSync } from 'node:child_process'; import { appendFileSync, existsSync, readdirSync, readFileSync, realpathSync, renameSync, statfsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { readAlertProblems } from '../telemetry/alerts.mjs'; // #2541 export function memAvailableMb(meminfo) { const m = /^MemAvailable:\s+(\d+)\s+kB/m.exec(meminfo); @@ -29,7 +33,7 @@ export function readCgroup(dir) { return { workingBytes: num('memory.current') - field(stat, 'inactive_file'), maxBytes: num('memory.max'), oomKills: field(events, 'oom_kill') }; } -export function evaluate({ diskTotalBytes, diskFreeBytes, memMb, units = {}, prevUnits = {}, backups = {}, worldProblems = [] }, t = {}) { +export function evaluate({ diskTotalBytes, diskFreeBytes, memMb, units = {}, prevUnits = {}, backups = {}, backupsHourly = {}, worldProblems = [], alertProblems = [] }, t = {}) { const minPct = t.minFreePct ?? 15, minGb = t.minFreeGb ?? 20, minMem = t.minMemMb ?? 1536, capPct = t.capPct ?? 90; const diskFreeGb = +(diskFreeBytes / 1e9).toFixed(1), diskFreePct = +(100 * diskFreeBytes / diskTotalBytes).toFixed(1); const problems = [], codes = new Set(); @@ -49,9 +53,13 @@ export function evaluate({ diskTotalBytes, diskFreeBytes, memMb, units = {}, pre for (const [stage, b] of Object.entries(backups)) { if (b.enabled && b.result && b.result !== 'success') flag(`backup:${stage}`, `sim-worlds-backup@${stage} last run: ${b.result}`); } + for (const [stage, b] of Object.entries(backupsHourly)) { // #2541: a timeout or a FAILED database fails the unit too + if (b.enabled && b.result && b.result !== 'success') flag(`backup-hourly:${stage}`, `sim-worlds-backup-hourly@${stage} last run: ${b.result}`); + } + for (const a of alertProblems) if (a?.code && a.text) flag(a.code, a.text); // lane L1: the launcher's guard (children.mjs problems(), via /world-problems.json): wedged, held, crashloop, bootSlow, halted for (const w of worldProblems) if (w?.code && w.text) flag(w.code, w.text); - return { ok: problems.length === 0, diskFreeGb, diskFreePct, memAvailableMb: memMb, problems, codes: [...codes], units: unitStatus, backups, + return { ok: problems.length === 0, diskFreeGb, diskFreePct, memAvailableMb: memMb, problems, codes: [...codes], units: unitStatus, backups, backupsHourly, thresholds: { minFreePct: minPct, minFreeGb: minGb, minMemMb: minMem, capPct } }; } @@ -66,16 +74,16 @@ function readUnits(slice = CGROUP_SLICE) { } /** Backups count only while their timer is enabled (Gene, 2026-10-19: scheduled backups are off until he wants them). */ -function readBackups(stages) { +function readBackups(stages, unit = 'sim-worlds-backup') { const backups = {}; const show = (unit, props) => Object.fromEntries(execFileSync('systemctl', ['show', unit, '-p', props], { encoding: 'utf8', timeout: 10_000 }) .trim().split('\n').map(l => l.split('='))); for (const stage of stages) { try { - const timer = show(`sim-worlds-backup@${stage}.timer`, 'UnitFileState'); + const timer = show(`${unit}@${stage}.timer`, 'UnitFileState'); if (timer.UnitFileState !== 'enabled') { backups[stage] = { enabled: false }; continue; } // lane DASH: when the last run started and ended (the owner's health page shows them; '' = never ran) - const s = show(`sim-worlds-backup@${stage}.service`, 'Result,ExecMainStartTimestamp,ExecMainExitTimestamp,ExecMainStatus'); + const s = show(`${unit}@${stage}.service`, 'Result,ExecMainStartTimestamp,ExecMainExitTimestamp,ExecMainStatus'); const iso = (v) => { const t = Date.parse(v ?? ''); return Number.isFinite(t) ? new Date(t).toISOString() : null; }; backups[stage] = { enabled: true, result: s.Result, startedAt: iso(s.ExecMainStartTimestamp), endedAt: iso(s.ExecMainExitTimestamp), exitStatus: s.ExecMainStatus === undefined ? null : Number(s.ExecMainStatus) }; } catch {} @@ -101,7 +109,7 @@ function main(argv) { const s = statfsSync(diskPath); const units = readUnits(); status = evaluate({ diskTotalBytes: s.blocks * s.bsize, diskFreeBytes: s.bavail * s.bsize, memMb: memAvailableMb(readFileSync(`${process.env.SIM_PROC_ROOT ?? ''}/proc/meminfo`, 'utf8')), - units, prevUnits: prev?.units ?? {}, backups: readBackups(Object.keys(units)), worldProblems: readWorldProblems(dataDir) }, + units, prevUnits: prev?.units ?? {}, backups: readBackups(Object.keys(units)), backupsHourly: readBackups(Object.keys(units), 'sim-worlds-backup-hourly'), worldProblems: readWorldProblems(dataDir), alertProblems: readAlertProblems(dataDir) }, { minFreePct: Number(opt('min-free-pct', 15)), minFreeGb: Number(opt('min-free-gb', 20)), minMemMb: Number(opt('min-mem-mb', 1536)), capPct: Number(opt('cap-pct', 90)) }); } catch (e) { status = { ok: false, problems: [`check failed: ${e.message}`], codes: ['check'] }; } status = { ...status, at: new Date().toISOString(), diskPath }; diff --git a/tools/world-launcher/deploy/install.sh b/tools/world-launcher/deploy/install.sh index 9f13bbd39..93cc87bf8 100755 --- a/tools/world-launcher/deploy/install.sh +++ b/tools/world-launcher/deploy/install.sh @@ -95,5 +95,8 @@ install -m 644 "$HERE/sim-worlds-backup@.service" /etc/systemd/system/sim-worlds install -m 644 "$HERE/sim-worlds-backup@.timer" /etc/systemd/system/sim-worlds-backup@.timer install -m 644 "$HERE/sim-worlds-health@.service" /etc/systemd/system/sim-worlds-health@.service install -m 644 "$HERE/sim-worlds-health@.timer" /etc/systemd/system/sim-worlds-health@.timer +# #2541: the telemetry collector and its alerts. Installed, never enabled here (this runs on every release): enabling is +# the maintainer's step, docs/operations/telemetry.md. +install -m 644 "$HERE/sim-worlds-telemetry@.service" /etc/systemd/system/sim-worlds-telemetry@.service systemctl daemon-reload -echo "installed: user simworlds, /srv/sim-worlds{,-data}, /etc/sim-worlds/{staging,production}.conf, sim-worlds@.service, sim-worlds-{backup,health}@.{service,timer} (nothing started)" +echo "installed: user simworlds, /srv/sim-worlds{,-data}, /etc/sim-worlds/{staging,production}.conf, sim-worlds@.service, sim-worlds-{backup,health}@.{service,timer}, sim-worlds-telemetry@.service (nothing started)" diff --git a/tools/world-launcher/deploy/sim-worlds-telemetry@.service b/tools/world-launcher/deploy/sim-worlds-telemetry@.service new file mode 100644 index 000000000..627d1e2a5 --- /dev/null +++ b/tools/world-launcher/deploy/sim-worlds-telemetry@.service @@ -0,0 +1,27 @@ +# #2541: the telemetry collector for sim-worlds@: scrapes the launcher's loopback /__sim/metrics every 15 s into +# /srv/sim-worlds-data//telemetry, evaluates the alert rules after each scrape, and writes the firing rules that +# page to alert-problems.json, which /__sim/healthz folds into hostOk/hostCodes (alert:) for the outside check. +# install.sh installs it and never enables it. Enabling is the maintainer's step (docs/operations/telemetry.md): +# systemctl enable --now sim-worlds-telemetry@production +[Unit] +Description=Telemetry collector and alerts for the sim-worlds@%i launcher +After=sim-worlds@%i.service + +[Service] +Type=simple +User=simworlds +Group=simworlds +Nice=10 +EnvironmentFile=/etc/sim-worlds/%i.conf +ExecStart=/usr/bin/node --no-warnings /srv/sim-worlds/%i/tools/world-launcher/telemetry.mjs collect --launcher-args --dir /srv/sim-worlds-data/%i/telemetry --alerts --problems /srv/sim-worlds-data/%i/alert-problems.json +Restart=always +RestartSec=10 +MemoryMax=512M +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/srv/sim-worlds-data/%i +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/tools/world-launcher/health-dash-backup.test.mjs b/tools/world-launcher/health-dash-backup.test.mjs index e702242b8..6a35cfc76 100644 --- a/tools/world-launcher/health-dash-backup.test.mjs +++ b/tools/world-launcher/health-dash-backup.test.mjs @@ -86,3 +86,17 @@ test('the page says when the first record comes instead of "none"', () => { assert.doesNotMatch(page, /tonight\\'s/); new Function(page.split('')[0].replace(/^\s*import .*$/gm, '')); // the inline script still parses }); + +test('#2541: a failed hourly backup, a FAILED database or a timeout, is an alert on the page', async () => { + const f = setup(); + writeFileSync(join(f.dataDir, 'backup-hourly-last.json'), JSON.stringify({ at: isoAgo(0.5), ok: false, dbsOk: 6, dbs: 7, failed: ['worlds/alpha/sim.db: database disk image is malformed'] })); + writeFileSync(join(f.root, 'legacy', 'backup-hourly-last.json'), JSON.stringify({ at: isoAgo(2), ok: true, dbsOk: 7, dbs: 7 })); + const saved = host.backupsHourly; + host.backupsHourly = { legacy: { enabled: true, result: 'timeout' }, stale: { enabled: false, result: 'timeout' } }; + try { + const texts = (await dashFor(f).json()).alerts.map((a) => a.text); + assert.ok(texts.includes('production hourly backup failed: worlds/alpha/sim.db: database disk image is malformed'), texts.join(' | ')); + assert.ok(texts.includes('legacy hourly backup failed: timeout'), texts.join(' | ')); + assert.ok(!texts.some((t) => /^stale hourly/.test(t)), 'a disabled hourly timer is not judged'); + } finally { host.backupsHourly = saved; } +}); diff --git a/tools/world-launcher/health-dash.mjs b/tools/world-launcher/health-dash.mjs index de95a6f51..f09cda6e7 100644 --- a/tools/world-launcher/health-dash.mjs +++ b/tools/world-launcher/health-dash.mjs @@ -266,7 +266,9 @@ export function healthDash(o) { const st = stages().map((s) => { const k = s.self ? keeper : readJson(join(s.dir, 'keeper.json')); return { stage: s.stage, self: s.self, build: s.self ? o.build : k?.active?.build ?? null, since: k?.active?.since ?? null, - backup: lastBackup(s.dir, s.stage, host?.backups?.[s.stage], backupRoot), timer: host?.backups?.[s.stage] ?? null }; + backup: lastBackup(s.dir, s.stage, host?.backups?.[s.stage], backupRoot), timer: host?.backups?.[s.stage] ?? null, + // #2541: the hourly off-site run's own record and unit (a timeout leaves no record, only the unit's result) + hourly: readJson(join(s.dir, 'backup-hourly-last.json')), hourlyTimer: host?.backupsHourly?.[s.stage] ?? null }; }); const mem = memory(); const errs = o.errors.summary({ limit: 20 }); @@ -308,6 +310,8 @@ export function healthDash(o) { } for (const s of d.stages) { if (s.backup && s.backup.ok === false) bad(`${s.stage} backup failed${s.backup.error ? `: ${String(s.backup.error).slice(0, 120)}` : ''}`); + const hu = s.hourlyTimer?.enabled && s.hourlyTimer.result && s.hourlyTimer.result !== 'success' ? s.hourlyTimer.result : null, hr = s.hourly?.ok === false ? s.hourly.error ?? s.hourly.failed?.[0] ?? 'see backup-hourly-last.json' : null; + if (hu || hr) bad(`${s.stage} hourly backup failed: ${String(hr ?? hu).slice(0, 120)}${hu && hr ? ` (unit: ${hu})` : ''}`); // No backup seen at all (a new stage before its first night) is not an alert: the page says when the first record comes. if (s.timer?.enabled && s.backup?.at && ago(s.backup.at) > 26) warn(`${s.stage}: no backup in ${Number.isFinite(ago(s.backup.at)) ? `${Math.round(ago(s.backup.at))} h` : 'its record'}`); } diff --git a/tools/world-launcher/host-health.test.mjs b/tools/world-launcher/host-health.test.mjs index 5d3b86d70..cd04af19f 100644 --- a/tools/world-launcher/host-health.test.mjs +++ b/tools/world-launcher/host-health.test.mjs @@ -59,3 +59,18 @@ test('#1150: the timer run on a halted world writes its ALERT line, and RECOVERE const after = run([]); assert.equal(after.length, 2); assert.match(after[1], / RECOVERED disk /); }); + +test('#2541: a failed hourly backup and a firing telemetry alert set short codes; a stale alerts file is alert:stale', async (t) => { + const { evaluate } = await import('./deploy/host-health.mjs'); const { readAlertProblems } = await import('./telemetry/alerts.mjs'); + const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs'); const { join } = await import('node:path'); const { tmpdir } = await import('node:os'); + const base = { diskTotalBytes: 301 * GB, diskFreeBytes: 150 * GB, memMb: 8000 }; + const r = evaluate({ ...base, backupsHourly: { production: { enabled: true, result: 'timeout' }, staging: { enabled: false, result: 'exit-code' } }, alertProblems: [{ code: 'alert:disk-free', text: 'share of the data disk free: over its threshold 0.2' }] }); + assert.deepEqual(r.codes, ['backup-hourly:production', 'alert:disk-free']); + assert.match(r.problems.join(), /sim-worlds-backup-hourly@production last run: timeout/); + const d = mkdtempSync(join(tmpdir(), 'alert-problems-')); t.after(() => rmSync(d, { recursive: true, force: true })); + const now = Date.parse('2026-11-16T08:00:00Z'); + assert.deepEqual(readAlertProblems(d, now), [], 'no file: the collector is not enabled'); + writeFileSync(join(d, 'alert-problems.json'), JSON.stringify({ at: new Date(now - 20_000).toISOString(), everyMs: 15_000, problems: [{ code: 'alert:worlds-created', text: 'x' }, { code: 'disk', text: 'not an alert code' }] })); + assert.deepEqual(readAlertProblems(d, now).map((p) => p.code), ['alert:worlds-created']); + assert.deepEqual(readAlertProblems(d, now + 5 * 60_000).map((p) => p.code), ['alert:stale']); +}); diff --git a/tools/world-launcher/launcher.mjs b/tools/world-launcher/launcher.mjs index a4aeae9ba..957831248 100644 --- a/tools/world-launcher/launcher.mjs +++ b/tools/world-launcher/launcher.mjs @@ -106,7 +106,8 @@ import { readGlitches } from './glitches.mjs'; import { usageRollup } from './usage.mjs'; import { ownerOverview } from './owner-overview.mjs'; import { connect } from 'node:net'; -import { createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, statfsSync, writeFileSync } from 'node:fs'; +import { monitorEventLoopDelay } from 'node:perf_hooks'; import { brotliCompressSync, brotliDecompressSync, gunzipSync, gzipSync, constants as zlibConstants } from 'node:zlib'; import { dirname, extname, join, normalize, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -121,6 +122,7 @@ import { EXISTING_WORLDS, decorateCommand, memAvailableMb, productionSettings } import { makeHandoff } from './handoff.mjs'; import { MicroCache, launcherFlood, listenTimeoutMs, refusalBody, requestClass, short, soulCeilings } from './flood.mjs'; import { readLive, serveMetrics } from './metrics.mjs'; // lane L7 +import { readAlertProblems } from './telemetry/alerts.mjs'; // #2541 import { defaultSiteLog } from '../../apps/server/src/moderation/store.ts'; import { withWorldMeta } from '../../apps/server/src/inline-config.ts'; import { archiveWorld, archivedSlugs, audit, listArchive, readAudit, restoreWorld, tokenLabel } from './worldops.mjs'; @@ -298,6 +300,13 @@ export async function openLauncher(cfg = {}) { const waitHandBack = async (slug, e) => (e.code === 'handed-back' || canaryRun?.slug === slug) && (await canaryRun?.passed) === false && handedBack(); const adopting = Array.isArray(cfg.adopt) ? cfg.adopt : null; const createLimit = cfg.createLimit ?? Infinity, made = new Map(); + // #2541: what a spike needs exported (metrics.mjs): worlds created here since start, and this process's own loop delay + let worldsCreated = 0; const loopDelay = monitorEventLoopDelay({ resolution: 20 }); loopDelay.enable(); + const launcherGauges = () => { + const p99 = loopDelay.count ? loopDelay.percentile(99) / 1e6 : null; loopDelay.reset(); // ns → ms, since the previous scrape + let disk = null; try { const s = statfsSync(dataDir); disk = { free: s.bavail * s.bsize, size: s.blocks * s.bsize }; } catch { /* left out: "no data", not 0 */ } + return { loopDelayP99Ms: p99, worldsCreated, diskFreeBytes: disk?.free, diskSizeBytes: disk?.size }; + }; /** Open WebSocket (upgraded) connections per world: a world someone is watching never goes to sleep. */ const live = new Map(); /** HTTP requests in flight per world (a god's long-poll listen): such a world is not idle either. */ @@ -551,7 +560,7 @@ export async function openLauncher(cfg = {}) { if (!/^[A-Za-z0-9_:.-]{3,96}$/.test(owner)) return [400, { ok: false, code: 'invalid', message: 'owner (account id) or ownerOf (slug of a world it owns) is required' }]; if (!accountExists(owner)) return [404, { ok: false, code: 'no-account', message: `no account ${owner} in this launcher's account registry` }]; const run = creating.then(() => createWorld(body, undefined, { owner })); creating = run.catch(() => {}); - return run; + return run.then((out) => { if (out[0] === 201) worldsCreated++; return out; }); } // ---------- operator: forks of the old worlds (lane FORKS). Only FORK_SLUGS, only from this machine, only onto a @@ -1332,7 +1341,7 @@ export async function openLauncher(cfg = {}) { const recent = (made.get(who) ?? []).filter((t) => Date.now() - t < 10 * 60_000); if (recent.length >= createLimit) { json(res, 429, { ok: false, code: 'slow-down', message: `at most ${createLimit} new worlds per 10 minutes from one place` }, { 'retry-after': '120' }); return; } const run = creating.then(() => createWorld(body, req.headers.authorization)); creating = run.catch(() => {}); - const out = await run; if (out[0] === 201) made.set(who, [...recent, Date.now()]); + const out = await run; if (out[0] === 201) { made.set(who, [...recent, Date.now()]); worldsCreated++; } json(res, out[0], out[1], out[1].retry_after ? { 'retry-after': String(out[1].retry_after) } : {}); return; } if (path === '/__sim/op/create-world' && req.method === 'POST') { @@ -1464,7 +1473,7 @@ export async function openLauncher(cfg = {}) { json(res, 200, { origin, publicOrigin: prod.publicOrigin, devAuth: prod.devAuth, dataDir, range, maxWorlds: MAX_WORLDS, maxRunning: prod.maxRunning, minFreeMb: prod.minFreeMb, memAvailableMb: memAvailableMb(), children: registry.worlds.map((w) => ({ slug: w.slug, viewers: live.get(w.slug) ?? 0, ...(unloaded(w.slug) && pool.status(w.slug).status !== 'running' ? { unloaded: horizons.get(w.slug) } : {}), heapMb: w.fork ? (w.worldHeapMb ?? cfg.forkHeapMb ?? 1536) : worldHeapMb(w, prod.worldHeapMb), ...pool.status(w.slug), state: stateOf(w.slug) })), worldsMemMb: prod.worldsMemMb, memory: pool.memUse() }); return; } // lane L7: Prometheus text for the worlds and the launcher (LOAD-CONTRACT §5), loopback only - if (path === '/__sim/metrics' && req.method === 'GET') { if (!direct(req)) { json(res, 404, { ok: false, code: 'not-found' }); return; } serveMetrics(res, { launcher: { build: BUILD, maxRunning: prod.maxRunning, maxWorlds: MAX_WORLDS, memAvailableMb: memAvailableMb() }, clientErrors: errors, worlds: registry.worlds.map((w) => ({ slug: w.slug, ...pool.status(w.slug), refused: ceilings.refused(w.slug), listenHeld: ceilings.held(w.slug), ...(readStats.has(w.slug) ? { read: { ...readStats.get(w.slug) } } : {}), live: readLive(registry.worldDir(w.slug)) })) }); return; } + if (path === '/__sim/metrics' && req.method === 'GET') { if (!direct(req)) { json(res, 404, { ok: false, code: 'not-found' }); return; } serveMetrics(res, { launcher: { build: BUILD, maxRunning: prod.maxRunning, maxWorlds: MAX_WORLDS, memAvailableMb: memAvailableMb(), ...launcherGauges() }, clientErrors: errors, worlds: registry.worlds.map((w) => ({ slug: w.slug, ...pool.status(w.slug), viewers: live.get(w.slug) ?? 0, refused: ceilings.refused(w.slug), listenHeld: ceilings.held(w.slug), ...(readStats.has(w.slug) ? { read: { ...readStats.get(w.slug) } } : {}), live: readLive(registry.worldDir(w.slug)) })) }); return; } // #1150: every world's recoveries and halts from its own store, asleep or not; ?world= adds that world's records. Loopback only. if (path === '/__sim/glitches' && req.method === 'GET') { if (!direct(req)) { json(res, 404, { ok: false, code: 'not-found' }); return; } @@ -1477,8 +1486,10 @@ export async function openLauncher(cfg = {}) { const running = registry.worlds.filter((w) => pool.status(w.slug).status === 'running').length; const host = hostHealth(dataDir); // deploy/host-health.mjs (timer): `hostOk` (+ short `hostCodes` when not ok) public, the numbers loopback-only // lane L1: the guard's findings now, not at the timer's next run: wedged, held, crashloop, bootSlow, halted (codes public, texts loopback) - const worldProblems = pool.problems(), wcodes = [...new Set(worldProblems.map((x) => x.code))]; - const hostOk = (host ? host.ok : true) && !wcodes.length, hostCodes = [...new Set([...(host && !host.ok ? host.codes ?? [] : []), ...wcodes])]; + const worldProblems = pool.problems(), wcodes = [...new Set([...worldProblems.map((x) => x.code), ...readAlertProblems(dataDir).map((x) => x.code)])]; // #2541: firing telemetry alerts too + // the alerts file read now supersedes the timer's copy of it, so a resolved alert clears at once + const hc = host && !host.ok ? (host.codes ?? []).filter((c) => !c.startsWith('alert:')) : []; + const hostOk = (host ? host.ok || (host.codes?.length > 0 && !hc.length) : true) && !wcodes.length, hostCodes = [...new Set([...hc, ...wcodes])]; json(res, 200, { ok: true, service: 'world-launcher', build: BUILD, auth: auth ? 'clerk' : 'dev', worlds: registry.worlds.length, running, maxWorlds: MAX_WORLDS, maxRunning: prod.maxRunning, ...(host || wcodes.length ? { hostOk, ...(hostOk ? {} : { hostCodes }) } : {}), ...(direct(req) && worldProblems.length ? { worldProblems } : {}), clientErrorsNew: errors.newSinceRelease(), clientErrorsRate10m: errors.rate10m(), refused: refusedByWorld(), ...(direct(req) ? { memAvailableMb: memAvailableMb(), loads: pool.loadStats(), ...(host ? { host } : {}) } : {}) }); return; } const to = redirects?.target(req, path, url); @@ -1771,7 +1782,7 @@ export async function openLauncher(cfg = {}) { const live = () => [...sockets].filter((s) => !s.simUpgraded); if (gentle) for (const t0 = Date.now(); live().length && Date.now() - t0 < drainMs && (Date.now() - t0 < idleMs || live().some((s) => s.simBusy));) await new Promise((r) => setTimeout(r, 20)); const done = new Promise((r) => server.close(() => r())); // (Node's close() also cuts idle connections: only now) - clearInterval(problemsTimer); for (const s of sockets) s.destroy(); await pool.closeAll(); await done; + clearInterval(problemsTimer); loopDelay.disable(); for (const s of sockets) s.destroy(); await pool.closeAll(); await done; }, }; } diff --git a/tools/world-launcher/metrics.mjs b/tools/world-launcher/metrics.mjs index 9fd13ffff..2c82c6e4d 100644 --- a/tools/world-launcher/metrics.mjs +++ b/tools/world-launcher/metrics.mjs @@ -64,7 +64,10 @@ function registry() { * listenHeld = { served, refused, gone, ended, seconds } (lane srv-shed) → sim_world_listen_held_total{world,outcome}, * sim_world_listen_held_seconds_sum/_count{world} * (status from the launcher's pool; live = readLive(world.dataDir) or null) - * launcher: {build?, maxRunning?, maxWorlds?, memAvailableMb?} + * viewers = the launcher's open viewer sockets for the world (its `live` map) → sim_world_viewers{world} (#2541) + * launcher: {build?, maxRunning?, maxWorlds?, memAvailableMb?, loopDelayP99Ms?, worldsCreated?, diskFreeBytes?, diskSizeBytes?} + * (#2541: the launcher's own event-loop delay p99 since the last scrape, worlds created since it started, and the + * free and total bytes of the disk its data is on) * clientErrors: the clientErrors() table (client-errors.mjs) or null */ export function metricsText({ worlds = [], launcher = {}, clientErrors = null, now = Date.now() } = {}) { @@ -76,6 +79,10 @@ export function metricsText({ worlds = [], launcher = {}, clientErrors = null, n g('sim_launcher_max_running', 'Cap on running worlds.', {}, launcher.maxRunning); g('sim_launcher_max_worlds', 'Cap on worlds.', {}, launcher.maxWorlds); g('sim_host_mem_available_bytes', 'MemAvailable of the host as the launcher sees it.', {}, num(launcher.memAvailableMb) === null ? null : launcher.memAvailableMb * 1048576); + g('sim_launcher_event_loop_delay_p99_seconds', 'Event loop delay p99 of the launcher itself since the previous scrape.', {}, num(launcher.loopDelayP99Ms) === null ? null : launcher.loopDelayP99Ms / 1000); + c('sim_site_worlds_created_total', 'Worlds created on this site since the launcher started.', {}, launcher.worldsCreated); + g('sim_host_disk_free_bytes', 'Free bytes (for an unprivileged user) on the disk the launcher data is on.', {}, launcher.diskFreeBytes); + g('sim_host_disk_size_bytes', 'Total bytes of the disk the launcher data is on.', {}, launcher.diskSizeBytes); g('sim_metrics_scrape_timestamp_seconds', 'When this scrape was rendered (a dead-man reads it).', {}, now / 1000); for (const w of worlds) { @@ -98,6 +105,7 @@ export function metricsText({ worlds = [], launcher = {}, clientErrors = null, n r.add('sim_world_listen_held_seconds', 'summary', 'Seconds listen/wait long-polls spent held for a slot.', L, lh.seconds, '_sum'); r.add('sim_world_listen_held_seconds', 'summary', 'Seconds listen/wait long-polls spent held for a slot.', L, lh.ended, '_count'); } + g('sim_world_viewers', 'Viewer sockets open on the world at the launcher.', L, w.viewers); g('sim_world_last_boot_seconds', 'Seconds the world\'s last boot took (start to healthy).', L, num(w.lastBootMs) === null ? null : w.lastBootMs / 1000); // Stage B (lane B-port): C2 reads the launcher sent to the world's read workers, and those it then asked main (a // worker error, a 5xx or no answer in 2 s); only for worlds with read workers on diff --git a/tools/world-launcher/telemetry-alerts.test.mjs b/tools/world-launcher/telemetry-alerts.test.mjs index 81cc80ee3..26299db3c 100644 --- a/tools/world-launcher/telemetry-alerts.test.mjs +++ b/tools/world-launcher/telemetry-alerts.test.mjs @@ -15,6 +15,8 @@ const live = (t, o = {}) => ({ v: 1, pid: 1, at: o.stale ? t - 60 * S : t - 2 * tick: { world: Math.round((t - T0) / 50), service5s: 1, p99Ms: o.p99 ?? 30, lateTotal: o.late ?? 0 }, heap: { usedMb: o.heap ?? 500, limitMb: 2048, gcPct10s: o.gc ?? 1, gcSecondsTotal: 0 }, level: 'green', queue: { depth: 0, oldestMs: o.oldest ?? 0 }, souls: { awake: 3, waiting: 0, connected: 3 }, shed: { reads: o.shed ?? 0, acts: 0 } }); const world = (slug, t, o = {}) => ({ slug, status: o.status ?? 'running', restarts: o.restarts ?? 0, refused: o.refused, live: o.nolive ? null : live(t, o) }); +/** #2541: the launcher's own series, quiet (a launcher of this release exports them). */ +const LAUNCHER = { loopDelayP99Ms: 5, worldsCreated: 0, diskFreeBytes: 200e9, diskSizeBytes: 300e9 }; const fetchOf = (body, status = 200) => async () => ({ ok: status === 200, status, text: async () => body }); /** A collector on a fake clock. scene(t) → [world...] or 'fail' (HTTP 500) or 'silent' (no scrape at all). */ @@ -23,7 +25,7 @@ async function run(dir, scene, { from = 0, to, eachStep = false } = {}) { for (let s = from; s < to; s += EVERY / S) { const t = T0 + s * S, w = scene(s); if (w === 'silent') continue; - const body = w === 'fail' ? '' : metricsText({ now: t, launcher: { build: 'test', maxRunning: 10, memAvailableMb: 8192 }, worlds: w }); + const body = w === 'fail' ? '' : metricsText({ now: t, launcher: { build: 'test', maxRunning: 10, memAvailableMb: 8192, ...LAUNCHER }, worlds: w }); await scrapeOnce(store, 'http://127.0.0.1:1/__sim/metrics', { now: () => t, fetchImpl: fetchOf(body, w === 'fail' ? 500 : 200) }); if (eachStep) events.push(...catchUp(dir, { everyMs: EVERY, now: t }).events); } diff --git a/tools/world-launcher/telemetry-spike.test.mjs b/tools/world-launcher/telemetry-spike.test.mjs new file mode 100644 index 000000000..a2a54abc8 --- /dev/null +++ b/tools/world-launcher/telemetry-spike.test.mjs @@ -0,0 +1,148 @@ +// #2541 (Scaling H5): the numbers a spike needs are exported, alerted on, and reach a person through /__sim/healthz. +// A1: each new series and each new rule (fire and resolve), through the real collector path on a fake clock. +// A2: a firing rule that pages makes /__sim/healthz report hostOk: false with its code; resolving clears it. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { metricsText } from './metrics.mjs'; +import { metricsUrlFromArgs, openStore, scrapeOnce } from './telemetry.mjs'; +import { catchUp, problemsOf, rulesWith } from './telemetry/alerts.mjs'; +import { openLauncher } from './launcher.mjs'; +import { testPort } from '../../apps/server/test/test-port.ts'; + +const ROOT = new URL('../../', import.meta.url).pathname; +const tmp = (t) => { mkdirSync(join(ROOT, '.local'), { recursive: true }); const d = mkdtempSync(join(ROOT, '.local', 'spike-')); t.after(() => rmSync(d, { recursive: true, force: true })); return d; }; +const S = 1000, MIN = 60 * S, EVERY = 15 * S, T0 = Date.UTC(2026, 10, 16, 0, 0, 0), GB = 1e9; +const val = (text, sel) => { const line = text.split('\n').find((l) => l.startsWith(`${sel} `)); return line === undefined ? undefined : Number(line.slice(sel.length + 1)); }; +const live = (t) => ({ v: 1, pid: 1, at: t - 2 * S, phase: 'serving', loop: { lastTurnAgeMs: 10, delayP99Ms: 20, elu: 0.3 }, tick: { world: 1, service5s: 1, p99Ms: 30, lateTotal: 0 }, + heap: { usedMb: 500, limitMb: 2048, gcPct10s: 1 }, level: 'green', queue: { depth: 0, oldestMs: 0 }, souls: { awake: 1, waiting: 0, connected: 1 }, shed: {} }); +const QUIET = { loopDelayP99Ms: 5, worldsCreated: 0, diskFreeBytes: 200 * GB, diskSizeBytes: 300 * GB }; + +/** scene(s) → {launcher?: {...}, viewers?: n}; one scrape every 15 s, alerts evaluated after each (as `collect --alerts`). */ +async function run(dir, scene, { to, problems = null } = {}) { + const store = openStore(dir, { everyMs: EVERY }), events = []; + for (let s = 0; s < to; s += EVERY / S) { + const t = T0 + s * S, x = scene(s); + const body = metricsText({ now: t, launcher: { build: 'test', memAvailableMb: 8192, ...QUIET, ...x.launcher }, worlds: [{ slug: 'a', status: 'running', restarts: 0, viewers: x.viewers ?? 3, live: live(t) }] }); + await scrapeOnce(store, 'http://127.0.0.1:1/__sim/metrics', { now: () => t, fetchImpl: async () => ({ ok: true, status: 200, text: async () => body }) }); + events.push(...catchUp(dir, { everyMs: EVERY, now: t, problems }).events.map((e) => ({ ...e, s }))); + } + store.saveState(); + return events; +} +const of = (ev, rule) => ev.filter((e) => e.rule === rule).map((e) => [e.to, e.s]); + +test('A1 series: viewers per world, the launcher\'s loop delay, worlds created, disk free and size; labels are the world slug only', () => { + const t = metricsText({ now: T0, launcher: { loopDelayP99Ms: 312, worldsCreated: 7, diskFreeBytes: 41 * GB, diskSizeBytes: 300 * GB }, + worlds: [{ slug: 'eden-2026', status: 'running', viewers: 1204 }, { slug: 'quiet', status: 'asleep', viewers: 0 }] }); + assert.equal(val(t, 'sim_world_viewers{world="eden-2026"}'), 1204); + assert.equal(val(t, 'sim_world_viewers{world="quiet"}'), 0, 'zero viewers is a reading, not missing'); + assert.equal(val(t, 'sim_launcher_event_loop_delay_p99_seconds'), 0.312); + assert.equal(val(t, 'sim_site_worlds_created_total'), 7); + assert.match(t, /# TYPE sim_site_worlds_created_total counter/); + assert.equal(val(t, 'sim_host_disk_free_bytes'), 41 * GB); + assert.equal(val(t, 'sim_host_disk_size_bytes'), 300 * GB); + const bare = metricsText({ now: T0, worlds: [{ slug: 'eden-2026', status: 'running' }] }); + for (const n of ['sim_world_viewers{world="eden-2026"}', 'sim_launcher_event_loop_delay_p99_seconds', 'sim_site_worlds_created_total', 'sim_host_disk_free_bytes']) assert.equal(val(bare, n), undefined, `${n}: not reported, left out`); + for (const l of t.split('\n').filter((x) => /^sim_(world_viewers|launcher_event|site_|host_disk)/.test(x))) assert.match(l, /^[a-z0-9_]+(\{world="[a-z0-9-]+"\})? /, l); +}); + +test('A1 rules: the launcher\'s loop stall fires within 60 s and resolves below 0.15 s', async (t) => { + const ev = await run(tmp(t), (s) => ({ launcher: { loopDelayP99Ms: s >= 300 && s < 600 ? 400 : s >= 600 && s < 700 ? 200 : 5 } }), { to: 900 }); + const got = of(ev, 'launcher-loop-stall'); + assert.deepEqual(got.map((x) => x[0]), ['firing', 'resolved']); + assert.ok(got[0][1] - 300 <= 60, `fired at ${got[0][1]} s`); + assert.ok(got[1][1] >= 700, 'hysteresis: 0.2 s is still over the 0.15 s clear threshold'); +}); + +test('A1 rules: more than 10 worlds created in an hour fires, across a launcher restart, and resolves when the hour has passed', async (t) => { + // 6 worlds in the first 5 min, a restart (the counter starts again at 0), 5 more by 10 min: 11 in the hour + const created = (s) => (s < 300 ? Math.floor(s / 45) : s < 315 ? 0 : Math.min(5, Math.floor((s - 315) / 60) + 1)); + const ev = await run(tmp(t), (s) => ({ launcher: { worldsCreated: created(s) } }), { to: 80 * 60 }); + const got = of(ev, 'worlds-created'); + assert.deepEqual(got.map((x) => x[0]), ['firing', 'resolved']); + assert.ok(got[0][1] <= 600, `fired at ${got[0][1]} s`); + assert.ok(got[1][1] > 3600 && got[1][1] <= 3600 + 300 + 120, `resolved at ${got[1][1]} s, once the first creations left the hour`); + const ten = await run(tmp(t), (s) => ({ launcher: { worldsCreated: Math.min(10, Math.floor(s / 30)) } }), { to: 900 }); + assert.deepEqual(of(ten, 'worlds-created'), [], 'ten in an hour is not more than ten'); +}); + +test('A1 rules: disk free under 20 % fires and resolves; viewers over 1,000 on one world is recorded and never pages', async (t) => { + const dir = tmp(t), problems = join(dir, 'alert-problems.json'), seen = []; + const scene = (s) => ({ launcher: { diskFreeBytes: (s >= 120 && s < 400 ? 0.15 : 0.4) * 300 * GB }, viewers: s >= 200 && s < 500 ? 1500 : 40 }); + const store = openStore(dir, { everyMs: EVERY }), ev = []; + for (let s = 0; s < 800; s += 15) { // the same path as run(), reading the problems file at each step + const t1 = T0 + s * S, x = scene(s); + const body = metricsText({ now: t1, launcher: { ...QUIET, ...x.launcher }, worlds: [{ slug: 'a', status: 'running', viewers: x.viewers, live: live(t1) }] }); + await scrapeOnce(store, 'http://127.0.0.1:1/__sim/metrics', { now: () => t1, fetchImpl: async () => ({ ok: true, status: 200, text: async () => body }) }); + ev.push(...catchUp(dir, { everyMs: EVERY, now: t1, problems }).events.map((e) => ({ ...e, s }))); + seen.push(JSON.parse(readFileSync(problems, 'utf8')).problems.map((p) => p.code).join(',')); + } + assert.deepEqual(of(ev, 'disk-free').map((x) => x[0]), ['firing', 'resolved']); + const v = ev.filter((e) => e.rule === 'viewers-high'); + assert.deepEqual(v.map((e) => [e.to, e.world]), [['firing', 'a'], ['resolved', 'a']]); + const lines = readFileSync(join(dir, 'alerts.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l)); + assert.ok(lines.some((l) => l.rule === 'viewers-high' && l.to === 'firing' && l.value === 1500), 'in alerts.jsonl'); + assert.ok(seen.some((c) => c === 'alert:disk-free'), 'disk-free pages'); + assert.ok(!seen.some((c) => c.includes('viewers')), 'viewers-high never pages'); + assert.equal(seen.at(-1), '', 'resolved: no codes left'); +}); + +test('A1 rules: refusals stay information; every other rule pages', () => { + const pages = rulesWith().filter((r) => r.page !== false).map((r) => r.id); + assert.ok(!pages.includes('viewers-high') && !pages.includes('refusals')); + for (const id of ['launcher-loop-stall', 'worlds-created', 'disk-free', 'loop-stall', 'host-memory', 'telemetry-gap']) assert.ok(pages.includes(id), id); + const st = { keys: { 'disk-free|': { state: 'firing' }, 'viewers-high|a': { state: 'firing' }, 'loop-stall|a': { state: 'unknown' } } }; + assert.deepEqual(problemsOf(st, rulesWith()).map((p) => p.code), ['alert:disk-free'], 'only firing rules that page; unknown is not firing'); +}); + +test('the unit\'s collector finds the launcher from its own arguments', () => { + assert.equal(metricsUrlFromArgs('--base /worlds --port 4620 --range 4621-4659 --data /srv/x'), 'http://127.0.0.1:4620/worlds/__sim/metrics'); + assert.equal(metricsUrlFromArgs('--port 4600'), 'http://127.0.0.1:4600/__sim/metrics'); + assert.throws(() => metricsUrlFromArgs('--base /worlds'), /no --port/); +}); + +test('A2: a firing rule makes /__sim/healthz report hostOk false with its code; resolving clears it at once; the launcher exports the series', async (t) => { + const dir = tmp(t), P0 = await testPort(4), P = P0 + 3, base = `http://127.0.0.1:${P}`; + const templatesDir = join(dir, 'templates'); mkdirSync(templatesDir); + const webDir = join(dir, 'web'); mkdirSync(webDir); writeFileSync(join(webDir, 'index.html'), ''); + const fake = new URL('./fake-world.mjs', import.meta.url).pathname, dataDir = join(dir, 'data'); + const command = (w, port) => ({ cmd: process.execPath, args: [fake], env: { ...process.env, WORLD_KEY: w.slug, PORT: String(port) } }); + const L = await openLauncher({ port: P, range: [P0, P0 + 2], dataDir, webDir, templatesDir, command, eager: 0, log: () => {}, prewarm: false, publicOrigin: 'http://sim.test', devAuth: true }); + t.after(() => L.close()); + const health = async () => (await fetch(`${base}/__sim/healthz`)).json(); + const { slug } = await (await fetch(`${base}/__sim/worlds`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Spike' }) })).json(); + const m = await (await fetch(`${base}/__sim/metrics`)).text(); + assert.equal(val(m, 'sim_site_worlds_created_total'), 1); + assert.equal(val(m, `sim_world_viewers{world="${slug}"}`), 0); + assert.ok(val(m, 'sim_launcher_event_loop_delay_p99_seconds') >= 0); + assert.ok(val(m, 'sim_host_disk_free_bytes') > 0 && val(m, 'sim_host_disk_size_bytes') >= val(m, 'sim_host_disk_free_bytes')); + assert.equal((await health()).hostOk, undefined, 'no host check and no alerts file: nothing to report'); + + // the collector's own evaluation, firing then resolving, writing the file the launcher reads (as the unit runs it); + // real time, so the file is fresh, and no hold or clear window, so one scrape each way + const tele = join(dataDir, 'telemetry'), problems = join(dataDir, 'alert-problems.json'), store = openStore(tele, { everyMs: EVERY }); + const rules = rulesWith({ 'disk-free': { holdMs: 0, clearMs: 0 } }); + let last = 0; + const step = async (freeShare) => { + const at = Math.max(Date.now(), last + 1); last = at; + const body = metricsText({ now: at, launcher: { ...QUIET, diskFreeBytes: freeShare * 300 * GB }, worlds: [] }); + await scrapeOnce(store, 'http://127.0.0.1:1/__sim/metrics', { now: () => at, fetchImpl: async () => ({ ok: true, status: 200, text: async () => body }) }); + return catchUp(tele, { rules, everyMs: EVERY, now: at, problems }).events.filter((e) => e.rule === 'disk-free').map((e) => e.to); + }; + assert.deepEqual(await step(0.1), ['firing']); + const h1 = await health(); + assert.equal(h1.hostOk, false); assert.deepEqual(h1.hostCodes, ['alert:disk-free']); + assert.deepEqual(await step(0.5), ['resolved']); + const done = JSON.parse(readFileSync(problems, 'utf8')); + assert.deepEqual(done.problems, []); + const h2 = await health(); + assert.notEqual(h2.hostOk, false); assert.equal(h2.hostCodes, undefined); + // a timer copy of the old alert in host-health.json does not hold it: the alerts file read now wins + writeFileSync(join(dataDir, 'host-health.json'), JSON.stringify({ ok: false, at: new Date().toISOString(), codes: ['alert:disk-free'], problems: ['x'] })); + assert.equal((await health()).hostOk, true); + // a collector that died (the file stopped changing) is a problem too + writeFileSync(problems, JSON.stringify({ ...done, at: new Date(Date.now() - 10 * MIN).toISOString() })); + assert.deepEqual((await health()).hostCodes, ['alert:stale']); +}); diff --git a/tools/world-launcher/telemetry.mjs b/tools/world-launcher/telemetry.mjs index 7599dfb31..f61e37433 100644 --- a/tools/world-launcher/telemetry.mjs +++ b/tools/world-launcher/telemetry.mjs @@ -8,12 +8,14 @@ // node tools/world-launcher/telemetry.mjs report --dir [--range 1h|24h|7d|..] [--json] // node tools/world-launcher/telemetry.mjs close --dir roll up, gzip and prune finished hours now // node tools/world-launcher/telemetry.mjs alerts --dir [--rules ] [--json] evaluate alert rules (#739) -// collect ... --alerts [--rules ] the same evaluation after every scrape +// collect ... --alerts [--rules ] [--problems /alert-problems.json] the same evaluation after every +// scrape; --problems writes the firing rules that page as host problem codes (alert:, #2541) for healthz +// collect --launcher-args ... the --url from $SIM_WORLDS_ARGS (--port, --base), as the sim-worlds-telemetry@ unit runs it // node tools/world-launcher/telemetry.mjs cost --dir --cost [--range 24h] [--json] // node tools/world-launcher/telemetry.mjs packet --dir [--cost ] [--bench ] [--range 24h] [--json] context packet // node tools/world-launcher/telemetry.mjs units --dir --cost [--range 24h] [--json] unit costs (#739 S3) // node tools/world-launcher/telemetry.mjs scenarios --cost --bench [--worlds 10,50,100,200,400] [--json] -import { existsSync, readFileSync } from 'node:fs'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { pathToFileURL } from 'node:url'; @@ -44,6 +46,13 @@ export async function scrapeOnce(store, url, { now = Date.now, timeoutMs = 10_00 } catch (e) { store.append({ t, ok: false, error: e?.name === 'TimeoutError' ? 'timeout' : String(e?.cause?.code ?? e?.message ?? e) }); return false; } } +/** #2541: the launcher's loopback metrics URL from its own arguments (/etc/sim-worlds/.conf SIM_WORLDS_ARGS). */ +export function metricsUrlFromArgs(args = '') { + const port = /(?:^|\s)--port[= ](\d+)/.exec(args)?.[1], base = /(?:^|\s)--base[= ](\/[^\s]*)/.exec(args)?.[1] ?? ''; + if (!port) throw new Error('--launcher-args: no --port in SIM_WORLDS_ARGS'); + return `http://127.0.0.1:${port}${base.replace(/\/$/, '')}/__sim/metrics`; +} + export function openStore(dir, o = {}) { return new Store(dir, { ...o, reduceHour }); } const readJson = (file, what) => { try { return JSON.parse(readFileSync(file, 'utf8')); } catch (e) { throw new Error(`${what} ${file}: ${e.message}`); } }; /** The cost section for a report window: the private cost file applied to the window's world-hours. */ @@ -69,22 +78,25 @@ async function main() { } const dir = opt('dir'); if (!dir) throw new Error('--dir is required'); if (cmd === 'collect') { - const url = opt('url'); if (!url || !/^http:\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d+)?\//.test(url)) throw new Error('--url must be the launcher\'s loopback /__sim/metrics'); + const url = flag('launcher-args') ? metricsUrlFromArgs(process.env.SIM_WORLDS_ARGS) : opt('url'); if (!url || !/^http:\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d+)?\//.test(url)) throw new Error('--url must be the launcher\'s loopback /__sim/metrics'); const everyMs = num('every', DEFAULTS.everyMs / 1000) * 1000; const store = openStore(dir, { everyMs, fineDays: num('fine-days', DEFAULTS.fineDays), rollupDays: num('rollup-days', DEFAULTS.rollupDays), maxBytes: num('max-bytes', DEFAULTS.maxBytes) }); try { store.close(Date.now()); } catch { store.state.loss.failedWrites++; } // finish hours a previous run left open; never fatal if (flag('once')) { await scrapeOnce(store, url, { timeoutMs: Math.min(10_000, everyMs) }); store.saveState(); return; } const rules = flag('alerts') ? rulesFrom(opt('rules')) : null; // read once: a bad rules file stops here, not mid-run + const problems = rules ? opt('problems', null) : null; let stop = false; const quit = () => { stop = true; }; process.on('SIGINT', quit); process.on('SIGTERM', quit); const t0 = Date.now(); let k = 0; while (!stop) { await scrapeOnce(store, url, { timeoutMs: Math.min(10_000, everyMs) }); store.state.collector = { rssMb: Math.round(process.memoryUsage().rss / 2 ** 20), at: Date.now(), everyMs }; if (k === 0) store.saveState(); - if (rules) try { catchUp(dir, { rules, everyMs, lookbackMs: 3 * everyMs }); } catch { store.state.loss.failedWrites++; } // alerts never stop the collector + if (rules) try { catchUp(dir, { rules, everyMs, lookbackMs: 3 * everyMs, problems }); } catch { store.state.loss.failedWrites++; } // alerts never stop the collector const next = t0 + ++k * everyMs; await new Promise((r) => setTimeout(r, Math.max(0, next - Date.now()))); // fixed cadence: a slow scrape does not shift the grid } - store.saveState(); return; + store.saveState(); + if (problems) rmSync(problems, { force: true }); // a clean stop is not a stale collector (a crash leaves it: alert:stale) + return; } if (cmd === 'close') { const s = openStore(dir); s.close(Date.now()); return; } // the collector's own options (state.json); do not run while it is collecting if (cmd === 'report') { @@ -92,7 +104,7 @@ async function main() { process.stdout.write(flag('json') ? JSON.stringify(r, null, 1) + '\n' : text(r)); return; } if (cmd === 'alerts') { - const s = openStore(dir), r = catchUp(dir, { rules: rulesFrom(opt('rules')), everyMs: s.o.everyMs }); + const s = openStore(dir), r = catchUp(dir, { rules: rulesFrom(opt('rules')), everyMs: s.o.everyMs, problems: opt('problems', null) }); if (flag('json')) { process.stdout.write(JSON.stringify(r, null, 1) + '\n'); return; } const line = (e) => `${new Date(e.t).toISOString()} ${e.rule}${e.world ? ` ${e.world}` : ''}: ${e.from} -> ${e.to} (${e.what} ${e.value ?? 'missing'}, threshold ${e.threshold})`; process.stdout.write([`${r.rows} new row(s), ${r.events.length} transition(s)`, ...r.events.map(line), diff --git a/tools/world-launcher/telemetry/alerts.mjs b/tools/world-launcher/telemetry/alerts.mjs index a810becb2..a5bcb578b 100644 --- a/tools/world-launcher/telemetry/alerts.mjs +++ b/tools/world-launcher/telemetry/alerts.mjs @@ -3,8 +3,11 @@ // /alerts.jsonl one line per transition: firing | resolved | unknown | known. Unchanged state writes nothing. // /alerts-state.json the evaluator state (windows, last row), so a restart neither repeats nor loses a transition // Rules have a hold window (the condition must hold that long before it fires), a clear window and an optional clear -// threshold (hysteresis). A running world without a rule's series becomes `unknown`, never healthy. No delivery: a -// local event only (paging is #461's). +// threshold (hysteresis). A running world without a rule's series becomes `unknown`, never healthy. +// Reaching a person (#2541): `writeProblems` puts every firing rule that pages (all but `page: false`) in a problems file +// as `alert:`; the launcher folds it into /__sim/healthz (`hostOk: false`, `hostCodes`) at once and +// deploy/host-health.mjs into host-health.json, and the outside check watches `hostOk`. A monitor on the same machine +// cannot report that machine failing; the outside check can, so nothing here sends anything. import { appendFileSync, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { labelsOf } from './prom.mjs'; @@ -26,10 +29,15 @@ export const RULES = [ { id: 'queue-stall', scope: 'world', what: 'oldest queued request (s)', value: (c) => c.g('sim_world_queue_oldest_seconds'), op: '>', threshold: 2, clearAt: 1 }, // the independent clock measurement (#737/#799): late ticks per second, not the pace-normalized service ratio { id: 'clock-lag', scope: 'world', what: 'late ticks per second', value: (c) => c.rate('sim_world_ticks_late_total'), op: '>', threshold: 1, clearAt: 0.2 }, - { id: 'refusals', scope: 'world', what: 'refused or shed requests per second', value: (c) => { const a = c.rate('sim_world_refused_total'), b = c.rate('sim_world_shed_total'); return a == null && b == null ? null : (a ?? 0) + (b ?? 0); }, op: '>', threshold: 0, missingOk: true, holdMs: 0, clearMs: 5 * MIN }, + { id: 'refusals', scope: 'world', what: 'refused or shed requests per second', value: (c) => { const a = c.rate('sim_world_refused_total'), b = c.rate('sim_world_shed_total'); return a == null && b == null ? null : (a ?? 0) + (b ?? 0); }, op: '>', threshold: 0, missingOk: true, holdMs: 0, clearMs: 5 * MIN, page: false }, // the ceilings doing their job { id: 'stale-live', scope: 'world', what: 'live.json age (s)', value: (c) => c.g('sim_world_live_age_seconds'), op: '>', threshold: 30, holdMs: 0 }, // a crash loop is judged whatever the world's status (a restarting world is not running) { id: 'crash-loop', scope: 'any-world', what: 'non-planned restarts in 10 min', value: (c) => c.inc('sim_world_restarts_total', 10 * MIN, 'planned'), op: '>=', threshold: 3, holdMs: 0, clearMs: 10 * MIN }, + // #2541: a spike. `viewers-high` is information only (alerts.jsonl): it never raises a host problem code. + { id: 'viewers-high', scope: 'world', what: 'viewer sockets on the world', value: (c) => c.g('sim_world_viewers'), op: '>', threshold: 1000, missingOk: true, holdMs: 0, page: false }, + { id: 'launcher-loop-stall', scope: 'fleet', what: 'launcher event-loop delay p99 (s)', value: (c) => c.g('sim_launcher_event_loop_delay_p99_seconds'), op: '>', threshold: 0.25, clearAt: 0.15 }, + { id: 'worlds-created', scope: 'fleet', what: 'worlds created in the last hour', value: (c) => c.inc('sim_site_worlds_created_total', 60 * MIN), op: '>', threshold: 10, holdMs: 0 }, + { id: 'disk-free', scope: 'fleet', what: 'share of the data disk free', value: (c) => { const f = c.g('sim_host_disk_free_bytes'), s = c.g('sim_host_disk_size_bytes'); return f == null || !s ? null : f / s; }, op: '<', threshold: 0.2 }, { id: 'host-memory', scope: 'fleet', what: 'host memory available (GiB)', value: (c) => { const v = c.g('sim_host_mem_available_bytes'); return v == null ? null : v / GIB; }, op: '<', threshold: 1, clearAt: 1.5 }, { id: 'telemetry-gap', scope: 'collector', what: 'seconds since the last answered scrape', op: '>', threshold: null, holdMs: 0, clearMs: 0 }, ]; @@ -41,7 +49,18 @@ export function rulesWith(over = {}) { return RULES.filter((r) => !over[r.id]?.off).map((r) => ({ holdMs: HOLD, clearMs: CLEAR, ...r, ...Object.fromEntries(Object.entries(over[r.id] ?? {}).filter(([k]) => ['threshold', 'clearAt', 'holdMs', 'clearMs'].includes(k))) })); } -export function initialState() { return { version: ALERTS_VERSION, lastT: null, lastOkT: null, prev: null, keys: {}, restarts: {} }; } +export function initialState() { return { version: ALERTS_VERSION, lastT: null, lastOkT: null, prev: null, keys: {}, restarts: {}, fleet: {} }; } + +/** A fleet counter's increase over `windowMs`, across launcher restarts (a counter that went down started again from 0): + * `total` adds every rise seen; `points` keeps its change points, the one at or before the window start the baseline. */ +function fleetInc(st, name, x, t, windowMs) { + const h = ((st.fleet ??= {})[name] ??= { last: null, total: 0, points: [] }); + if (h.last != null) h.total += x >= h.last ? x - h.last : x; + h.last = x; + if (!h.points.length || h.points.at(-1)[1] !== h.total) h.points.push([t, h.total]); + while (h.points.length > 1 && h.points[1][0] <= t - windowMs) h.points.shift(); + return h.total - h.points[0][1]; // rises before the first scrape were never seen: not counted +} /** The per-row view of one world: gauges, sums across label sets and rates against the previous adjacent row. */ function worldsOf(values) { @@ -79,7 +98,7 @@ export function evaluate(rows, rules, state = initialState(), { everyMs = 15_000 gapAt(row.t, 0); st.lastOkT = row.t; const worlds = worldsOf(row.values), prevWorlds = prev ? worldsOf(new Map(Object.entries(prev.values))) : new Map(); for (const rule of rules) { - if (rule.scope === 'fleet') step(rule, null, row.t, rule.value({ g: (n) => row.values.get(n) ?? null })); + if (rule.scope === 'fleet') step(rule, null, row.t, rule.value({ g: (n) => row.values.get(n) ?? null, inc: (n, windowMs) => { const x = row.values.get(n); return x == null ? null : fleetInc(st, n, x, row.t, windowMs); } })); if (rule.scope !== 'world' && rule.scope !== 'any-world') continue; for (const [w, m] of worlds) { const isRunning = running(m, w); @@ -109,8 +128,31 @@ export function open(state) { return Object.entries(state.keys).filter(([, k]) => k.state !== 'ok').map(([key, k]) => { const [rule, world] = key.split('|'); return { rule, ...(world ? { world } : {}), state: k.state }; }); } -/** Read the store's rows after the saved state, evaluate, append the transitions, save the state. */ -export function catchUp(dir, { rules = rulesWith(), everyMs = 15_000, now = Date.now(), lookbackMs = 2 * 3_600_000 } = {}) { +/** #2541: the host problems a person must hear about: every firing rule that pages, as an `alert:` code. */ +export function problemsOf(state, rules) { + const byId = new Map(rules.map((r) => [r.id, r])), out = []; + for (const o of open(state)) { const r = byId.get(o.rule); if (o.state === 'firing' && r && r.page !== false) out.push({ code: `alert:${o.rule}`, text: `${r.what}${o.world ? ` (${o.world})` : ''}: over its threshold ${r.threshold}` }); } + return out; +} +/** Write them where deploy/host-health.mjs and the launcher's healthz read them (`/alert-problems.json`, atomically). + * `everyMs` lets a reader tell a stopped collector (a stale file) from a quiet one. */ +export function writeProblems(file, state, rules, { now = Date.now(), everyMs = 15_000 } = {}) { + writeFileSync(file + '.tmp', JSON.stringify({ at: new Date(now).toISOString(), everyMs, problems: problemsOf(state, rules) }) + '\n'); renameSync(file + '.tmp', file); +} + +/** #2541: the paging alerts writeProblems left in /alert-problems.json. None without the file (the collector is + * not enabled); `alert:stale` when it is older than 4 scrapes and 2 min (the collector died; a clean stop removes it). + * The launcher's healthz reads it on every request, deploy/host-health.mjs every 10 min. */ +export function readAlertProblems(dataDir, now = Date.now()) { + let j; try { j = JSON.parse(readFileSync(join(dataDir, 'alert-problems.json'), 'utf8')); } catch (e) { return e?.code === 'ENOENT' ? [] : [{ code: 'alert:stale', text: 'alert-problems.json unreadable' }]; } + const age = now - Date.parse(j?.at), limit = Math.max(120_000, 4 * (Number(j?.everyMs) || 15_000)); + if (!(age < limit)) return [{ code: 'alert:stale', text: `the telemetry collector last wrote its alerts ${Number.isFinite(age) ? `${Math.round(age / 60_000)} min` : 'at an unknown time'} ago` }]; + return Array.isArray(j.problems) ? j.problems.filter((p) => typeof p?.code === 'string' && p.code.startsWith('alert:')) : []; +} + +/** Read the store's rows after the saved state, evaluate, append the transitions, save the state (and, with `problems`, + * write the paging problems file). */ +export function catchUp(dir, { rules = rulesWith(), everyMs = 15_000, now = Date.now(), lookbackMs = 2 * 3_600_000, problems = null } = {}) { const file = join(dir, 'alerts-state.json'); let state; try { state = JSON.parse(readFileSync(file, 'utf8')); if (state.version !== ALERTS_VERSION) state = initialState(); } catch { state = initialState(); } const from = state.lastT != null ? Math.max(state.lastT, now - lookbackMs) : now - lookbackMs; @@ -118,5 +160,6 @@ export function catchUp(dir, { rules = rulesWith(), everyMs = 15_000, now = Date const r = evaluate(rows, rules, state, { everyMs, now }); if (r.events.length) appendFileSync(join(dir, 'alerts.jsonl'), r.events.map((e) => JSON.stringify(e)).join('\n') + '\n'); writeFileSync(file + '.tmp', JSON.stringify(r.state) + '\n'); renameSync(file + '.tmp', file); + if (problems) writeProblems(problems, r.state, rules, { now, everyMs }); return { events: r.events, open: open(r.state), rows: rows.length, notExported: NOT_EXPORTED }; }