From 78f664b3556bdfa98d28bcc1b90ba8dd1ad1a392 Mon Sep 17 00:00:00 2001 From: Jared Scott Date: Fri, 28 Aug 2026 10:29:28 +0800 Subject: [PATCH 1/2] fix(runtime): read the boot envelope as a rendering, and walk the scan window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Spacedock strip published nothing while a workflow was plainly running, reporting "A workflow exists, but nothing is fresh enough to show" — which reads like a freshness problem and is not one. Two independent defects, each fatal on its own. The envelope arrives rendered, not raw. `boot_records` looked for the literal `{"command"` of a JSON object. The first officer's skill tells it to run `status --boot --identify --json`, but nothing tells it to echo that JSON, and both real first-officer sessions measured piped it through a formatter, so the transcript carried an indented key/value rendering and the object never appeared. Across 120 transcripts over 21 days the JSON branch matched exactly one file, and that one was this repository's own fixtures catted into a tool result: the strip had never fired on a real session. So the fields are read line by line as well. The trust model is unchanged — a rendered path is gated on a top-level `command: boot` exactly as the JSON branch is gated on `envelope["command"] == "boot"`, only column-0 keys are read so a nested decoy cannot nominate one, and every downstream guard still stands between an extracted path and anything published. And it is not in the head. The scan read the first `spacedock_boot_scan_bytes` on the reasoning that boot output is written at session start. A first officer greets and discovers before it boots: the two sessions measured booted at 69% and 73% of the way through their transcripts, at bytes 803,503 and 821,199. Raising the cap only moves the guess and is the expensive direction, because the old `(path, size)` key misses on every write below the cap. The window walks instead — each pass reads at most `spacedock_boot_scan_bytes` of not-yet-scanned bytes and remembers how far it reached, stopping on a line boundary and stepping over a record longer than the window. Verified end to end against the live transcript: the strip now renders roadmap-burndown with drc-4029 live on triage, matching the running worker. Records the reasoning as S-7 in docs/design-spacedock.md. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jared Scott --- .../cargento/cargento_runtime/spacedock.py | 155 ++++++++++++++++-- .../skills/cargento/cargento_runtime/state.py | 2 +- .../skills/cargento/tests/test_spacedock.py | 148 +++++++++++++++++ docs/design-spacedock.md | 44 ++++- 4 files changed, 329 insertions(+), 20 deletions(-) diff --git a/cargento/skills/cargento/cargento_runtime/spacedock.py b/cargento/skills/cargento/cargento_runtime/spacedock.py index cf074f2..8025f46 100644 --- a/cargento/skills/cargento/cargento_runtime/spacedock.py +++ b/cargento/skills/cargento/cargento_runtime/spacedock.py @@ -30,7 +30,8 @@ # the transcript's first records carry an ``agentSetting``. That alone proves # the session is Spacedock, and costs nothing: it is in the head bytes the # subagent classifier already reads. -# 2. The first officer runs `spacedock status --boot` at startup and the JSON +# 2. The first officer runs `spacedock status --boot` — measured here at 69% +# and 73% of the way through a transcript, not at startup — and the # envelope lands in the transcript as a tool result, carrying the ABSOLUTE # workflow directory and the ABSOLUTE entity-state directory, so nothing has # to be discovered by scanning. @@ -298,8 +299,87 @@ def _usable_dir(value: object) -> TypeGuard[str]: return True +def rendered_pair(raw: str) -> tuple[str, str] | None: + """``(key, value)`` from one line of a rendered boot envelope, or None. + + Only the shape ``key: value`` is assumed. The key may carry JSON's quotes + or none and the value may carry quotes and a trailing comma, because what + reaches the transcript is whatever the session chose to print. Partitioning + on the FIRST colon keeps a Windows value (``C:\\...``) intact. + """ + line = raw.strip() + head, sep, tail = line.partition(":") + if not sep: + return None + key = head.strip().strip("\"'") + if not key: + return None + return (key, tail.strip().rstrip(",").strip().strip("\"'")) + + +def rendered_envelope(config: RuntimeConfig, text: str) -> dict[str, Any] | None: + """The boot envelope a session PRINTED rather than pasted, or None. + + The first officer is told to consume ``status --boot --json``, and nothing + tells it to echo that JSON verbatim. Measured on this machine: every real + first-officer session piped the envelope through a formatter, so the + transcript carried an indented rendering and the raw object never appeared + once in 120 transcripts over 21 days — the JSON branch above has never + matched a genuine session, only Cargento's own source and fixtures catted + into a tool result. The fields the strip needs survive any such rendering, + so they are read line by line rather than decoded. + + Gated on a top-level ``command: boot`` for the same reason the JSON branch + checks ``envelope["command"] == "boot"``: without it, a tool result that + merely prints these key names — this module's own source — nominates a path. + Only top-level keys are read, so a nested decoy cannot supply one either. + + No line bound is imposed here: the text is already a slice of the at most + ``spacedock_boot_scan_bytes`` the caller read, and this scan is linear with + trivial per-line work, unlike the ``raw_decode`` the candidate cap protects. + """ + out: dict[str, Any] = {} + dispatchable: list[dict[str, str]] = [] + item: dict[str, str] = {} + booted = False + in_dispatchable = False + for raw in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"): + pair = rendered_pair(raw) + if pair is None: + continue + key, value = pair + if not raw[:1].isspace(): + in_dispatchable = key == "dispatchable" + if key == "command": + booted = booted or value == "boot" + elif key in ("definition_dir", "entity_dir") and value and key not in out: + # First occurrence wins, so a decoy in a trailing blob cannot + # displace the envelope the session actually booted from. + out[key] = value + continue + if not in_dispatchable or len(dispatchable) >= config.spacedock_max_entities: + continue + # Paired on `slug` rather than on the list syntax, so both the + # `-`-delimited and inline renderings of an item read the same. + if key == "slug" and value: + if item: + dispatchable.append(item) + item = {"slug": value} + elif key == "current" and value and item and "current" not in item: + item["current"] = value + if item and len(dispatchable) < config.spacedock_max_entities: + dispatchable.append(item) + if not booted or "definition_dir" not in out: + return None + out["command"] = "boot" + out["dispatchable"] = [ + entry for entry in dispatchable if entry.get("slug") and entry.get("current") + ] + return out + + def boot_records(config: RuntimeConfig, data: bytes) -> list[dict[str, Any]]: - """Every ``spacedock status --boot`` envelope in a transcript head. + """Every ``spacedock status --boot`` envelope in a slice of a transcript. Decoded line by line as the JSONL it is, so the JSON decoder does the unescaping and each envelope is located inside already-plain text. An @@ -319,6 +399,7 @@ def boot_records(config: RuntimeConfig, data: bytes) -> list[dict[str, Any]]: if not isinstance(record, dict): continue for text in tool_result_text(record): + found = len(out) position = 0 for _ in range(config.spacedock_max_boot_candidates): begin = text.find('{"command"', position) @@ -343,6 +424,13 @@ def boot_records(config: RuntimeConfig, data: bytes) -> list[dict[str, Any]]: out.append(envelope) if len(out) >= config.spacedock_max_boot_records: return out + if len(out) > found: + continue + rendered = rendered_envelope(config, text) + if rendered is not None: + out.append(rendered) + if len(out) >= config.spacedock_max_boot_records: + return out return out @@ -410,34 +498,67 @@ def boot_entity_dir(envelopes: list[dict[str, Any]], workflow_dir: str) -> str: def transcript_boot(config: RuntimeConfig, state: RuntimeState, path: str) -> list[dict[str, Any]]: - """Boot envelopes from a transcript's head, cached per (path, size). - - Boot output is written once at session start and never rewritten, so the - scan is amortised: keying on size lets a still-growing session pick the - envelope up on a later refresh without rescanning an unchanged prefix. + """Boot envelopes from a transcript, scanned forward across refreshes. + + An earlier version read only the first ``spacedock_boot_scan_bytes``, on the + reasoning that boot output is written once at session start and never + rewritten. Measured against real first-officer sessions on this machine, it + is not: two of them booted 69% and 73% of the way through their transcripts, + at bytes 803,503 and 821,199, because a first officer greets and discovers + before it boots and Claude Code writes records up to 109 KB apiece. A + head-only window found neither. + + So the window walks rather than sits. Each pass reads at most + ``spacedock_boot_scan_bytes`` of not-yet-scanned bytes and remembers how far + it reached, which holds the per-refresh cost the head scan had while the + whole file is covered eventually. Progress is cached per path rather than + per ``(path, size)`` so a growing session keeps its place instead of + restarting the walk on every write. """ try: size = os.path.getsize(path) except OSError: return [] - key = (path, min(size, config.spacedock_boot_scan_bytes)) with state.cache_lock: - cached = state.spacedock_boot_cache.get(key) - if cached is not None: - return cached - envelope_records: list[dict[str, Any]] = [] + cached = state.spacedock_boot_cache.get(path) + records: list[dict[str, Any]] + scanned: int + records, scanned = cached if cached is not None else ([], 0) + if scanned > size: + # The file is shorter than the walk already covered, so it is not the + # one that progress was recorded against. Start it over. + records, scanned = [], 0 + if scanned >= size or len(records) >= config.spacedock_max_boot_records: + return records try: with open(path, "rb") as handle: + handle.seek(scanned) blob = handle.read(config.spacedock_boot_scan_bytes) - if b"definition_dir" in blob: - envelope_records = boot_records(config, blob) except OSError: - return [] + return records + chunk = blob + if scanned + len(blob) >= size: + consumed = len(blob) + else: + cut = blob.rfind(b"\n") + 1 + if cut: + # Stop on a line boundary so an envelope straddling the window edge + # is read whole on the next pass instead of halved on this one. + chunk, consumed = blob[:cut], cut + else: + # One record longer than the whole window. Nothing in it can be + # read as a line, so step over it rather than stall here forever. + chunk, consumed = b"", len(blob) + if b"definition_dir" in chunk: + records = (records + boot_records(config, chunk))[: config.spacedock_max_boot_records] with state.cache_lock: runtime_state.bounded_put( - state.spacedock_boot_cache, key, envelope_records, limit=config.max_cache_entries + state.spacedock_boot_cache, + path, + (records, scanned + consumed), + limit=config.max_cache_entries, ) - return envelope_records + return records def open_regular(path: str) -> int | None: diff --git a/cargento/skills/cargento/cargento_runtime/state.py b/cargento/skills/cargento/cargento_runtime/state.py index d3bfc49..883d765 100644 --- a/cargento/skills/cargento/cargento_runtime/state.py +++ b/cargento/skills/cargento/cargento_runtime/state.py @@ -86,7 +86,7 @@ class RuntimeState: turn_scan: dict[str, Any] = field(default_factory=dict) agent_class_cache: dict[str, tuple[bool, str, str]] = field(default_factory=dict) spacedock_role_cache: dict[str, str] = field(default_factory=dict) - spacedock_boot_cache: dict[tuple[str, int], list[dict[str, Any]]] = field(default_factory=dict) + spacedock_boot_cache: dict[str, tuple[list[dict[str, Any]], int]] = field(default_factory=dict) spacedock_workflow_cache: dict[tuple[str, int, int], dict[str, Any] | None] = field( default_factory=dict ) diff --git a/cargento/skills/cargento/tests/test_spacedock.py b/cargento/skills/cargento/tests/test_spacedock.py index 8b8dba5..d29f2be 100644 --- a/cargento/skills/cargento/tests/test_spacedock.py +++ b/cargento/skills/cargento/tests/test_spacedock.py @@ -13,6 +13,7 @@ from cargento_runtime import spacedock from .support import ( + config_patch, make_runtime, runtime, state_of, @@ -486,6 +487,75 @@ def test_boot_scan_is_bounded_against_decoy_candidates(self) -> None: self.assertLess(time.monotonic() - started, 1.0) + RENDERED_BOOT = ( + 'command: "boot"\n' + "mods:\n" + " idle:\n" + ' - "pr-merge"\n' + 'id_style: "slug"\n' + "dispatchable:\n" + " -\n" + ' id: "drc-4029"\n' + ' slug: "drc-4029"\n' + ' current: "selection"\n' + ' next: "triage"\n' + " -\n" + ' slug: "drc-4021"\n' + ' current: "selection"\n' + 'definition_dir: "/w/one"\n' + 'entity_dir: "/w/one/.spacedock-state"\n' + 'entity_dir_present: "true"\n' + ) + + def tool_result(self, text: str) -> bytes: + return json.dumps( + {"type": "user", "message": {"content": [{"type": "tool_result", "content": text}]}} + ).encode() + + def test_boot_records_read_an_envelope_the_session_rendered(self) -> None: + """The first officer is told to consume `status --boot --json`, not to + echo it verbatim, and every real session measured here piped it through + a formatter — so the raw object never reaches the transcript and the + JSON branch alone found nothing in 120 transcripts over 21 days. + Falsifying edit: drop the `rendered_envelope` call from `boot_records` + and this returns [].""" + config, _runtime = runtime() + + records = spacedock.boot_records(config, self.tool_result(self.RENDERED_BOOT)) + + self.assertEqual(1, len(records)) + self.assertEqual("/w/one", records[0]["definition_dir"]) + self.assertEqual("/w/one/.spacedock-state", spacedock.boot_entity_dir(records, "/w/one")) + self.assertEqual(["/w/one"], spacedock.workflow_dirs(config, records)) + self.assertEqual( + {"drc-4029": "selection", "drc-4021": "selection"}, + spacedock.boot_entities(records, "/w/one"), + ) + + def test_a_rendering_without_a_boot_command_nominates_nothing(self) -> None: + """This module's own source names every key the renderer prints, and it + gets catted into tool results routinely. `command: boot` is what keeps + that from nominating a path, exactly as the JSON branch requires.""" + config, _runtime = runtime() + source = 'value = record.get("definition_dir")\nentity_dir: "/w/two"\n' + + self.assertEqual([], spacedock.boot_records(config, self.tool_result(source))) + + def test_a_nested_rendered_key_cannot_nominate_a_path(self) -> None: + """Only column-0 keys are read, so a `definition_dir` printed inside a + nested block is data the session displayed, not the envelope's own.""" + config, _runtime = runtime() + text = 'command: "boot"\nfindings:\n definition_dir: "/w/evil"\ndefinition_dir: "/w/ok"\n' + + records = spacedock.boot_records(config, self.tool_result(text)) + + self.assertEqual(["/w/ok"], spacedock.workflow_dirs(config, records)) + + def test_a_rendered_envelope_needs_a_definition_dir(self) -> None: + config, _runtime = runtime() + + self.assertEqual([], spacedock.boot_records(config, self.tool_result('command: "boot"\n'))) + def test_workflow_dirs_reject_relative_and_nul_paths(self) -> None: config, _runtime = runtime() records: list[dict[str, Any]] = [ @@ -523,6 +593,84 @@ def setUp(self) -> None: state.spacedock_role_cache.clear() state.spacedock_entity_cache.clear() + def test_the_boot_scan_walks_forward_across_refreshes(self) -> None: + """A first officer does not necessarily boot at session start. The two + real sessions measured here booted at 69% and 73% of their transcripts, + past any head-only window, so the window advances instead of sitting: + nothing on the first pass, the envelope on a later one, and the file is + never rescanned from the top. Falsifying edit: pin the read back to + `handle.read(scan_bytes)` from offset 0 and the envelope is never seen.""" + holder = tempfile.TemporaryDirectory(prefix="cargento-boot-") + self.addCleanup(holder.cleanup) + path = str(Path(holder.name) / "session.jsonl") + filler = json.dumps({"type": "user", "message": {"content": "x" * 900}}) + "\n" + envelope = ( + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "content": 'command: "boot"\ndefinition_dir: "/w/late"\n', + } + ] + }, + } + ) + + "\n" + ) + with open(path, "w", encoding="utf-8") as handle: + handle.write(filler * 4 + envelope) + + with config_patch(spacedock_boot_scan_bytes=1024): + config, state = runtime() + first = spacedock.transcript_boot(config, state, path) + passes = 1 + while not spacedock.transcript_boot(config, state, path) and passes < 12: + passes += 1 + found = spacedock.transcript_boot(config, state, path) + + self.assertEqual([], first) + self.assertEqual(["/w/late"], spacedock.workflow_dirs(config, found)) + # Progress is remembered, so the walk terminates instead of restarting. + self.assertLessEqual(passes, 6) + self.assertGreaterEqual(state.spacedock_boot_cache[path][1], os.path.getsize(path)) + + def test_a_shorter_transcript_restarts_the_walk(self) -> None: + """Progress is recorded against a file, not a path. A path that now + holds fewer bytes than the walk already covered is not that file, and + resuming mid-way through it would skip whatever it now begins with.""" + holder = tempfile.TemporaryDirectory(prefix="cargento-boot-") + self.addCleanup(holder.cleanup) + path = str(Path(holder.name) / "session.jsonl") + envelope = ( + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "content": 'command: "boot"\ndefinition_dir: "/w/fresh"\n', + } + ] + }, + } + ) + + "\n" + ) + config, state = runtime() + with open(path, "w", encoding="utf-8") as handle: + handle.write(json.dumps({"type": "user", "message": {"content": "x" * 4000}}) + "\n") + self.assertEqual([], spacedock.transcript_boot(config, state, path)) + with open(path, "w", encoding="utf-8") as handle: + handle.write(envelope) + + records = spacedock.transcript_boot(config, state, path) + + self.assertEqual(["/w/fresh"], spacedock.workflow_dirs(config, records)) + def workflow(self, body: str | None = None) -> Path: holder = tempfile.TemporaryDirectory(prefix="cargento-sd-") self.addCleanup(holder.cleanup) diff --git a/docs/design-spacedock.md b/docs/design-spacedock.md index 96f6ddf..4106fec 100644 --- a/docs/design-spacedock.md +++ b/docs/design-spacedock.md @@ -99,8 +99,9 @@ officer, and the same payload carries the paths. Only a first officer runs `spac so presence is good evidence, and it needs no new source. It costs two things, both accepted rather than overlooked. Every Pi session pays a bounded transcript -head scan on refresh, where Claude pays a cached lookup and stops; the scan is capped at -`spacedock_boot_scan_bytes` and cached on `(path, size)`, so a settled transcript costs one `stat`. +scan on refresh, where Claude pays a cached lookup and stops; each pass is capped at +`spacedock_boot_scan_bytes` and progress is cached per path (S-7), so a settled transcript costs +one `stat`. And classification now depends on tool output rather than a launch-time declaration, which is a weaker signal: tool output is whatever a tool printed. The guards that matter sit downstream and are unchanged, so a crafted envelope still has to survive path canonicalisation, the symlink and @@ -155,6 +156,45 @@ items, which record the shell invocation together with its captured stdout, and records carry a boot envelope at all. The mention is the command line, not its output. So the shape would add a maintenance surface and no session. +## S-7: Read the envelope as a rendering, and walk the window to find it + +S-6 settled *where* the envelope may come from. Two later measurements showed both remaining +assumptions about *what it looks like* and *where in the file it sits* were wrong, and each was +independently fatal — the strip published nothing at all while a workflow was plainly running, and +said so as "A workflow exists, but nothing is fresh enough to show", which reads like a freshness +problem and is not one. + +**It arrives rendered, not raw.** The reader looked for the literal `{"command"` of a JSON object. +The first officer's own skill tells it to run `status --boot --identify --json` and "consume JSON, +not the human table" — but nothing tells it to *echo* that JSON, and both real first-officer +sessions measured here piped it through a formatter, so what reached the transcript was an indented +key/value rendering and the object never appeared. Across 120 transcripts over 21 days the JSON +branch matched exactly one file, and that one was this repository's own test fixtures catted into a +tool result. The feature had never once fired on a real session. + +So the fields are read line by line rather than decoded, which survives any rendering that keeps +`key: value`. The trust model is unchanged, deliberately: a rendered path is gated on a top-level +`command: boot` exactly as the JSON branch is gated on `envelope["command"] == "boot"`, only +column-0 keys are read so a nested decoy cannot nominate one, and every downstream guard — the +`_usable_dir` shape check, canonicalisation, the symlink and identity checks, and +`commissioned-by: spacedock@` — still stands between an extracted path and anything published. + +**It is not in the head.** The scan read the first `spacedock_boot_scan_bytes` on the reasoning that +boot output is written once at session start, so the read could be amortised on `(path, size)`. A +first officer greets and discovers before it boots: the two sessions measured booted at 69% and 73% +of the way through their transcripts, at bytes 803,503 and 821,199 of files near 1.1 MB. Claude Code +writes single records up to 109 KB, so a head window is small in lines even when it is large in +bytes. + +Raising the cap was rejected. It only moves the guess, and it is the expensive direction: the cache +key is `min(size, cap)`, so a live transcript below the cap misses on every write and re-reads the +whole file under the collection lock. Instead the window walks — each pass reads at most +`spacedock_boot_scan_bytes` of not-yet-scanned bytes and remembers how far it reached, keeping the +per-refresh cost the head scan had while covering the file eventually. A pass stops on a line +boundary so an envelope straddling the edge is read whole on the next one, and a record longer than +the whole window is stepped over rather than stalled on. The cost is latency, not coverage: an +envelope two windows in appears on the second refresh rather than the first. + ## Rejected alternatives worth keeping rejected Run `spacedock status` ourselves. It would answer every question above directly and correctly. It From d0bab382c57c0e901a32308b94950af7cb0a0491 Mon Sep 17 00:00:00 2001 From: Jared Scott Date: Fri, 28 Aug 2026 10:34:13 +0800 Subject: [PATCH 2/2] docs: reconcile the boot-envelope reads with the walking scan window The shipped skill body told readers no strip appears when the boot output sits outside the scanned head of a long transcript. The window walks now, so a late boot is a lag of a refresh or two rather than a permanent blank. The architecture doc described `spacedock_boot_cache` as keyed on a path and a bounded size; it holds a scan position beside its records instead. Its `spacedock.py` row gains the S-7 distinction that provenance settles where an envelope may come from, not what it looks like once it is there. S-7 rewritten to the voice standard: no em dashes, no boldface leads. Marker left alone, per the parallel-branch rule. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jared Scott --- cargento/skills/cargento/SKILL.md | 2 +- docs/design-runtime-architecture.md | 4 +-- docs/design-spacedock.md | 50 ++++++++++++++--------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cargento/skills/cargento/SKILL.md b/cargento/skills/cargento/SKILL.md index f13555d..1d61f43 100644 --- a/cargento/skills/cargento/SKILL.md +++ b/cargento/skills/cargento/SKILL.md @@ -317,7 +317,7 @@ Paths 2 and 3 are complementary and can both be installed. Keep `Notification` o - **"This request" ETA** = per-session current-turn estimate shown while Working. Estimated total = median of that session's past turns that lasted at least as long as the current one has so far. Turn boundaries: user prompt → last event before the next prompt (Claude, Gemini, Droid); active-branch user messages for Pi; explicit start/end events (Codex `task_started`/`task_complete`, Copilot `user.message`/`session.task_complete`); or DB message timestamps (OpenCode, Goose). JSONL harnesses use an incremental whole-file scanner (survives turns longer than the transcript tail). Pi retains the latest 50 completed durations. No ETA for Cursor. "running longer than recent turns" = no past turn was this long. Naive by design. A ⚠️ appears when elapsed or estimated total ≥ 15 min (`LONG_TURN_WARN_SEC`). Elapsed measures generation, not waiting: a mid-turn quiet stretch longer than 5 minutes (`TURN_GAP_RESET_SEC` — permission prompt, open question, sleep) re-anchors the clock at the post-gap event. - **Loop detection** = four tool calls in a row (`LOOP_ERROR_RUN_THRESHOLD`) coming back as errors inside one request. **Claude Code only**, because Claude is the only harness that records whether a tool call failed: Codex's tool-output records carry no error field, Copilot writes no tool-end record at all, and Droid's records look right but no failing Droid call has been captured, so nothing is inferred for any of them and their rows never carry the signal. It reads the transcript, never the tool's input, so no command text is held anywhere. Four is measured rather than picked: across the 25 most recent local transcripts, runs of three and runs of four each appeared in the same single session, and five in none — so four costs no detections and keeps clear of the benign runs (an `ls` that found nothing, a `git` in a deleted worktree) that filled the sample. The count is the longest run inside the current request and it clears at your next prompt, not when the failures stop, since a loop that has just gone quiet is what you walked back to the machine to find. It raises no flag of its own — the `long turn` chip and the ⚠️ still fire on duration alone, and a loop only changes what they say when you reach for them, which is the difference between "it is slow" and "it is repeating a failure". Where the request is not long enough for either, the working card and the calm detail panel say it anyway. The pattern is honest but not proof: iterating on a failing test looks the same from outside. - **Est. remaining** (per tracked-task session) = average duration of that session's completed tasks × open task count. Naive by design; "no estimate" until a session has a completed task that took ≥30s. -- **Spacedock stage strip** = one line per in-flight entity, showing that workflow's stages in declaration order with the entity's current stage highlighted. A bold entity name means a worker for it is running now; a plain one is read from the entity's state file or, failing that, the boot snapshot. Only Claude reports workers, so a Pi strip is never bold. In flight means moving: an entity resting on the initial stage or a terminal one is left out unless boot called it dispatchable, so a queue of thirty waiting on `intake` does not crowd out the two being worked. Entities whose state file has not been touched inside the freshness window are history, not work, and are skipped — that is what keeps a long-retired workflow off the card of a first officer that merely discovered it. A long workflow is windowed around the current stage, with `…` standing in for the stages elided. An over-long entity name is elided in the **middle**, never the tail — entities in one workflow share a long prefix and differ only at the end — and hovering it shows the full slug. No strip appears when the boot output is outside the scanned head of a long transcript, when the workflow README cannot be read, or when its frontmatter uses a construct the reader does not model — it renders nothing rather than a guess. +- **Spacedock stage strip** = one line per in-flight entity, showing that workflow's stages in declaration order with the entity's current stage highlighted. A bold entity name means a worker for it is running now; a plain one is read from the entity's state file or, failing that, the boot snapshot. Only Claude reports workers, so a Pi strip is never bold. In flight means moving: an entity resting on the initial stage or a terminal one is left out unless boot called it dispatchable, so a queue of thirty waiting on `intake` does not crowd out the two being worked. Entities whose state file has not been touched inside the freshness window are history, not work, and are skipped — that is what keeps a long-retired workflow off the card of a first officer that merely discovered it. A long workflow is windowed around the current stage, with `…` standing in for the stages elided. An over-long entity name is elided in the **middle**, never the tail — entities in one workflow share a long prefix and differ only at the end — and hovering it shows the full slug. A strip can lag rather than vanish when the boot output sits deep in a long transcript: the scan reads a bounded slice per refresh and walks forward, so a first officer that boots late in its transcript gets its strip a refresh or two after the session appears. No strip appears when the workflow README cannot be read, or when its frontmatter uses a construct the reader does not model — it renders nothing rather than a guess. - **Session title** = the harness's own generated title where it writes one (Claude records these, and they read like "Debug Spacedock workflow steps not displaying"), otherwise a prompt from the session itself — the **opening** one on Claude and Pi, which is what the great majority of titled rows fall back to, and the **newest** one only on Codex, which writes no generated title at all. That difference is why the line beneath the title exists: an opening prompt names the session durably and goes stale as work moves on. It is cleaned up rather than shown raw: a slash command reads as `/plugin` instead of the markup the harness wrapped it in, a dispatched worker's prompt shows the instruction instead of the envelope, absolute paths collapse to their last segment so the path does not eat the whole line, and an over-long title is cut on a word boundary. Relative paths and URLs are left whole, because the repo and PR number in a link are the informative part. Nothing is summarized by a model, so no session text leaves the machine. - **The line beneath the title** (on the opt-in `http://127.0.0.1:4553/?next=true` UI) answers a different question from the title, which is why it is a second line and not a better first one. A title names the session; this names what it is working on now. It always carries a label and the age of the record it came from, because that is what tells you how much to trust it: `asked, 4m:` is the newest thing you told it, `agent, 4m:` is the agent's own statement of what it was starting this turn, and `earlier, 2h:` is an older instruction shown because the newest one was a bare "proceed" that names no work. Where none of those can be established honestly the line is absent and the row shows its title alone, which is the old behaviour: a confident wrong line would mask the project name underneath it for as long as the session lives. Claude keeps its generated title on line 1 even when that title has gone stale, because it is how you recognise the row. Codex shows no second line when its title is already that same newest instruction — not always in full, since 167 of 280 local Codex titles are cut at the title width and 144 of those suppress the line while holding back a median 56 further characters. Repeating a prompt under itself costs a row two lines to say one thing, and the fuller reading is a keystroke away in the session panel. - **`…REDACTED` in a title, in the line beneath it, or in a task row** means the text there matched a credential shape and was replaced before it reached the page. The marker keeps the prefix that names the kind (`sk-ant-…REDACTED`, `AKIA…REDACTED`, `ghp_…REDACTED`) and the words around it, because the instruction is the useful part of the line. Read it as a finding rather than a rendering quirk: the value is still in the harness's own transcript on disk, where the next scan will read it again and so will anything else with access to that store, so the answer is to rotate the credential. It is shape matching over what a person typed, so it is neither complete nor infallible — a format nobody has measured goes through unmarked, and a long identifier that happens to look like a key is marked when it is not. diff --git a/docs/design-runtime-architecture.md b/docs/design-runtime-architecture.md index 3e1a0c5..7c9e657 100644 --- a/docs/design-runtime-architecture.md +++ b/docs/design-runtime-architecture.md @@ -49,7 +49,7 @@ Everything else lives in one file per responsibility: | Module | Owns | |---|---| | `config.py` | Immutable process configuration, store-root resolution, every tunable limit. | -| `state.py` | Mutable caches, locks, bounded-cache helpers, the server start stamp, and the runtime's published snapshot. Cache validity is not one rule. Of the eighteen caches, six turn on a file's `(mtime_ns, size)`: the four Claude and Codex title, user-event and instruction caches carry it in the value, and the Spacedock readme and entity caches carry it in the key. The other twelve do something else. `metadata_cache`, `cwd_cache`, `agent_class_cache` and `spacedock_role_cache` key on a path and never stat it, because each holds a fact that is fixed once the file has it. `spacedock_boot_cache` keys on a path and a bounded size. `claude_subagent_cache` keys on a session directory and turns on directory mtimes, since appending to a transcript moves no directory. `cursor_metadata_cache` turns on an `st_mtime` float across four derived keys per store. `pi_scan` and `turn_scan` hold an incremental scan position rather than a validity stamp. `usage_fetch_cache` and `usage_receipts` key on a vendor name and stamp the fetch time. `dispute_episodes` keys on `(harness, sid)`. Most are bounded at `max_cache_entries`; the two quota caches are bounded by the vendor list, and `dispute_episodes` by the sessions the current collection saw. The two instruction caches are honest about what they do not buy: the key moves on every event, so they never hit for a live session, and they exist for the idle and `?all=1` rows re-read on each refresh. | +| `state.py` | Mutable caches, locks, bounded-cache helpers, the server start stamp, and the runtime's published snapshot. Cache validity is not one rule. Of the eighteen caches, six turn on a file's `(mtime_ns, size)`: the four Claude and Codex title, user-event and instruction caches carry it in the value, and the Spacedock readme and entity caches carry it in the key. The other twelve do something else. `metadata_cache`, `cwd_cache`, `agent_class_cache` and `spacedock_role_cache` key on a path and never stat it, because each holds a fact that is fixed once the file has it. `spacedock_boot_cache` keys on a path and holds a scan position beside its records, because a first officer does not necessarily boot at session start (S-7). `claude_subagent_cache` keys on a session directory and turns on directory mtimes, since appending to a transcript moves no directory. `cursor_metadata_cache` turns on an `st_mtime` float across four derived keys per store. `pi_scan` and `turn_scan` hold an incremental scan position rather than a validity stamp. `usage_fetch_cache` and `usage_receipts` key on a vendor name and stamp the fetch time. `dispute_episodes` keys on `(harness, sid)`. Most are bounded at `max_cache_entries`; the two quota caches are bounded by the vendor list, and `dispute_episodes` by the sessions the current collection saw. The two instruction caches are honest about what they do not buy: the key moves on every event, so they never hit for a live session, and they exist for the idle and `?all=1` rows re-read on each refresh. | | `snapshot.py` | The published response bytes per variant, and the restart-qualified revision that versions them. Imports no runtime module, which is what lets `state`, `aggregate` and `http_api` all depend on it without a cycle. | | `stream.py` | Connected SSE clients and their one-slot revision mailboxes, with the connection budget. Imports no runtime module, for the same reason `snapshot.py` does not. `state` owns the registry because a connected stream belongs to the runtime, not to whichever object serves a request. | | `asks.py` | Every outstanding `ask_operator` question and its one-slot answer mailbox, with the pending budget, the expiry sweep and the shutdown decline. Imports no runtime module, for the reason `stream.py` does not: `state` owns the registry because an outstanding question belongs to the runtime rather than to whichever request serves it, and a leaf is the only shape that lets `state`, `aggregate` and `http_api` all reach it without a cycle. It therefore cannot call `records.safe_text`, so every text bound is applied at the `http_api` ingress and it stores what it is handed. See [design-ask-lane.md](design-ask-lane.md). | @@ -62,7 +62,7 @@ Everything else lives in one file per responsibility: | `transcripts.py` | Shared metadata readers, prompt titles, the non-Claude analyzers, and the instruction line beneath a session title. `codex_instruction` walks a rollout backward for its newest genuine prompt, because a bounded tail read misses that prompt on 62% of the rollouts that carry one, and `instruction_from` turns the candidates into the one labelled line both harnesses publish. `states_work` is the pairing rule for `records.bare_continuation`, and it reads two different things off one prompt: the RENDERING decides the shape, since a slash command is sixty characters of markup that reads back as a five-word instruction, and the tag-stripped BODY decides the word count, since `prompt_title` returns line 1 only and counting there called 97 of 2,066 local newest prompts bare over a real instruction. The preamble it pairs with is bounded structurally rather than by the turn floor alone: only a record newer than the newest genuine prompt can supply one, because reaching a `task_started` proves that some turn opened and not that this one did. | | `turns.py` | Generic incremental turn scanning and turn display, including the model a Codex rollout declares, the run of failed tool calls inside the current turn (Claude only: `records.tool_outcome` is where that gate lives), output-token totals, and a transcript's first timestamp. The scan state carries `first_ts` and `scanned_from_zero`; crossing the unscanned-delta budget makes the latter false for that entry, so a bounded tail or rebuilt oversized entry can never publish its later first record as the session start or a partial lifetime token total. The current-turn total has a separate completeness guard and stays withheld until the forward scan observes that turn's opening boundary. The model and loop readings still use the backward context pass, while the start and token totals deliberately do not. | | `claude_data.py` | Claude transcript reads shared by the collector and the hook path, including the model a session ran on and the one each of its sidechain children ran on, kept apart because the `isSidechain` flag inverts between the two. It also reads a child's first bounded JSON record for its start stamp; mtime remains last activity and never substitutes for a start. `session_instruction` is the Claude half of the instruction line, walking backward beside `session_title` because that title is generated once from the opening prompt and never refreshed. | -| `spacedock.py` | Spacedock workflow and entity cartography. `tool_result_text` is the provenance gate the whole read surface rests on: a boot envelope counts only when it arrives as command output, and it does that in three transcript shapes, one per harness. Codex's is a `function_call_output` or `custom_tool_call_output` payload, which is why it was missing for so long; see [`design-spacedock.md`](design-spacedock.md) decision S-6 for the measured payload shapes and what accepting a `function_call`'s arguments instead would have cost. | +| `spacedock.py` | Spacedock workflow and entity cartography. `tool_result_text` is the provenance gate the whole read surface rests on: a boot envelope counts only when it arrives as command output, and it does that in three transcript shapes, one per harness. Codex's is a `function_call_output` or `custom_tool_call_output` payload, which is why it was missing for so long; see [`design-spacedock.md`](design-spacedock.md) decision S-6 for the measured payload shapes and what accepting a `function_call`'s arguments instead would have cost. Provenance settles where an envelope may come from, not what it looks like there: S-7 records that no real session pastes the raw JSON, so the envelope is also read as the key/value rendering a session printed, under the same `command: boot` gate and the same downstream guards. | | `observer.py` | The on-demand observer: goal, current stage and one open block for one named session, written to a sidecar under `~/.cargento/observer/`. A reader above `spacedock` and `transcripts` rather than beside the collectors, because it answers about one session a person asked about and a collector answers about every session there is. It derives, it does not summarize: the stage comes back through `read_entities`, so the freshness window and the declared-stage discriminator the project-read contract rests on both apply, and `--no-spacedock` withdraws that half exactly as it withdraws a strip. The transcript half reads three record shapes, not one: a nested `message.role` under `type: "message"` (Pi and Droid), `type: "user"`/`"assistant"` (Claude), and a `message` payload under `type: "response_item"` (Codex). The union is additive and takes no `harness` argument, because the three are disjoint across the whole local corpus and the caller has already resolved one file for one requested harness. It could not ship without `records.injected_prompt`, which every user record now goes through: the parser alone published a harness-injected shape as the goal on 51.2% of 457 Codex rollouts and 61.8% of a seeded 400-transcript Claude sample, which is a confident wrong answer in place of the silent one the unfixed parser gave. What survives is rendered by `transcripts.prompt_title`, so a slash command, the one shape that predicate deliberately admits, reads as `/name args` rather than as its wrapper. Sidechain records are excluded on the Claude arm, since a subagent's prompt is its parent's dispatch and not the operator's, and the Codex path excludes the same thing one level up: a subagent thread writes its own rollout under its PARENT's session id, so the transcript resolver drops subagent rollouts before it picks the newest file, the order `collectors/codex.py` already does it in. Two directives are refused rather than published: a generic skill-load opener, read on the raw text, and a bare harness-control slash command (`/clear`, `/login`, `/plugin`), read on what `prompt_title` renders, since the raw and rendered spellings of the same record never meet. The control list lives in `records.harness_control`, shared with the instruction line beneath a session title so the two surfaces cannot disagree about whether `/clear` is an objective, and it is measured names and not the structural rule "a bare command has no arguments, so it has no goal": a skill invoked with no arguments is an objective, and 39 local goals are exactly that. Refusing a directive leaves the one beneath it standing, so a `/clear` typed after real work does not erase it. The head and tail windows are cut apart on byte offsets rather than concatenated and deduped, because the two overlap on any file smaller than their sum and the dedup key falls back to the message text on the 76.4% of Codex user records carrying no `payload.id`, which dropped a verbatim-repeated prompt as a duplicate of its own first occurrence. Disjoint windows make that fallback positional, and the key is left to do the one job it is good at: collapsing a resumed transcript's replayed block, which carries its original ids. The block half is a keyword scan over the newest assistant message only, and the table is self-state phrases rather than bare words, with a trailing word-boundary test so `waiting for you` stops matching `waiting for your`: on the whole local Claude corpus the bare forms produced 7 blocks of which 4 sat in a quoted or fenced span, two of them clipped so the card showed no block language at all. A false block is the one field on the panel a reader would act on, so precision wins over recall by construction. | | `quota.py` | Quota acquisition: the per-vendor credential reads and outbound requests, the receipts a harness pushes in, and the shared cache with its per-vendor floor. The whole outbound network surface (see [design-usage-quota.md](design-usage-quota.md)). | | `dismissals.py` | The sessions the reader marked handled: the store's path, its bounded read and write, and the rule that decides when a mark lapses. The only module that writes user-authored state, and a leaf beside `records` for that reason: `aggregate` subtracts through it before `summary` is counted, `notifications` gates a popup on it, and `http_api` mutates it, and none of those three could depend on it if it depended on any of them. See [design-dismissals.md](design-dismissals.md). | diff --git a/docs/design-spacedock.md b/docs/design-spacedock.md index 4106fec..cd938bf 100644 --- a/docs/design-spacedock.md +++ b/docs/design-spacedock.md @@ -158,40 +158,40 @@ would add a maintenance surface and no session. ## S-7: Read the envelope as a rendering, and walk the window to find it -S-6 settled *where* the envelope may come from. Two later measurements showed both remaining -assumptions about *what it looks like* and *where in the file it sits* were wrong, and each was -independently fatal — the strip published nothing at all while a workflow was plainly running, and +S-6 settled *where* the envelope may come from. Two later measurements showed that both remaining +assumptions, about *what it looks like* and *where in the file it sits*, were wrong. Each was +independently fatal. The strip published nothing at all while a workflow was plainly running, and said so as "A workflow exists, but nothing is fresh enough to show", which reads like a freshness problem and is not one. -**It arrives rendered, not raw.** The reader looked for the literal `{"command"` of a JSON object. -The first officer's own skill tells it to run `status --boot --identify --json` and "consume JSON, -not the human table" — but nothing tells it to *echo* that JSON, and both real first-officer -sessions measured here piped it through a formatter, so what reached the transcript was an indented -key/value rendering and the object never appeared. Across 120 transcripts over 21 days the JSON -branch matched exactly one file, and that one was this repository's own test fixtures catted into a -tool result. The feature had never once fired on a real session. +It arrives rendered, not raw. The reader looked for the literal `{"command"` of a JSON object. The +first officer's own skill tells it to run `status --boot --identify --json` and to "consume JSON, +not the human table", but nothing tells it to *echo* that JSON, and both real first-officer sessions +measured here piped it through a formatter. What reached the transcript was an indented key/value +rendering, and the object never appeared. Across 120 transcripts over 21 days the JSON branch +matched exactly one file, and that one was this repository's own test fixtures catted into a tool +result. The feature had never once fired on a real session. So the fields are read line by line rather than decoded, which survives any rendering that keeps -`key: value`. The trust model is unchanged, deliberately: a rendered path is gated on a top-level -`command: boot` exactly as the JSON branch is gated on `envelope["command"] == "boot"`, only -column-0 keys are read so a nested decoy cannot nominate one, and every downstream guard — the -`_usable_dir` shape check, canonicalisation, the symlink and identity checks, and -`commissioned-by: spacedock@` — still stands between an extracted path and anything published. - -**It is not in the head.** The scan read the first `spacedock_boot_scan_bytes` on the reasoning that -boot output is written once at session start, so the read could be amortised on `(path, size)`. A -first officer greets and discovers before it boots: the two sessions measured booted at 69% and 73% -of the way through their transcripts, at bytes 803,503 and 821,199 of files near 1.1 MB. Claude Code -writes single records up to 109 KB, so a head window is small in lines even when it is large in +`key: value`. The trust model is unchanged, deliberately. A rendered path is gated on a top-level +`command: boot` exactly as the JSON branch is gated on `envelope["command"] == "boot"`; only +column-0 keys are read, so a nested decoy cannot nominate one; and every downstream guard still +stands between an extracted path and anything published, meaning the `_usable_dir` shape check, +canonicalisation, the symlink and identity checks, and `commissioned-by: spacedock@`. + +It is also not in the head. The scan read the first `spacedock_boot_scan_bytes` on the reasoning +that boot output is written once at session start, so the read could be amortised on `(path, size)`. +A first officer greets and discovers before it boots: the two sessions measured booted at 69% and +73% of the way through their transcripts, at bytes 803,503 and 821,199 of files near 1.1 MB. Claude +Code writes single records up to 109 KB, so a head window is small in lines even when it is large in bytes. Raising the cap was rejected. It only moves the guess, and it is the expensive direction: the cache key is `min(size, cap)`, so a live transcript below the cap misses on every write and re-reads the -whole file under the collection lock. Instead the window walks — each pass reads at most -`spacedock_boot_scan_bytes` of not-yet-scanned bytes and remembers how far it reached, keeping the -per-refresh cost the head scan had while covering the file eventually. A pass stops on a line -boundary so an envelope straddling the edge is read whole on the next one, and a record longer than +whole file under the collection lock. Instead the window walks. Each pass reads at most +`spacedock_boot_scan_bytes` of not-yet-scanned bytes and remembers how far it reached, which keeps +the per-refresh cost the head scan had while covering the file eventually. A pass stops on a line +boundary, so an envelope straddling the edge is read whole on the next one, and a record longer than the whole window is stepped over rather than stalled on. The cost is latency, not coverage: an envelope two windows in appears on the second refresh rather than the first.