From 9e042f1f2f4dde1111e3e41d55d8322985aebd3b Mon Sep 17 00:00:00 2001 From: Kent Chen Date: Thu, 27 Aug 2026 15:09:15 +0800 Subject: [PATCH 1/2] fix(spacedock): walk the transcript for boot output instead of one head transcript_boot read only the first spacedock_boot_scan_bytes on the stated assumption that boot output is written at session start. That holds for a first officer launched as the agent and not for a session that adopts the role mid-conversation, where boot lands wherever the conversation had reached. On a real transcript the envelope sat 3.3 MB into a 7.0 MB file, so no stage strip ever appeared. Each pass now reads about one budget of whole lines from a per-path cursor and keeps what it found, making the cost once per byte rather than once per byte per refresh. Whole lines through readline, because a record split across passes parses as neither and a tool result carrying an envelope can outgrow the budget on its own. A trailing line without its newline is left for the next pass, since transcripts are read while a harness writes them. Closes #221 Signed-off-by: Kent Chen --- cargento/skills/cargento/SKILL.md | 2 +- .../cargento/cargento_runtime/spacedock.py | 85 +++++-- .../skills/cargento/cargento_runtime/state.py | 4 +- cargento/skills/cargento/tests/support.py | 2 +- .../cargento/tests/test_config_diagnostics.py | 2 +- .../skills/cargento/tests/test_spacedock.py | 215 +++++++++++++++++- docs/design-spacedock.md | 37 ++- 7 files changed, 315 insertions(+), 32 deletions(-) diff --git a/cargento/skills/cargento/SKILL.md b/cargento/skills/cargento/SKILL.md index 80807d9..c8858ad 100644 --- a/cargento/skills/cargento/SKILL.md +++ b/cargento/skills/cargento/SKILL.md @@ -316,7 +316,7 @@ Paths 2 and 3 are complementary and can both be installed. Keep `Notification` o - **"This request" ETA** = per-session current-turn estimate shown while Working. Estimated total = median of that session's past turns that lasted at least as long as the current one has so far. Turn boundaries: user prompt → last event before the next prompt (Claude, Gemini, Droid); active-branch user messages for Pi; explicit start/end events (Codex `task_started`/`task_complete`, Copilot `user.message`/`session.task_complete`); or DB message timestamps (OpenCode, Goose). JSONL harnesses use an incremental whole-file scanner (survives turns longer than the transcript tail). Pi retains the latest 50 completed durations. No ETA for Cursor. "running longer than recent turns" = no past turn was this long. Naive by design. A ⚠️ appears when elapsed or estimated total ≥ 15 min (`LONG_TURN_WARN_SEC`). Elapsed measures generation, not waiting: a mid-turn quiet stretch longer than 5 minutes (`TURN_GAP_RESET_SEC` — permission prompt, open question, sleep) re-anchors the clock at the post-gap event. - **Loop detection** = four tool calls in a row (`LOOP_ERROR_RUN_THRESHOLD`) coming back as errors inside one request. **Claude Code only**, because Claude is the only harness that records whether a tool call failed: Codex's tool-output records carry no error field, Copilot writes no tool-end record at all, and Droid's records look right but no failing Droid call has been captured, so nothing is inferred for any of them and their rows never carry the signal. It reads the transcript, never the tool's input, so no command text is held anywhere. Four is measured rather than picked: across the 25 most recent local transcripts, runs of three and runs of four each appeared in the same single session, and five in none — so four costs no detections and keeps clear of the benign runs (an `ls` that found nothing, a `git` in a deleted worktree) that filled the sample. The count is the longest run inside the current request and it clears at your next prompt, not when the failures stop, since a loop that has just gone quiet is what you walked back to the machine to find. It raises no flag of its own — the `long turn` chip and the ⚠️ still fire on duration alone, and a loop only changes what they say when you reach for them, which is the difference between "it is slow" and "it is repeating a failure". Where the request is not long enough for either, the working card and the calm detail panel say it anyway. The pattern is honest but not proof: iterating on a failing test looks the same from outside. - **Est. remaining** (per tracked-task session) = average duration of that session's completed tasks × open task count. Naive by design; "no estimate" until a session has a completed task that took ≥30s. -- **Spacedock stage strip** = one line per in-flight entity, showing that workflow's stages in declaration order with the entity's current stage highlighted. A bold entity name means a worker for it is running now; a plain one is read from the entity's state file or, failing that, the boot snapshot. Only Claude reports workers, so a Pi strip is never bold. In flight means moving: an entity resting on the initial stage or a terminal one is left out unless boot called it dispatchable, so a queue of thirty waiting on `intake` does not crowd out the two being worked. Entities whose state file has not been touched inside the freshness window are history, not work, and are skipped — that is what keeps a long-retired workflow off the card of a first officer that merely discovered it. A long workflow is windowed around the current stage, with `…` standing in for the stages elided. An over-long entity name is elided in the **middle**, never the tail — entities in one workflow share a long prefix and differ only at the end — and hovering it shows the full slug. No strip appears when the boot output is outside the scanned head of a long transcript, when the workflow README cannot be read, or when its frontmatter uses a construct the reader does not model — it renders nothing rather than a guess. +- **Spacedock stage strip** = one line per in-flight entity, showing that workflow's stages in declaration order with the entity's current stage highlighted. A bold entity name means a worker for it is running now; a plain one is read from the entity's state file or, failing that, the boot snapshot. Only Claude reports workers, so a Pi strip is never bold. In flight means moving: an entity resting on the initial stage or a terminal one is left out unless boot called it dispatchable, so a queue of thirty waiting on `intake` does not crowd out the two being worked. Entities whose state file has not been touched inside the freshness window are history, not work, and are skipped — that is what keeps a long-retired workflow off the card of a first officer that merely discovered it. A long workflow is windowed around the current stage, with `…` standing in for the stages elided. An over-long entity name is elided in the **middle**, never the tail — entities in one workflow share a long prefix and differ only at the end — and hovering it shows the full slug. No strip appears when the workflow README cannot be read, or when its frontmatter uses a construct the reader does not model — it renders nothing rather than a guess. On a long transcript the strip can take a few refreshes to arrive rather than appearing on the first: the reader walks the transcript forward a bounded slice at a time looking for the boot output, which lands at the start of a session launched as a first officer and wherever the conversation had reached when one adopts the role mid-way. - **Session title** = the harness's own generated title where it writes one (Claude records these, and they read like "Debug Spacedock workflow steps not displaying"), otherwise a prompt from the session itself — the **opening** one on Claude and Pi, which is what the great majority of titled rows fall back to, and the **newest** one only on Codex, which writes no generated title at all. That difference is why the line beneath the title exists: an opening prompt names the session durably and goes stale as work moves on. It is cleaned up rather than shown raw: a slash command reads as `/plugin` instead of the markup the harness wrapped it in, a dispatched worker's prompt shows the instruction instead of the envelope, absolute paths collapse to their last segment so the path does not eat the whole line, and an over-long title is cut on a word boundary. Relative paths and URLs are left whole, because the repo and PR number in a link are the informative part. Nothing is summarized by a model, so no session text leaves the machine. - **The line beneath the title** (on the opt-in `http://127.0.0.1:4553/?next=true` UI) answers a different question from the title, which is why it is a second line and not a better first one. A title names the session; this names what it is working on now. It always carries a label and the age of the record it came from, because that is what tells you how much to trust it: `asked, 4m:` is the newest thing you told it, `agent, 4m:` is the agent's own statement of what it was starting this turn, and `earlier, 2h:` is an older instruction shown because the newest one was a bare "proceed" that names no work. Where none of those can be established honestly the line is absent and the row shows its title alone, which is the old behaviour: a confident wrong line would mask the project name underneath it for as long as the session lives. Claude keeps its generated title on line 1 even when that title has gone stale, because it is how you recognise the row. Codex shows no second line when its title is already that same newest instruction — not always in full, since 167 of 280 local Codex titles are cut at the title width and 144 of those suppress the line while holding back a median 56 further characters. Repeating a prompt under itself costs a row two lines to say one thing, and the fuller reading is a keystroke away in the session panel. - **Project** = the last two segments of the session's working directory (`spacedock/subspace`), the same rule on every harness so one directory reads identically whichever agent opened it. Bare basename is not enough: sibling worktrees are routinely all named the same thing. Claude is the only harness whose store does not hand over a path — its `projects/` directory name encodes one with every separator replaced by `-`, which cannot be split back apart — so the real working directory is read from the transcript records, and a transcript too young to carry one falls back to that encoded name whole. Cursor reads it from the `meta.json` beside the chat store, and falls back to the harness name when that file records nothing that resolves to a real directory. Two or more sessions on one label that are working or waiting on you are marked `N live` wherever that label is printed — the collision two agents in one checkout cause, which grouping alone left you to spot. The marker claims the label and not the directory: the label is the last two segments, so sibling worktrees read alike, and the tooltip says as much. A label that is only a harness name is never marked, because that is what a collector falls back to when no directory resolved at all — two Cursor windows with unreadable workspaces share the word `cursor` and nothing else. Idle sessions are not counted, and blocked ones are: a stopped session writes nothing, while answering a gate is exactly what sets the other half of a collision writing again. diff --git a/cargento/skills/cargento/cargento_runtime/spacedock.py b/cargento/skills/cargento/cargento_runtime/spacedock.py index 50a8eb7..323f70e 100644 --- a/cargento/skills/cargento/cargento_runtime/spacedock.py +++ b/cargento/skills/cargento/cargento_runtime/spacedock.py @@ -12,7 +12,7 @@ import os import re import stat as stat_module -from typing import TYPE_CHECKING, Any, TypeGuard +from typing import IO, TYPE_CHECKING, Any, TypeGuard from cargento_runtime import records, sessions from cargento_runtime import state as runtime_state @@ -363,34 +363,73 @@ def boot_entity_dir(envelopes: list[dict[str, Any]], workflow_dir: str) -> str: def transcript_boot(config: RuntimeConfig, state: RuntimeState, path: str) -> list[dict[str, Any]]: - """Boot envelopes from a transcript's head, cached per (path, size). - - Boot output is written once at session start and never rewritten, so the - scan is amortised: keying on size lets a still-growing session pick the - envelope up on a later refresh without rescanning an unchanged prefix. + """Boot envelopes from a transcript, scanned forward across refreshes. + + An earlier version read only the first ``spacedock_boot_scan_bytes`` on the + stated assumption that boot output "is written once at session start". That + holds for a first officer launched as the agent, and not for a session that + adopts the role mid-conversation by loading the skill: boot then lands + wherever the conversation had reached. Measured on a real transcript, the + envelope sat 3.3 MB into a 7.0 MB file, so the strip never appeared. + + So the cursor advances instead of the window widening. Each pass reads about + one budget's worth of whole lines from where the last pass stopped and keeps + what it found, which makes the total cost once per byte rather than once per + byte per refresh. Whole lines because a record split across two passes would + parse as neither, and read through ``readline`` rather than a fixed slice + because a tool result carrying an envelope can be longer than the budget on + its own and a fixed slice would stall the cursor on it forever. + + Serialized via the scanner lock, like ``turns.scan_turns``: two concurrent + /api/data requests would otherwise both advance the cursor and each parse + half the new bytes. """ try: size = os.path.getsize(path) except OSError: return [] - key = (path, min(size, config.spacedock_boot_scan_bytes)) - with state.cache_lock: - cached = state.spacedock_boot_cache.get(key) - if cached is not None: - return cached - envelope_records: list[dict[str, Any]] = [] - try: - with open(path, "rb") as handle: - blob = handle.read(config.spacedock_boot_scan_bytes) + with state.scanner_lock: + scan = state.spacedock_boot_scan.get(path) + if scan is None or int(scan["pos"]) > size: # new, truncated or rotated + scan = {"pos": 0, "envelopes": []} + runtime_state.bounded_put( + state.spacedock_boot_scan, path, scan, limit=config.max_cache_entries + ) + envelopes: list[dict[str, Any]] = scan["envelopes"] + if int(scan["pos"]) >= size or len(envelopes) >= config.spacedock_max_boot_records: + # Nothing appended, or the record cap is already met. Leaving the + # cursor where it is keeps a settled transcript at one `stat`. + return list(envelopes) + try: + with open(path, "rb") as handle: + handle.seek(int(scan["pos"])) + blob = _boot_pass_bytes(config, handle) + except OSError: + return list(envelopes) + scan["pos"] = int(scan["pos"]) + len(blob) if b"definition_dir" in blob: - envelope_records = boot_records(config, blob) - except OSError: - return [] - with state.cache_lock: - runtime_state.bounded_put( - state.spacedock_boot_cache, key, envelope_records, limit=config.max_cache_entries - ) - return envelope_records + room = config.spacedock_max_boot_records - len(envelopes) + envelopes.extend(boot_records(config, blob)[:room]) + return list(envelopes) + + +def _boot_pass_bytes(config: RuntimeConfig, handle: IO[bytes]) -> bytes: + """One pass of complete lines, stopping at or just past the byte budget. + + A trailing partial line is left unread so the next pass sees it whole. The + budget is therefore a floor on where a pass stops rather than a ceiling on + what it reads: one line longer than the budget is read whole, because the + alternative is never reading it. + """ + lines: list[bytes] = [] + read = 0 + while read < config.spacedock_boot_scan_bytes: + line = handle.readline() + if not line.endswith(b"\n"): + break # EOF, or a line still being written + lines.append(line) + read += len(line) + return b"".join(lines) def open_regular(path: str) -> int | None: diff --git a/cargento/skills/cargento/cargento_runtime/state.py b/cargento/skills/cargento/cargento_runtime/state.py index d3bfc49..a22803b 100644 --- a/cargento/skills/cargento/cargento_runtime/state.py +++ b/cargento/skills/cargento/cargento_runtime/state.py @@ -86,7 +86,9 @@ class RuntimeState: turn_scan: dict[str, Any] = field(default_factory=dict) agent_class_cache: dict[str, tuple[bool, str, str]] = field(default_factory=dict) spacedock_role_cache: dict[str, str] = field(default_factory=dict) - spacedock_boot_cache: dict[tuple[str, int], list[dict[str, Any]]] = field(default_factory=dict) + # {"pos": int, "envelopes": list} per transcript, advanced under scanner_lock. + # A cursor rather than a keyed cache: see S-6 in docs/design-spacedock.md. + spacedock_boot_scan: dict[str, dict[str, Any]] = field(default_factory=dict) spacedock_workflow_cache: dict[tuple[str, int, int], dict[str, Any] | None] = field( default_factory=dict ) diff --git a/cargento/skills/cargento/tests/support.py b/cargento/skills/cargento/tests/support.py index 3a23c52..33f4f81 100644 --- a/cargento/skills/cargento/tests/support.py +++ b/cargento/skills/cargento/tests/support.py @@ -304,13 +304,13 @@ def clear_state(state: RuntimeState) -> None: state.cwd_cache.clear() state.agent_class_cache.clear() state.spacedock_role_cache.clear() - state.spacedock_boot_cache.clear() state.spacedock_workflow_cache.clear() state.spacedock_entity_cache.clear() state.cursor_metadata_cache.clear() with state.scanner_lock: state.pi_scan.clear() state.turn_scan.clear() + state.spacedock_boot_scan.clear() with state.collect_memo_lock: state.snapshot.clear() with state.usage_fetch_lock: diff --git a/cargento/skills/cargento/tests/test_config_diagnostics.py b/cargento/skills/cargento/tests/test_config_diagnostics.py index 577d909..047c996 100644 --- a/cargento/skills/cargento/tests/test_config_diagnostics.py +++ b/cargento/skills/cargento/tests/test_config_diagnostics.py @@ -448,7 +448,7 @@ def test_runtime_states_retain_start_times_and_isolate_mutable_fields(self) -> N "turn_scan", "agent_class_cache", "spacedock_role_cache", - "spacedock_boot_cache", + "spacedock_boot_scan", "spacedock_workflow_cache", "spacedock_entity_cache", "cursor_metadata_cache", diff --git a/cargento/skills/cargento/tests/test_spacedock.py b/cargento/skills/cargento/tests/test_spacedock.py index b8616f2..a5a1136 100644 --- a/cargento/skills/cargento/tests/test_spacedock.py +++ b/cargento/skills/cargento/tests/test_spacedock.py @@ -426,9 +426,10 @@ def setUp(self) -> None: _config, state = runtime() with state_of().cache_lock: state.spacedock_workflow_cache.clear() - state.spacedock_boot_cache.clear() state.spacedock_role_cache.clear() state.spacedock_entity_cache.clear() + with state_of().scanner_lock: + state.spacedock_boot_scan.clear() def workflow(self, body: str | None = None) -> Path: holder = tempfile.TemporaryDirectory(prefix="cargento-sd-") @@ -915,3 +916,215 @@ def test_a_workflow_with_no_state_directory_still_costs_no_walk(self) -> None: self.assertEqual( [], spacedock.session_workflows(config, state, boot, [], time.time(), 3600) ) + + +class SpacedockBootCursorTest(unittest.TestCase): + """``transcript_boot`` walks a transcript forward across refreshes. + + The reader used to take one fixed head, on the assumption that boot output + lands at session start. A session that adopts the first-officer role + mid-conversation breaks that assumption: the envelope then sits wherever the + conversation had reached, and on a real 7.0 MB transcript that was 3.3 MB in. + """ + + ENVELOPE = ( + '{"command":"boot","id_style":"slug",' + '"dispatchable":[{"slug":"drc-1","current":"review","next":"posted"}],' + '"definition_dir":"/w/one","entity_dir":"/w/one/.state"}' + ) + + def boot_line(self, prefix: str = "") -> bytes: + """One transcript record carrying the envelope as tool output.""" + return ( + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "content": prefix + "=== BOOT ===\n" + self.ENVELOPE, + } + ] + }, + } + ).encode() + + b"\n" + ) + + def filler_line(self, size: int) -> bytes: + """A record with no envelope in it, sized to push the cursor along.""" + return ( + json.dumps( + {"type": "user", "message": {"content": [{"type": "text", "text": "x" * size}]}} + ).encode() + + b"\n" + ) + + def transcript(self, body: bytes) -> str: + holder = tempfile.TemporaryDirectory(prefix="cargento-boot-") + self.addCleanup(holder.cleanup) + path = Path(holder.name) / "session.jsonl" + path.write_bytes(body) + return str(path) + + def drain(self, config: Any, state: Any, path: str, *, limit: int = 50) -> list[dict[str, Any]]: + """Call until the cursor reaches EOF, the way successive refreshes do.""" + found: list[dict[str, Any]] = [] + for _ in range(limit): + found = spacedock.transcript_boot(config, state, path) + if int(state.spacedock_boot_scan[path]["pos"]) >= os.path.getsize(path): + return found + raise AssertionError("cursor never reached EOF") + + def test_envelope_past_the_first_pass_is_still_found(self) -> None: + """The regression. One pass' budget of filler, then the envelope: the + first pass must miss it and a later one must find it. Asserting the miss + matters as much as the find — it is what proves the walk moved, rather + than the whole file having been read in one go.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + path = self.transcript(self.filler_line(4_000) * 3 + self.boot_line()) + + self.assertEqual([], spacedock.transcript_boot(config, state, path)) + self.assertEqual( + ["/w/one"], spacedock.workflow_dirs(config, self.drain(config, state, path)) + ) + + def test_a_record_longer_than_one_pass_is_read_whole(self) -> None: + """A tool result carrying a boot envelope is exactly the kind of record + that outgrows the pass budget. A fixed-size slice would find no newline + in it, so the cursor would sit on that line for the life of the session; + ``readline`` reads it whole instead.""" + config, state = make_runtime(spacedock_boot_scan_bytes=512) + path = self.transcript(self.boot_line("y" * 4_000)) + + self.assertEqual(1, len(self.drain(config, state, path))) + + def test_a_record_straddling_the_budget_is_not_split(self) -> None: + """The envelope's own record begins inside one pass and ends in the next. + Handing half of it to the parser twice would parse neither half.""" + config, state = make_runtime(spacedock_boot_scan_bytes=1_024) + head = self.filler_line(900) + self.assertLess(len(head), 1_024) # so the boot record spans the boundary + path = self.transcript(head + self.boot_line("z" * 600)) + + self.assertEqual(1, len(self.drain(config, state, path))) + + def test_appended_bytes_are_scanned_without_rereading_the_prefix(self) -> None: + """A live session appends. The cursor picks up from where it stopped, so + an envelope written after the first refresh is found on a later one.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + path = self.transcript(self.filler_line(100)) + + self.assertEqual([], self.drain(config, state, path)) + before = int(state.spacedock_boot_scan[path]["pos"]) + with Path(path).open("ab") as handle: + handle.write(self.boot_line()) + + self.assertEqual(1, len(self.drain(config, state, path))) + self.assertGreater(int(state.spacedock_boot_scan[path]["pos"]), before) + + def test_a_half_written_record_is_left_for_the_next_pass(self) -> None: + """Transcripts are read while a harness is writing them, so the last line + is routinely incomplete. Consuming it would advance the cursor past a + record that had not arrived yet, and the envelope inside it would never + be parsed once the rest landed.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + whole = self.boot_line() + cut = len(whole) // 2 + path = self.transcript(whole[:cut]) + + self.assertEqual([], spacedock.transcript_boot(config, state, path)) + self.assertEqual(0, int(state.spacedock_boot_scan[path]["pos"])) # nothing consumed + + with Path(path).open("ab") as handle: + handle.write(whole[cut:]) + + self.assertEqual(1, len(self.drain(config, state, path))) + + def test_a_settled_transcript_costs_no_read(self) -> None: + """Once the cursor is at EOF the next refresh must not open the file. + That is the cost claim S-5 makes for every Pi session, and an + incremental reader keeps it only if it stops at EOF.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + path = self.transcript(self.boot_line()) + first = self.drain(config, state, path) + + with ( + mock.patch.object(Path, "open", side_effect=AssertionError("reopened")), + mock.patch("builtins.open", side_effect=AssertionError("reopened")), + ): + again = spacedock.transcript_boot(config, state, path) + + self.assertEqual(first, again) + self.assertEqual(1, len(again)) + + def test_truncation_restarts_the_walk(self) -> None: + """A rotated or truncated transcript leaves the cursor past the end. It + has to restart, or the new file is never read at all.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + path = self.transcript(self.filler_line(3_000) + self.boot_line()) + self.assertEqual(1, len(self.drain(config, state, path))) + + Path(path).write_bytes(self.boot_line()) + stale = int(state.spacedock_boot_scan[path]["pos"]) + + found = self.drain(config, state, path) + + self.assertGreater(stale, os.path.getsize(path)) # the cursor was past the end + # One envelope, not two: the reset drops what the old file had yielded + # rather than carrying it onto a file that no longer contains it. + self.assertEqual(1, len(found)) + self.assertEqual(os.path.getsize(path), int(state.spacedock_boot_scan[path]["pos"])) + + def test_the_record_cap_holds_across_passes(self) -> None: + """The cap bounds what one session can publish, and the accumulator has + to enforce it. ``boot_records`` caps each pass on its own, so a cap + applied only per pass still lets several passes sum past it: here one + envelope in the first pass and three in the second would total four + against a cap of three. Reaching the cap also pins the cursor, so a + transcript full of envelopes is not walked forever.""" + config, state = make_runtime(spacedock_boot_scan_bytes=1_024, spacedock_max_boot_records=3) + first_pass = self.boot_line() + self.filler_line(800) + self.assertGreater(len(first_pass), 1_024) # so the next boot lines land in pass two + path = self.transcript(first_pass + self.boot_line() * 3) + + self.assertEqual(1, len(spacedock.transcript_boot(config, state, path))) + found = spacedock.transcript_boot(config, state, path) + pinned = int(state.spacedock_boot_scan[path]["pos"]) + + self.assertEqual(3, len(found)) + self.assertEqual(3, len(spacedock.transcript_boot(config, state, path))) + self.assertEqual(pinned, int(state.spacedock_boot_scan[path]["pos"])) + + def test_the_cap_pins_the_cursor_short_of_the_end(self) -> None: + """The cap has to stop the walk on its own account, not only because the + file happened to run out. Here the cap is met with bytes still unread: + the cursor must stay put, or a transcript whose cap is met early goes on + being read to the end on every refresh for nothing.""" + config, state = make_runtime(spacedock_boot_scan_bytes=1_024, spacedock_max_boot_records=3) + path = self.transcript(self.boot_line() * 3 + self.filler_line(800) * 4) + + self.assertEqual(3, len(spacedock.transcript_boot(config, state, path))) + pinned = int(state.spacedock_boot_scan[path]["pos"]) + self.assertLess(pinned, os.path.getsize(path)) # there are bytes left + + self.assertEqual(3, len(spacedock.transcript_boot(config, state, path))) + self.assertEqual(pinned, int(state.spacedock_boot_scan[path]["pos"])) + + def test_the_accumulator_is_not_handed_to_callers(self) -> None: + """Callers mutating the returned list would corrupt the cursor's own + record of what it has found.""" + config, state = make_runtime(spacedock_boot_scan_bytes=2_048) + path = self.transcript(self.boot_line()) + + returned = self.drain(config, state, path) + returned.clear() + + self.assertEqual(1, len(spacedock.transcript_boot(config, state, path))) + + def test_an_unreadable_transcript_yields_nothing(self) -> None: + """Defensive parsing: a missing file is not an error the collector sees.""" + config, state = make_runtime() + + self.assertEqual([], spacedock.transcript_boot(config, state, "/nonexistent/session.jsonl")) diff --git a/docs/design-spacedock.md b/docs/design-spacedock.md index 21aaa5d..ab6d827 100644 --- a/docs/design-spacedock.md +++ b/docs/design-spacedock.md @@ -99,8 +99,9 @@ officer, and the same payload carries the paths. Only a first officer runs `spac so presence is good evidence, and it needs no new source. It costs two things, both accepted rather than overlooked. Every Pi session pays a bounded transcript -head scan on refresh, where Claude pays a cached lookup and stops; the scan is capped at -`spacedock_boot_scan_bytes` and cached on `(path, size)`, so a settled transcript costs one `stat`. +scan on refresh, where Claude pays a cached lookup and stops; each pass reads about +`spacedock_boot_scan_bytes` of whole lines from a per-path cursor (S-6), so a settled transcript +costs one `stat` and a growing one costs its new bytes once. And classification now depends on tool output rather than a launch-time declaration, which is a weaker signal: tool output is whatever a tool printed. The guards that matter sit downstream and are unchanged, so a crafted envelope still has to survive path canonicalisation, the symlink and @@ -111,6 +112,32 @@ today and strands every existing session. Rejected: hoisting classification abov which buys nothing while each harness answers the question from a different field, and moves harness knowledge into a module whose whole point is not having any. +## S-6: The boot envelope is reached with a cursor, not a head + +The reader used to take one fixed 512 KB head, on the stated ground that boot output "is written +once at session start". That is true of a first officer launched as the agent. It is false of a +session that adopts the role mid-conversation by loading the skill, which is a shape the product +supports and which Spacedock's own documentation leads a captain to: boot then runs wherever the +conversation had reached. On the transcript this was found from, the envelope sat 3.3 MB into a +7.0 MB file, so no strip ever appeared and the project panel read "declares no workflow". + +Widening the head would only move the cliff. Instead each pass reads about +`spacedock_boot_scan_bytes` from a per-path cursor and keeps what it found, so the total cost is +once per byte rather than once per byte per refresh, and the cliff is gone rather than further away. +A transcript that has been walked to its end costs one `stat`, which is what keeps S-5's per-session +Pi cost where it was; reaching `spacedock_max_boot_records` pins the cursor for the same reason. + +Two details are load-bearing rather than incidental. The pass reads whole lines through `readline`, +not a fixed slice: a record split across two passes parses as neither, and a tool result carrying a +boot envelope can be larger than the budget on its own, which a fixed slice would stall the cursor +on forever. And a trailing line without its newline is left unread, because transcripts are read +while a harness is writing them and consuming half a record advances the cursor past a line that +never gets parsed. + +The cursor lives in `state.spacedock_boot_scan` and advances under `scanner_lock`, like +`turns.scan_turns`. Two concurrent `/api/data` requests would otherwise both advance it and each +parse half the new bytes. + ## Rejected alternatives worth keeping rejected Run `spacedock status` ourselves. It would answer every question above directly and correctly. It @@ -122,8 +149,10 @@ arbitrary side effects. Watch for later `status` envelopes instead of adding a state read. Cheaper in principle, since it adds no new read surface, and it fails in practice. The transcripts examined contained `boot` envelopes and nothing else: a first officer's routine status checks are not emitted as JSON, so -there is no fresher envelope to find. Widening the 512 KiB head scan to the whole transcript would -have made the collection pass quadratic in transcript size for no additional data. +there is no fresher envelope to find. That is still the reason this is rejected, and it is a +different question from reaching the `boot` envelope itself, which S-6 had to widen the scan for. +An earlier note here called whole-transcript scanning quadratic in transcript size; that described +the brace-balancing parser `boot_records` replaced, and it is no longer true of either reader. Require the entity directory to sit inside the workflow directory. This is the natural containment check, and it would silently re-create the original bug for any `split-root` workflow, which From 7eb5c91b646c1f501e4dbda507f7b74243645529 Mon Sep 17 00:00:00 2001 From: Kent Chen Date: Thu, 27 Aug 2026 15:59:35 +0800 Subject: [PATCH 2/2] test(spacedock): assert a deep boot reaches the card through the collector The cursor tests pin transcript_boot on its own. The user-visible claim is a wiring one -- a declared first officer whose boot output sits past the first pass ends up with a stage strip -- so it needs the collector, the workflow read and the entity read exercised together. Reverting the reader to main's head-only read fails nine tests, two of them these. Signed-off-by: Kent Chen --- .../skills/cargento/tests/test_spacedock.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/cargento/skills/cargento/tests/test_spacedock.py b/cargento/skills/cargento/tests/test_spacedock.py index a5a1136..7468f87 100644 --- a/cargento/skills/cargento/tests/test_spacedock.py +++ b/cargento/skills/cargento/tests/test_spacedock.py @@ -11,7 +11,9 @@ from unittest import mock from cargento_runtime import spacedock +from cargento_runtime.collectors import claude as claude_collector +from . import support from .support import ( make_runtime, runtime, @@ -1128,3 +1130,115 @@ def test_an_unreadable_transcript_yields_nothing(self) -> None: config, state = make_runtime() self.assertEqual([], spacedock.transcript_boot(config, state, "/nonexistent/session.jsonl")) + + +class SpacedockDeepBootCollectorTest(unittest.TestCase): + """The wiring behind the cursor, through the Claude collector. + + `SpacedockBootCursorTest` pins `transcript_boot` on its own. This one asserts + the user-visible claim: a declared first officer whose boot output sits past + the first pass ends up with a stage strip on its card, which is a claim about + the collector, the workflow read and the entity read together. + """ + + README = ( + "---\n" + "commissioned-by: spacedock@0.22.0\n" + "state: .spacedock-state\n" + "stages:\n" + " states:\n" + " - name: intake\n" + " initial: true\n" + " - name: review\n" + " - name: posted\n" + " terminal: true\n" + "---\n" + ) + + def setUp(self) -> None: + _config, state = runtime() + with state.cache_lock: + state.spacedock_workflow_cache.clear() + state.spacedock_role_cache.clear() + state.spacedock_entity_cache.clear() + state.agent_class_cache.clear() + with state.scanner_lock: + state.spacedock_boot_scan.clear() + + def build(self, *, filler_passes: int) -> tuple[str, Path]: + """A workflow with one entity mid-flight, and an FO transcript whose boot + envelope sits behind ``filler_passes`` passes of unrelated records.""" + holder = tempfile.TemporaryDirectory(prefix="cargento-deep-") + self.addCleanup(holder.cleanup) + root = Path(holder.name).resolve() / "wf" + root.mkdir() + (root / "README.md").write_text(self.README, encoding="utf-8") + entity_dir = root / ".spacedock-state" + entity_dir.mkdir() + (entity_dir / "drc-1.md").write_text( + '---\nid:\ntitle: "a thing"\nstatus: review\n---\n\n# report\n', encoding="utf-8" + ) + + envelope = json.dumps( + { + "command": "boot", + "id_style": "slug", + "dispatchable": [{"slug": "drc-1", "current": "review", "next": "posted"}], + "definition_dir": str(root), + "entity_dir": str(entity_dir), + } + ) + head = json.dumps({"agentSetting": spacedock.SPACEDOCK_FO}).encode() + b"\n" + filler = ( + json.dumps( + {"type": "user", "message": {"content": [{"type": "text", "text": "x" * 4_000}]}} + ).encode() + + b"\n" + ) + boot = ( + json.dumps( + { + "type": "user", + "message": { + "content": [{"type": "tool_result", "content": "=== BOOT ===\n" + envelope}] + }, + } + ).encode() + + b"\n" + ) + path = Path(holder.name) / "session.jsonl" + path.write_bytes(head + filler * filler_passes + boot) + return str(path), root + + def strips(self, transcript: str) -> dict[str, Any] | None: + config, state = runtime() + return claude_collector.session_spacedock(config, state, transcript, [], time.time(), 3600) + + def test_a_deep_boot_reaches_the_card_after_a_few_refreshes(self) -> None: + transcript, _root = self.build(filler_passes=4) # ~16 KB of filler + + with mock.patch.dict(support.CONFIG_OVERRIDES, {"spacedock_boot_scan_bytes": 4_200}): + first = self.strips(transcript) + self.assertEqual("first-officer", (first or {})["role"]) + self.assertEqual([], (first or {})["workflows"]) # boot not reached yet + + for _ in range(12): + latest = self.strips(transcript) + if (latest or {})["workflows"]: + break + + workflows = (latest or {})["workflows"] + self.assertEqual(1, len(workflows)) + self.assertEqual("wf", workflows[0]["workflow"]) + self.assertEqual(["intake", "review", "posted"], workflows[0]["stages"]) + self.assertEqual(["drc-1"], [e["slug"] for e in workflows[0]["entities"]]) + self.assertEqual("review", workflows[0]["entities"][0]["stage"]) + + def test_a_shallow_boot_still_reaches_the_card_on_the_first_pass(self) -> None: + """The launch shape that already worked must not have become slower to + appear: an FO launched as the agent boots at the top of its transcript.""" + transcript, _root = self.build(filler_passes=0) + + first = self.strips(transcript) + + self.assertEqual("wf", (first or {})["workflows"][0]["workflow"])