From 32f6b22e60d9a22c96b1713c530e600aba617ad4 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 18:21:14 +0200 Subject: [PATCH 01/98] feat(flow): key recording sessions by flow path instead of module globals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool-server is a host-wide singleton: every MCP client, subagent and CLI call on the machine shares one process and, until now, one set of recording globals. A second flow-start-recording clobbered the first, any caller could finish a recording it never named (flow-finish-recording took no arguments at all), and replaying a flow rebound the shared project root. Recording state now lives in a Map keyed by /.argent/flows/ .yaml — the identity of the artifact being built — and flow-add-step, flow-add-echo and flow-finish-recording each take name + project_root, so every call is self-contained. Appends are serialized per session, since two in-flight calls on one recording are now legitimate. resolveFlowFilePath is pure, so the replay path no longer mutates anything shared, which retires FLOW_PROJECT_ROOT_REQUIRED. stop-all-simulator-servers gains a devices scope: every agent is told to call it at session end, and unscoped it tore down other agents' devtools mid-recording — degrading their flows to coordinate taps, silently. --- packages/registry/src/failure-codes.ts | 1 - .../src/tools/flows/flow-add-step.ts | 39 ++- .../src/tools/flows/flow-finish-recording.ts | 44 ++- .../src/tools/flows/flow-insert-echo.ts | 30 +- .../tool-server/src/tools/flows/flow-run.ts | 9 +- .../src/tools/flows/flow-start-recording.ts | 47 ++- .../tool-server/src/tools/flows/flow-utils.ts | 312 +++++++++++------- .../simulator/stop-all-simulator-servers.ts | 60 +++- 8 files changed, 364 insertions(+), 178 deletions(-) diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index ec8f41dd3..8f2139597 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -218,7 +218,6 @@ export const FAILURE_CODES = { SCREEN_RECORDING_STREAM_UNAVAILABLE: "SCREEN_RECORDING_STREAM_UNAVAILABLE", SCREEN_RECORDING_FFMPEG_NOT_FOUND: "SCREEN_RECORDING_FFMPEG_NOT_FOUND", - FLOW_PROJECT_ROOT_REQUIRED: "FLOW_PROJECT_ROOT_REQUIRED", FLOW_PROJECT_ROOT_INVALID: "FLOW_PROJECT_ROOT_INVALID", FLOW_NAME_INVALID: "FLOW_NAME_INVALID", FLOW_NO_ACTIVE_RECORDING: "FLOW_NO_ACTIVE_RECORDING", diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index fe2ac948e..ec146671e 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -3,9 +3,8 @@ import * as fs from "node:fs/promises"; import * as path from "node:path"; import { FAILURE_CODES, FailureError, type Registry, type ToolDefinition } from "@argent/registry"; import { - getActiveFlow, - getRecordingSession, - appendStepToActiveFlow, + requireRecordingSession, + appendStepToFlow, parseFlow, assertSafeFlowName, classifyOnDiskSpelling, @@ -29,6 +28,14 @@ import { } from "../../utils/ui-tree-match"; const zodSchema = z.object({ + name: z + .string() + .describe("Name of the flow being recorded — the one passed to flow-start-recording."), + project_root: z + .string() + .describe( + "Absolute path to the project root of the flow being recorded — the same value passed to flow-start-recording. Together with `name` it identifies which recording this step belongs to." + ), command: z.string().describe('MCP tool name (e.g. "tap", "screenshot", "launch-app")'), args: z .string() @@ -302,22 +309,23 @@ async function rewriteSiblingFlowPath( * actually ran, carrying the caller's own project_root. */ async function captureRunTarget( - session: RecordingSession | null, + session: RecordingSession, args: Record ): Promise<{ flow?: string; warning?: string }> { const name = typeof args.name === "string" ? args.name : undefined; if (name === undefined) { return { warning: "flow-execute call had no flow name; kept the raw step" }; } - if (!session || session.persist !== "host") { + if (session.persist !== "host") { return { warning: `kept the raw flow-execute step — run: composition is host-resolved, so a remote recording can't reference "${name}" portably`, }; } try { assertSafeFlowName(name); - // Resolve against the recording's own flows dir (the running flow-execute - // may have mutated the active-project-root global), not getFlowsDir() — + // Resolve against THIS recording's own flows dir, not the project root the + // nested flow-execute ran under: `run:` composes siblings of the flow being + // recorded, which is not necessarily the project that nested call ran in — // and against the recording's REAL file, because the runner resolves the // recorded `run:` against the canonical containing-file directory // (scopeFlowDir in flow-run.ts). When the recording is itself a symlink, @@ -435,20 +443,21 @@ export function createFlowAddStepTool( return { id: "flow-add-step", interaction: { - startedMsg: ({ params }) => `Adding ${params.command} step to recorded flow`, - completedMsg: ({ params }) => `Added ${params.command} step to recorded flow`, + // Name the flow: recordings are concurrent, so several of these lines can + // interleave in one log and "the recorded flow" would not identify which. + startedMsg: ({ params }) => `Adding ${params.command} step to flow ${params.name}`, + completedMsg: ({ params }) => `Added ${params.command} step to flow ${params.name}`, failedMsg: ({ params, failureSignal }) => - `Failed to add ${params.command} step to recorded flow: ${failureSignal.error_code}`, + `Failed to add ${params.command} step to flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Execute a tool call and record it as a step in the active flow. Use when recording a flow with flow-start-recording and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile } on success. If it fails an error is returned and nothing is recorded. + description: `Execute a tool call and record it as a step in the flow named by \`name\` + \`project_root\` (the recording must already be open — see flow-start-recording). Use when recording a flow and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile } on success. If it fails an error is returned and nothing is recorded. If a step was recorded by mistake, edit the .yaml file directly to remove it.`, zodSchema, services: () => ({}), async execute(_services, params, ctx) { - const flowName = getActiveFlow(); + const session = requireRecordingSession(params.project_root, params.name); const args: Record = params.args ? JSON.parse(params.args) : {}; - const session = getRecordingSession(); // A nested flow-execute must never carry a raw flow_path into the live // invoke — it has no boundary metadata there and would be rejected. if (params.command === RUN_TARGET_COMMAND) await rewriteSiblingFlowPath(session, args); @@ -530,10 +539,10 @@ If a step was recorded by mistake, edit the .yaml file directly to remove it.`, }; } - const { flowFile, savedTo } = await appendStepToActiveFlow(step); + const { flowFile, savedTo } = await appendStepToFlow(session, step); return { - message: `Step added to "${flowName}" flow${warning ? ` — ${warning}` : ""}`, + message: `Step added to "${params.name}" flow${warning ? ` — ${warning}` : ""}`, toolResult, flowFile, savedTo, diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index f1ad5c1ff..6730ddb00 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -2,10 +2,8 @@ import { z } from "zod"; import * as fs from "node:fs/promises"; import type { ToolDefinition } from "@argent/registry"; import { - getFlowPath, - getActiveFlow, - getRecordingSession, - clearActiveFlow, + requireRecordingSession, + clearRecordingSession, clientFileDirective, parseFlow, serializeFlow, @@ -41,7 +39,16 @@ function textConditionLabel( : `text ${selector} contains ${JSON.stringify(expected)}`; } -const zodSchema = z.object({}); +const zodSchema = z.object({ + name: z + .string() + .describe("Name of the flow being recorded — the one passed to flow-start-recording."), + project_root: z + .string() + .describe( + "Absolute path to the project root of the flow being recorded — the same value passed to flow-start-recording. Together with `name` it identifies which recording to finish." + ), +}); export const flowFinishRecordingTool: ToolDefinition< z.infer, @@ -57,7 +64,13 @@ export const flowFinishRecordingTool: ToolDefinition< > = { id: "flow-finish-recording", interaction: { - startedMsg: () => "Finishing flow recording", + // Name the flow: other recordings stay live across this call, so an + // unqualified "Finishing flow recording" would not identify which one. + startedMsg: ({ params }) => `Finishing recording of flow ${params.name}`, + // Derived from the resolved path rather than `params.name` so the line + // reports the file that was actually written. Holds in client mode too: + // `path` is still the resolved spelling, it just names a file on the + // client's disk rather than this host's, and only its basename is read. completedMsg: ({ result }) => { const flowName = result.path @@ -66,24 +79,23 @@ export const flowFinishRecordingTool: ToolDefinition< ?.replace(/\.ya?ml$/, "") ?? "flow"; return `Saved recorded flow ${flowName}`; }, - failedMsg: ({ failureSignal }) => - `Failed to finish flow recording: ${failureSignal.error_code}`, + failedMsg: ({ params, failureSignal }) => + `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Finish recording the active flow. Returns a summary of all recorded steps and the final YAML content. Use when you have added all desired steps and want to finalize the flow file. Fails if no active flow recording is in progress. + description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving any other recordings in progress untouched. Returns a summary of all recorded steps and the final YAML content. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. You can still edit the .yaml file directly afterwards to remove or reorder steps.`, zodSchema, services: () => ({}), - async execute(_services, _params) { - const flowName = getActiveFlow(); - const session = getRecordingSession(); + async execute(_services, params) { + const session = requireRecordingSession(params.project_root, params.name); // Host mode re-reads the file so manual edits made during the recording // survive into the summary; in client mode this host never has the file, // so the in-memory copy is the truth and travels back in the directive. - const filePath = session?.filePath ?? getFlowPath(flowName); + const filePath = session.filePath; let flowFile: string; let savedTo: FlowSavedTo; - if (session?.persist === "client") { + if (session.persist === "client") { flowFile = serializeFlow(session.flow); savedTo = clientFileDirective(filePath, flowFile); } else { @@ -147,10 +159,10 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps } }); - clearActiveFlow(); + clearRecordingSession(params.project_root, params.name); return { - message: `Finished recording "${flowName}" flow (${flow.steps.length} steps)`, + message: `Finished recording "${params.name}" flow (${flow.steps.length} steps)`, path: filePath, executionPrerequisite: flow.executionPrerequisite, steps: flow.steps.length, diff --git a/packages/tool-server/src/tools/flows/flow-insert-echo.ts b/packages/tool-server/src/tools/flows/flow-insert-echo.ts index cf773e586..e70af3cc1 100644 --- a/packages/tool-server/src/tools/flows/flow-insert-echo.ts +++ b/packages/tool-server/src/tools/flows/flow-insert-echo.ts @@ -1,8 +1,16 @@ import { z } from "zod"; import type { ToolDefinition } from "@argent/registry"; -import { getActiveFlow, appendStepToActiveFlow, type FlowSavedTo } from "./flow-utils"; +import { requireRecordingSession, appendStepToFlow, type FlowSavedTo } from "./flow-utils"; const zodSchema = z.object({ + name: z + .string() + .describe("Name of the flow being recorded — the one passed to flow-start-recording."), + project_root: z + .string() + .describe( + "Absolute path to the project root of the flow being recorded — the same value passed to flow-start-recording. Together with `name` it identifies which recording this echo belongs to." + ), message: z.string().describe("Message to echo when the flow is replayed"), }); @@ -12,26 +20,28 @@ export const flowInsertEchoTool: ToolDefinition< > = { id: "flow-add-echo", interaction: { - startedMsg: () => "Adding note to recorded flow", - completedMsg: () => "Added note to recorded flow", - failedMsg: ({ failureSignal }) => - `Failed to add note to recorded flow: ${failureSignal.error_code}`, + // Name the flow: recordings are concurrent, so several of these lines can + // interleave in one log and "the recorded flow" would not identify which. + startedMsg: ({ params }) => `Adding note to flow ${params.name}`, + completedMsg: ({ params }) => `Added note to flow ${params.name}`, + failedMsg: ({ params, failureSignal }) => + `Failed to add note to flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Record an echo step in the active flow. Echo steps print a message when the flow is replayed — useful as labels between tool calls. + description: `Record an echo step in the flow named by \`name\` + \`project_root\`. Echo steps print a message when the flow is replayed — useful as labels between tool calls. Use when you want to annotate a recorded flow with a human-readable label or checkpoint message. -Returns { message, flowFile }. Fails if no active flow recording is in progress.`, +Returns { message, flowFile }. Fails if that flow has no recording in progress.`, zodSchema, services: () => ({}), async execute(_services, params) { - const flowName = getActiveFlow(); + const session = requireRecordingSession(params.project_root, params.name); - const { flowFile, savedTo } = await appendStepToActiveFlow({ + const { flowFile, savedTo } = await appendStepToFlow(session, { kind: "echo", message: params.message, }); return { - message: `Echo added to "${flowName}" flow`, + message: `Echo added to "${params.name}" flow`, flowFile, savedTo, }; diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 688fbeafe..d615a2f98 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -29,7 +29,6 @@ import { getFlowPath, parseFlow, runTargetName, - setActiveProjectRoot, type FlowFile, type FlowSelector, type FlowStep, @@ -2195,6 +2194,10 @@ function errMsg(err: unknown): string { * carries, and the name is what keys the report and `__baselines__/` (see * {@link classifyOnDiskSpelling}). Name and project_root are validated in every * branch. + * + * Resolution is pure: it reads and mutates no shared state, so replaying a flow + * in one project can never rebind the paths of a recording in progress in + * another (or in the same project on another agent). */ export async function resolveFlowSource( params: { @@ -2218,8 +2221,6 @@ export async function resolveFlowSource( }); } - setActiveProjectRoot(params.project_root); - if (params.flow_path !== undefined) { if (flowPathInput?.viaUpload) { throw new FailureError( @@ -2390,7 +2391,7 @@ export async function resolveFlowSource( const flowName = params.name!; assertSafeFlowName(flowName); - const expected = getFlowPath(flowName); + const expected = getFlowPath(params.project_root, flowName); // A path the boundary materialized from uploaded content is a fresh temp // file this process itself created (see file-inputs.ts) — trusted as-is, and // returned ahead of the on-disk-spelling gate below deliberately: the only diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 02eaed494..60f7225e8 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -4,8 +4,6 @@ import type { FileInputSpec, ToolDefinition } from "@argent/registry"; import { getFlowsDir, getFlowPath, - getActiveFlowOrNull, - setActiveProjectRoot, startRecordingSession, clientFileDirective, serializeFlow, @@ -48,7 +46,13 @@ const fileInputs: FileInputSpec[] = [ export const flowStartRecordingTool: ToolDefinition< z.infer, - { message: string; previousFlow?: string; flowFile: string; savedTo: FlowSavedTo } + { + message: string; + restarted?: true; + discardedSteps?: number; + flowFile: string; + savedTo: FlowSavedTo; + } > = { id: "flow-start-recording", interaction: { @@ -58,9 +62,14 @@ export const flowStartRecordingTool: ToolDefinition< }, description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory. Use when you want to capture a reusable sequence of device interactions for later replay. -Returns { message, flowFile, savedTo } and optionally { previousFlow } if a prior recording was abandoned. +Returns { message, flowFile, savedTo }, plus { restarted, discardedSteps } when this call re-started a recording of the SAME flow — the earlier take is discarded and its .yaml is reset to an empty flow, so re-record from the top rather than expecting to resume. Fails if the .argent/flows/ directory cannot be created or the flow file cannot be written. +Recordings are independent: several flows can be recorded at once (different +names, different projects, different devices) with no cross-talk. Every +subsequent recording tool takes the same \`name\` + \`project_root\` to say which +one it is addressing. + After starting, use flow-add-step to append tool calls — each step is executed LIVE so you can verify it works before it gets recorded. For a self-contained e2e flow, record a restart-app of the app under test as the FIRST step (captured @@ -74,10 +83,7 @@ to remove or reorder steps.`, fileInputs, services: () => ({}), async execute(_services, params, ctx) { - setActiveProjectRoot(params.project_root); - const previousFlow = getActiveFlowOrNull(); - - const filePath = getFlowPath(params.name); + const filePath = getFlowPath(params.project_root, params.name); // A recording's type emerges from its steps: recording a `restart-app` // first makes it an e2e flow (captured as a leading `launch` step by // flow-add-step); declaring an executionPrerequisite documents a fragment. @@ -95,21 +101,32 @@ to remove or reorder steps.`, let savedTo: FlowSavedTo; if (persist === "host") { - await fs.mkdir(getFlowsDir(), { recursive: true }); + await fs.mkdir(getFlowsDir(params.project_root), { recursive: true }); await fs.writeFile(filePath, flowFile, "utf8"); savedTo = filePath; } else { savedTo = clientFileDirective(filePath, flowFile); } - startRecordingSession(params.name, { persist, filePath, flow }); + const replaced = startRecordingSession({ + name: params.name, + projectRoot: params.project_root, + persist, + filePath, + flow, + }); - if (previousFlow && previousFlow !== params.name) { + // Only a same-key restart replaces anything — the documented "re-record it + // to fix it" workflow. Starting a *different* flow abandons nothing, so + // there is no longer a switched-away-from flow to report. + if (replaced) { + const discardedSteps = replaced.flow.steps.length; return { message: - `Switched active flow from "${previousFlow}" to "${params.name}". ` + - `Recording "${previousFlow}" was abandoned - but the flow .yaml file has been saved to disk. ` + - `Now recording "${params.name}".`, - previousFlow, + `Restarted recording "${params.name}" — the previous take ` + + `(${discardedSteps} step${discardedSteps === 1 ? "" : "s"}) was discarded and ` + + `${filePath} reset to an empty flow.`, + restarted: true, + discardedSteps, flowFile, savedTo, }; diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index d659bc2d9..e21e90e49 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -30,38 +30,12 @@ const FLOWS_DIR_NAME = path.join(".argent", "flows"); // ── Paths ──────────────────────────────────────────────────────────── -// ── Active session state ───────────────────────────────────────────── - -let activeFlowName: string | null = null; -let activeProjectRoot: string | null = null; - /** - * Where the active recording's YAML is persisted: - * - `"host"` — this process writes `/.argent/flows/.yaml` - * directly (the original behavior; correct whenever the caller's - * project root is on this machine). - * - `"client"` — the caller's project root is NOT on this machine (remote - * tool-server). The flow lives in memory here and every mutating - * tool returns a {@link ClientFileDirective} so the *client* - * writes the YAML into the agent's project. + * Validate a caller-supplied `project_root`. Every path helper below joins the + * flows dir under this root, so the two rules it enforces (absolute, no "..") + * are what keep a recording's files inside the project the agent named. */ -export type FlowPersistMode = "host" | "client"; - -export interface RecordingSession { - persist: FlowPersistMode; - /** - * Absolute path of the flow file as the CALLER knows it. A real host path in - * "host" mode; in "client" mode it is only echoed back inside the directive - * (it names a file on the client's machine, never touched here). - */ - filePath: string; - /** In-memory flow content — authoritative in "client" mode. */ - flow: FlowFile; -} - -let recordingSession: RecordingSession | null = null; - -export function setActiveProjectRoot(root: string): void { +export function assertValidProjectRoot(root: string): void { if (!path.isAbsolute(root)) { throw new FailureError( `project_root must be an absolute path (got "${root}"). ` + @@ -87,36 +61,22 @@ export function setActiveProjectRoot(root: string): void { error_kind: "validation", }); } - activeProjectRoot = root; -} - -export function requireActiveProjectRoot(): string { - if (!activeProjectRoot) { - throw new FailureError( - "No active project root. The calling flow tool must pass project_root before any path is resolved.", - { - error_code: FAILURE_CODES.FLOW_PROJECT_ROOT_REQUIRED, - failure_stage: "flow_project_root_require", - failure_area: "tool_server", - error_kind: "validation", - } - ); - } - return activeProjectRoot; } -export function clearActiveProjectRoot(): void { - activeProjectRoot = null; -} - -/** The flows dir under an explicit root — for callers that must not resolve - * against the active-project-root global (see flow-add-step). */ +/** + * The flows dir under an explicit root, as pure path math. It validates + * nothing, so a caller that already rejects a bad root with its own + * tool-specific message (see flow-add-step) does not also raise a second, + * differently-worded one from here. + */ export function flowsDirFor(root: string): string { return path.join(root, FLOWS_DIR_NAME); } -export function getFlowsDir(): string { - return flowsDirFor(requireActiveProjectRoot()); +/** The flows dir under a root that has not been validated yet. */ +export function getFlowsDir(projectRoot: string): string { + assertValidProjectRoot(projectRoot); + return flowsDirFor(projectRoot); } export function assertSafeFlowName(name: string): void { @@ -134,12 +94,20 @@ export function assertSafeFlowName(name: string): void { } } -export function getFlowPath(name: string): string { +/** + * The flow file `/.argent/flows/.yaml`. Pure path math over + * two validated inputs — it reads no shared state, so two callers naming two + * different projects can never collide. `path.join` normalizes the root, so a + * trailing slash cannot mint a second identity for the same file (this path + * doubles as the recording-session key, see {@link startRecordingSession}). + */ +export function getFlowPath(projectRoot: string, name: string): string { + const flowsDir = getFlowsDir(projectRoot); assertSafeFlowName(name); - const filePath = path.join(getFlowsDir(), `${name}.yaml`); + const filePath = path.join(flowsDir, `${name}.yaml`); // Defense-in-depth: ensure the resolved path stays inside the flows // directory even if the regex above is ever weakened. - const rel = path.relative(getFlowsDir(), filePath); + const rel = path.relative(flowsDir, filePath); if (rel.startsWith("..") || path.isAbsolute(rel)) { throw new FailureError(`Invalid flow name "${name}": resolves outside the flows directory.`, { error_code: FAILURE_CODES.FLOW_NAME_INVALID, @@ -199,52 +167,150 @@ export async function classifyOnDiskSpelling(dir: string, base: string): Promise return { state: "case_folded", actual, addressable: FLOW_FILE_NAME_PATTERN.test(actual) }; } -export function setActiveFlow(name: string): void { - activeFlowName = name; +// ── Recording sessions ─────────────────────────────────────────────── + +/** + * Where a recording's YAML is persisted: + * - `"host"` — this process writes `/.argent/flows/.yaml` + * directly (the original behavior; correct whenever the caller's + * project root is on this machine). + * - `"client"` — the caller's project root is NOT on this machine (remote + * tool-server). The flow lives in memory here and every mutating + * tool returns a {@link ClientFileDirective} so the *client* + * writes the YAML into the agent's project. + */ +export type FlowPersistMode = "host" | "client"; + +export interface RecordingSession { + /** Flow name, as passed to every recording tool. */ + name: string; + /** Caller-supplied project root, as passed to every recording tool. */ + projectRoot: string; + persist: FlowPersistMode; + /** + * Absolute path of the flow file as the CALLER knows it. A real host path in + * "host" mode; in "client" mode it is only echoed back inside the directive + * (it names a file on the client's machine, never touched here). + */ + filePath: string; + /** In-memory flow content — authoritative in "client" mode. */ + flow: FlowFile; + /** Serializes appends to this session — see {@link appendStepToFlow}. */ + tail: Promise; + /** Wall-clock of the last touch, for the LRU eviction backstop. */ + lastTouchedAtMs: number; +} + +/** + * Live recordings, keyed by {@link getFlowPath} — the identity of the artifact + * being built. Two sessions on one key mean two writers on one output file (a + * genuine collision); two different keys are independent, so concurrent agents + * recording different flows — in one project or across projects, against one + * device or several — never see each other's state. + * + * The tool-server is a host-wide singleton shared by every MCP client, subagent + * and CLI call on the machine, so this map is the only thing standing between + * two agents and a clobbered flow file. + */ +const recordings = new Map(); + +/** + * Leak backstop only. Sessions are small and auto-spawned servers idle out + * after 30 min, but a long-lived server could accumulate recordings an agent + * started and never finished. Well past any realistic concurrent-agent count, + * so evicting is never something an agent should observe. + */ +const MAX_RECORDINGS = 32; + +function evictIfOverCapacity(): void { + while (recordings.size > MAX_RECORDINGS) { + let oldestKey: string | undefined; + let oldestAt = Infinity; + for (const [key, session] of recordings) { + if (session.lastTouchedAtMs < oldestAt) { + oldestAt = session.lastTouchedAtMs; + oldestKey = key; + } + } + if (oldestKey === undefined) return; + recordings.delete(oldestKey); + } } -/** Begin a recording session (replacing any abandoned one). */ -export function startRecordingSession(name: string, session: RecordingSession): void { - activeFlowName = name; - recordingSession = session; +export interface RecordingSessionInit { + name: string; + projectRoot: string; + persist: FlowPersistMode; + filePath: string; + flow: FlowFile; } -export function getRecordingSession(): RecordingSession | null { - return recordingSession; +/** + * Begin a recording. Returns the session it replaced when one was already live + * on the same key (a re-record of the same flow, which discards the earlier + * take), or null — the common case, including starting a second, unrelated + * recording while others are in progress. + */ +export function startRecordingSession(init: RecordingSessionInit): RecordingSession | null { + const key = getFlowPath(init.projectRoot, init.name); + const previous = recordings.get(key) ?? null; + recordings.set(key, { + ...init, + tail: Promise.resolve(), + lastTouchedAtMs: Date.now(), + }); + evictIfOverCapacity(); + return previous; } -function requireRecordingSession(): RecordingSession { - if (!activeFlowName || !recordingSession) { - throw new FailureError("No active flow. Call flow-start-recording first.", { - error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, - failure_stage: "flow_require_recording", - failure_area: "tool_server", - error_kind: "validation", - }); - } - return recordingSession; +export function getRecordingSession( + projectRoot: string, + name: string +): RecordingSession | undefined { + return recordings.get(getFlowPath(projectRoot, name)); } -/** Returns the active flow name, or null if none is active. */ -export function getActiveFlowOrNull(): string | null { - return activeFlowName; +/** Every live recording, for diagnostics and the not-found error message. */ +export function listActiveRecordings(): { name: string; projectRoot: string; steps: number }[] { + return [...recordings.values()].map((s) => ({ + name: s.name, + projectRoot: s.projectRoot, + steps: s.flow.steps.length, + })); } -export function getActiveFlow(): string { - if (!activeFlowName) { - throw new FailureError("No active flow. Call flow-start-recording first.", { - error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, - failure_stage: "flow_active_recording_require", - failure_area: "tool_server", - error_kind: "validation", - }); +export function requireRecordingSession(projectRoot: string, name: string): RecordingSession { + const session = getRecordingSession(projectRoot, name); + if (!session) { + // Name what was asked for AND what is live: with concurrent recordings the + // usual cause is a typo or the wrong project_root, and the agent can only + // self-correct if it can see the keys that do exist. + const active = listActiveRecordings(); + const activeList = active.length + ? active.map((r) => `"${r.name}" (${r.projectRoot})`).join(", ") + : "none"; + throw new FailureError( + `No active recording for flow "${name}" in ${projectRoot}. ` + + `Call flow-start-recording first. Active recordings: ${activeList}.`, + { + error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, + failure_stage: "flow_require_recording", + failure_area: "tool_server", + error_kind: "validation", + } + ); } - return activeFlowName; + session.lastTouchedAtMs = Date.now(); + return session; } -export function clearActiveFlow(): void { - activeFlowName = null; - recordingSession = null; +export function clearRecordingSession(projectRoot: string, name: string): void { + recordings.delete(getFlowPath(projectRoot, name)); +} + +/** Drop every recording — test reset. */ +export function clearAllRecordings(): void { + recordings.clear(); } // ── Types ──────────────────────────────────────────────────────────── @@ -2204,28 +2270,50 @@ export function clientFileDirective(filePath: string, content: string): ClientFi export type FlowSavedTo = string | ClientFileDirective; /** - * Append a step to the active recording and persist it. In "host" mode the - * file on disk is re-read first (the original behavior — a manual edit made - * mid-recording is honored); in "client" mode this process never sees the - * client's disk, so the in-memory copy is authoritative and the updated YAML - * travels back in the directive. + * Serialize work against one recording. `appendStep` is read → await → write, + * a lost-update window that only mattered while a single recording could have + * a single caller; now that an agent can legitimately have two `flow-add-step` + * calls in flight (and two agents can share one server), appends on a session + * are chained so the second reads what the first wrote. + * + * Per session, not global: two recordings write two different files and must + * not queue behind each other. + */ +async function withSessionLock(session: RecordingSession, fn: () => Promise): Promise { + // `.then(fn, fn)` — a prior append that rejected must not wedge the chain, + // and `session.tail` swallows the result so an unobserved rejection on the + // tail can never surface as an unhandled rejection. + const run = session.tail.then(fn, fn); + session.tail = run.catch(() => {}); + return run; +} + +/** + * Append a step to a recording and persist it. In "host" mode the file on disk + * is re-read first (the original behavior — a manual edit made mid-recording is + * honored); in "client" mode this process never sees the client's disk, so the + * in-memory copy is authoritative and the updated YAML travels back in the + * directive. */ -export async function appendStepToActiveFlow( +export async function appendStepToFlow( + session: RecordingSession, step: FlowStep -): Promise<{ flowFile: string; savedTo: FlowSavedTo; session: RecordingSession }> { - const session = requireRecordingSession(); - if (session.persist === "host") { - const flowFile = await appendStep(session.filePath, step); - session.flow = parseFlow(flowFile); - return { flowFile, savedTo: session.filePath, session }; - } - session.flow.steps.push(step); - try { - validateFlow(session.flow); - } catch (err) { - session.flow.steps.pop(); // keep the in-memory copy consistent: nothing recorded - throw err; - } - const flowFile = serializeFlow(session.flow); - return { flowFile, savedTo: clientFileDirective(session.filePath, flowFile), session }; +): Promise<{ flowFile: string; savedTo: FlowSavedTo }> { + return withSessionLock(session, async () => { + session.lastTouchedAtMs = Date.now(); + if (session.persist === "host") { + const flowFile = await appendStep(session.filePath, step); + session.flow = parseFlow(flowFile); + return { flowFile, savedTo: session.filePath }; + } + session.flow.steps.push(step); + try { + validateFlow(session.flow); + } catch (err) { + session.flow.steps.pop(); // keep the in-memory copy consistent: nothing recorded + throw err; + } + const flowFile = serializeFlow(session.flow); + return { flowFile, savedTo: clientFileDirective(session.filePath, flowFile) }; + }); } diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index ec3d230d8..4c1b8e2b9 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -1,3 +1,4 @@ +import { z } from "zod"; import { ServiceState, isLiveServiceState } from "@argent/registry"; import type { Registry, ToolDefinition } from "@argent/registry"; import { SIMULATOR_SERVER_NAMESPACE } from "../../blueprints/simulator-server"; @@ -22,25 +23,74 @@ const PREFIXES = [ `${ANDROID_TV_CONTROL_NAMESPACE}:`, ]; +const zodSchema = z.object({ + devices: z + .array(z.string()) + .optional() + .describe( + "Device ids (iOS UDID / Android serial / Chromium id) to scope the teardown to — pass the devices THIS session actually used. Omit only for a deliberate machine-wide cleanup: the tool-server is shared by every agent on the host, so an unscoped stop also kills devices another agent is mid-session on." + ), +}); + +/** + * Does `urn` belong to `deviceId`? Every URN in {@link PREFIXES} is + * `:`, optionally with a trailing transport discriminator + * (`NativeDevtools::tcp`). Device ids can themselves contain a colon (a + * wireless-adb serial is `192.168.1.5:5555`), so the tail is compared whole + * rather than split on ":". + */ +function urnTargetsDevice(urn: string, deviceIds: string[]): boolean { + const prefix = PREFIXES.find((p) => urn.startsWith(p)); + if (!prefix) return false; + const tail = urn.slice(prefix.length); + // Case-insensitive: iOS UDIDs are conventionally upper-case but agents pass + // through whatever they were given, and a case mismatch must not silently + // widen a scoped stop into a no-op. + return deviceIds.some((id) => { + const lower = id.toLowerCase(); + const t = tail.toLowerCase(); + return t === lower || t.startsWith(`${lower}:`); + }); +} + export function createStopAllSimulatorServersTool( registry: Registry -): ToolDefinition { +): ToolDefinition, { stopped: string[] }> { return { id: "stop-all-simulator-servers", interaction: { - startedMsg: () => "Stopping all simulator servers", + // "all" only holds for the unscoped sweep; a scoped call touches just the + // ids it was given, and saying otherwise would misreport a teardown that + // deliberately left another agent's devices running. + startedMsg: ({ params }) => { + const devices = params?.devices; + return devices + ? `Stopping simulator servers for ${devices.length} ${devices.length === 1 ? "device" : "devices"}` + : "Stopping all simulator servers"; + }, completedMsg: ({ result }) => `Stopped ${result.stopped.length} simulator ${result.stopped.length === 1 ? "server" : "servers"}`, failedMsg: ({ failureSignal }) => `Failed to stop simulator servers: ${failureSignal.error_code}`, }, - description: `Stop all running simulator-server processes (iOS + Android), native devtools services, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. Returns { stopped } — an array of URNs that were shut down. Fails silently if no servers are running.`, + description: `Stop running simulator-server processes (iOS + Android), native devtools services, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. +PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps, silently). Omit \`devices\` only when a machine-wide cleanup is what you actually want. +Returns { stopped } — an array of URNs that were shut down. Fails silently if no matching servers are running.`, + zodSchema, services: () => ({}), - async execute() { + async execute(_services, params) { + const devices = params?.devices; + // Present-but-empty scopes to nothing rather than falling back to the + // machine-wide sweep: a caller that computed a device list and got none + // must not accidentally tear down every other agent's services. + const scoped = devices !== undefined; const snapshot = registry.getSnapshot(); const stopped: string[] = []; for (const [urn, entry] of snapshot.services) { - if (PREFIXES.some((p) => urn.startsWith(p)) && entry.state !== ServiceState.IDLE) { + const matches = scoped + ? urnTargetsDevice(urn, devices) + : PREFIXES.some((p) => urn.startsWith(p)); + if (matches && entry.state !== ServiceState.IDLE) { // Dispose any non-IDLE node (this also clears ERROR/TERMINATING // nodes), but only report the ones that were actually live — an // ERROR node (e.g. a tvOS SimulatorServer that refused to start) From eb317b3ef254bf475e18b82847c8a054b5894f46 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 18:21:14 +0200 Subject: [PATCH 02/98] test(flow): cover concurrent recordings and device-scoped teardown Ports the flow suite to the map-keyed sessions, adds flow-concurrent-recording.test.ts for the isolation and append-ordering guarantees, and covers the devices scope on stop-all-simulator-servers. Docs and MCP instructions updated to match. --- packages/argent-mcp/src/mcp-server.ts | 3 +- packages/skills/rules/argent.md | 5 +- .../skills/skills/argent-create-flow/SKILL.md | 47 +- .../argent-react-native-app-workflow/SKILL.md | 22 +- .../test/failure-classification.test.ts | 21 +- .../flows/flow-concurrent-recording.test.ts | 416 ++++++++++++++++++ .../test/flows/flow-feature-flag-gate.test.ts | 12 +- .../test/flows/flow-record-tap.test.ts | 27 +- .../test/flows/flow-remote-recording.test.ts | 106 ++++- .../tool-server/test/flows/flow-tools.test.ts | 407 +++++++++++++---- .../tool-server/test/flows/flow-utils.test.ts | 208 ++++++--- packages/tool-server/test/stop-tools.test.ts | 167 ++++++- 12 files changed, 1210 insertions(+), 231 deletions(-) create mode 100644 packages/tool-server/test/flows/flow-concurrent-recording.test.ts diff --git a/packages/argent-mcp/src/mcp-server.ts b/packages/argent-mcp/src/mcp-server.ts index 5bb4646fc..dace59f45 100644 --- a/packages/argent-mcp/src/mcp-server.ts +++ b/packages/argent-mcp/src/mcp-server.ts @@ -240,7 +240,8 @@ export async function startMcpServer(options: StartMcpServerOptions): Promise/.argent/flows/.yaml`, so several can be open at once — different names, different projects, on one device or several — and never cross-talk. Re-calling `flow-start-recording` for the **same** name + project restarts that one and only that one: the response carries `restarted: true` and `discardedSteps`, and the `.yaml` is reset to an empty flow. Starting a _different_ flow abandons nothing. +- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...` — the tail lists every live recording as `"name" (project_root)`, or `none`, so a typo or a wrong `project_root` is visible in the error itself. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. ### flow-add-step arguments @@ -176,22 +176,22 @@ Every other recorded tool (a velocity-dependent `gesture-swipe`, a fixed-distanc ``` flow-start-recording { name: "open-about", project_root: "/Users/dev/MyApp" } -flow-add-echo { message: "Start Settings from scratch" } -flow-add-step { command: "restart-app", args: "{\"udid\": \"ABC\", \"bundleId\": \"com.apple.Preferences\"}" } # ⇒ captured as `- launch: com.apple.Preferences` — this is now an e2e flow -flow-add-echo { message: "On the Settings root list, tapping the 'General' row" } -flow-add-step { command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.35}" } # ⇒ captured as `- tap: { text: General }` (portable selector, no udid) -flow-add-step { command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"About\"}}" } # gate the transition -flow-add-echo { message: "On Settings > General, tapping 'About'" } -flow-add-step { command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.17}" } -flow-add-step { command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"Model Name\"}}" } -flow-finish-recording {} +flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "Start Settings from scratch" } +flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "restart-app", args: "{\"udid\": \"ABC\", \"bundleId\": \"com.apple.Preferences\"}" } # ⇒ captured as `- launch: com.apple.Preferences` — this is now an e2e flow +flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "On the Settings root list, tapping the 'General' row" } +flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.35}" } # ⇒ captured as `- tap: { text: General }` (portable selector, no udid) +flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"About\"}}" } # gate the transition +flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "On Settings > General, tapping 'About'" } +flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.17}" } +flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"Model Name\"}}" } +flow-finish-recording { name: "open-about", project_root: "/Users/dev/MyApp" } ``` Then polish the saved file: the two `await-ui-element` steps become `await:` directives (see the file below). ## Replaying -Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here; the stored-for-the-session shortcut applies only to the recording tools. If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. +Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow is the exception: with no `device` it boots its own instance from the `launch` path and tears it down after, so leave `device` unset there unless you mean to attach to a running one.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. **What each step reports.** Raw `tool:` steps include the underlying tool's full `result` (screenshots and other outputs render as usual). The directive steps are summarized: `tap`/`type`/`await`/`assert` report only `status` + `reason`, and `snapshot` adds `artifacts` only when there is something to look at — a failed comparison (baseline/current/diff paths), a missing-baseline failure (`current` only), or a baseline write; a clean pass reports just `status` + `reason`. So converting a `tool: gesture-tap` into a `tap:` directive during cleanup drops only that tap's (uninteresting) raw result — output-bearing tools like `screenshot` have no directive form and stay `tool:` steps, so their results keep flowing through. @@ -250,6 +250,11 @@ For silent misfires and partial divergence, echo annotations (see _Making flows 1. Note the failure step index and error message (if hard error). 2. Call `screenshot` to see where the app actually is now. 3. Call `describe` or `debugger-component-tree` to get the current element tree. Remember `describe` shows less than the flow tree — a testID missing from its output can still resolve as a selector (see Selectors). + + `debugger-component-tree` is an **authoring aid only — never record a `debugger-*` step into a flow.** `device_id` is stripped at record time and re-injected at replay, but `port` is not a device-bind key, so a recorded debugger step carries whatever `port` it was given (or falls through to the 8081 default at replay) and runs against whatever Metro happens to be on that port. + + When calling it directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. + 4. Compare current state to what the failed step expected. Classify the root cause: | Root cause | Symptoms | diff --git a/packages/skills/skills/argent-react-native-app-workflow/SKILL.md b/packages/skills/skills/argent-react-native-app-workflow/SKILL.md index eba0c7975..2cc4797f5 100644 --- a/packages/skills/skills/argent-react-native-app-workflow/SKILL.md +++ b/packages/skills/skills/argent-react-native-app-workflow/SKILL.md @@ -137,17 +137,17 @@ Once you discover the correct build/run workflow for a project, **save it to pro ### 3.5 Device Control -| Action | Tool / Command | -| -------------------------- | ---------------------------------------------------------------------- | -| List devices | `list-devices` tool (iOS + Android) | -| Boot an iOS simulator | `boot-device` tool with `udid` | -| Boot an Android emulator | `boot-device` tool with `avdName` | -| Launch an app | `launch-app` tool (pass device id + bundle id / package name) | -| Restart an app | `restart-app` tool (pass device id + bundle id / package name) | -| Open a URL / deep link | `open-url` tool (pass device id + URL) | -| Rotate device | `rotate` tool | -| Stop simulator server | `stop-simulator-server` tool (iOS UDID or Android serial — one device) | -| Stop all simulator servers | `stop-all-simulator-servers` tool (iOS + Android) | +| Action | Tool / Command | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| List devices | `list-devices` tool (iOS + Android) | +| Boot an iOS simulator | `boot-device` tool with `udid` | +| Boot an Android emulator | `boot-device` tool with `avdName` | +| Launch an app | `launch-app` tool (pass device id + bundle id / package name) | +| Restart an app | `restart-app` tool (pass device id + bundle id / package name) | +| Open a URL / deep link | `open-url` tool (pass device id + URL) | +| Rotate device | `rotate` tool | +| Stop simulator server | `stop-simulator-server` tool (iOS UDID or Android serial — one device) | +| Stop all simulator servers | `stop-all-simulator-servers` tool — pass `devices: [...]` to scope the teardown to this session's devices (an unscoped call also tears down other agents' devices; use it only for a machine-wide cleanup) | For full simulator setup workflow, refer to the `argent-ios-simulator-setup` skill. diff --git a/packages/tool-server/test/failure-classification.test.ts b/packages/tool-server/test/failure-classification.test.ts index e9f1994ce..bb6a4c234 100644 --- a/packages/tool-server/test/failure-classification.test.ts +++ b/packages/tool-server/test/failure-classification.test.ts @@ -2,12 +2,7 @@ import { createServer, type Server } from "node:http"; import { describe, it, expect, afterEach } from "vitest"; import { FAILURE_CODES, getFailureSignal, type FailureCode } from "@argent/registry"; -import { - setActiveProjectRoot, - clearActiveProjectRoot, - assertSafeFlowName, - getFlowPath, -} from "../src/tools/flows/flow-utils"; +import { assertValidProjectRoot, assertSafeFlowName } from "../src/tools/flows/flow-utils"; import type { DeviceInfo, Registry } from "@argent/registry"; import { makeChromiumImpl } from "../src/tools/keyboard/platforms/chromium"; import { chromiumCdpBlueprint } from "../src/blueprints/chromium-cdp"; @@ -77,8 +72,6 @@ function startServer(handler: (path: string, res: import("node:http").ServerResp } afterEach(async () => { - // setActiveProjectRoot mutates module state; reset so cases don't leak. - clearActiveProjectRoot(); // Tear down any local servers a case spun up. await Promise.all(openServers.splice(0).map((s) => new Promise((r) => s.close(() => r())))); }); @@ -86,26 +79,18 @@ afterEach(async () => { describe("flow-utils classifications", () => { it("classifies a relative project_root as FLOW_PROJECT_ROOT_INVALID", () => { expectCode( - captureSync(() => setActiveProjectRoot("relative/path")), + captureSync(() => assertValidProjectRoot("relative/path")), FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID ); }); it("classifies a project_root containing '..' as FLOW_PROJECT_ROOT_INVALID", () => { expectCode( - captureSync(() => setActiveProjectRoot("/a/../b")), + captureSync(() => assertValidProjectRoot("/a/../b")), FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID ); }); - it("classifies path resolution with no active project_root as FLOW_PROJECT_ROOT_REQUIRED", () => { - // No active root → getFlowPath → getFlowsDir → requireActiveProjectRoot throws. - expectCode( - captureSync(() => getFlowPath("valid-name")), - FAILURE_CODES.FLOW_PROJECT_ROOT_REQUIRED - ); - }); - it("classifies an unsafe flow name as FLOW_NAME_INVALID", () => { expectCode( captureSync(() => assertSafeFlowName("bad name!")), diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts new file mode 100644 index 000000000..80c288eb2 --- /dev/null +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -0,0 +1,416 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; +import type { Registry } from "@argent/registry"; + +import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording"; +import { flowInsertEchoTool } from "../../src/tools/flows/flow-insert-echo"; +import { flowFinishRecordingTool } from "../../src/tools/flows/flow-finish-recording"; +import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; +import { createRunFlowTool } from "../../src/tools/flows/flow-run"; +import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; +import { + clearAllRecordings, + getRecordingSession, + listActiveRecordings, + parseFlow, + serializeFlow, + type FlowFile, + type FlowStep, +} from "../../src/tools/flows/flow-utils"; + +/** + * Concurrency contract of the recording tools. The tool-server is a host-wide + * singleton shared by every MCP client, subagent and CLI call on the machine, + * so several agents can legitimately be recording at the same moment — in one + * project or across projects. A recording is identified by its + * (project_root, name) key, and these tests assert the ISOLATION that follows: + * one recording's steps never land in another's file, addressing a key that + * isn't live fails loudly (naming the ones that are), replaying a flow + * elsewhere rebinds nothing, and appends to one session can't lose each other. + */ + +const IOS_DEVICE = "00000000-0000-0000-0000-0000000000ab"; + +// ── Harness ────────────────────────────────────────────────────────── + +let roots: string[] = []; + +/** A real temp dir standing in for one agent's project root. */ +async function makeRoot(label: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), `flow-concurrent-${label}-`)); + roots.push(dir); + return dir; +} + +function createMockRegistry(): Registry { + return { + invokeTool: vi.fn(async (id: string) => { + if (id === "list-devices") return { devices: [] }; + // Yield a macrotask: flow-add-step runs the step LIVE before it appends, + // so this is what lets several calls issued without an await in between + // reach the append phase concurrently (see the lost-update test). + await new Promise((resolve) => setTimeout(resolve, 0)); + return { ok: true }; + }), + getTool: vi.fn(() => ({ inputSchema: { properties: { udid: {} } } })), + } as unknown as Registry; +} + +const registry = createMockRegistry(); +const addStepTool = createFlowAddStepTool(registry); + +const flowPath = (root: string, name: string): string => + path.join(root, ".argent", "flows", `${name}.yaml`); + +function start(root: string, name: string, executionPrerequisite?: string) { + return flowStartRecordingTool.execute({}, { name, project_root: root, executionPrerequisite }); +} + +/** Record a `tool` step tagged with `marker`, so its file of origin is provable. */ +function addStep(root: string, name: string, marker: string) { + return addStepTool.execute( + {}, + { + name, + project_root: root, + command: "keyboard", + args: JSON.stringify({ text: marker }), + } + ); +} + +function addEcho(root: string, name: string, message: string) { + return flowInsertEchoTool.execute({}, { name, project_root: root, message }); +} + +function finish(root: string, name: string) { + return flowFinishRecordingTool.execute({}, { name, project_root: root }); +} + +async function writeSavedFlow(root: string, name: string, flow: FlowFile): Promise { + await fs.mkdir(path.dirname(flowPath(root, name)), { recursive: true }); + await fs.writeFile(flowPath(root, name), serializeFlow(flow), "utf8"); +} + +/** Collapse steps to their markers so a file's contents read at a glance. */ +function markers(steps: FlowStep[]): string[] { + return steps.map((step) => { + if (step.kind === "echo") return `echo:${step.message}`; + if (step.kind === "tool") return `tool:${String(step.args.text)}`; + return step.kind; + }); +} + +async function readMarkers(root: string, name: string): Promise { + return markers(parseFlow(await fs.readFile(flowPath(root, name), "utf8")).steps); +} + +beforeEach(() => { + clearAllRecordings(); + roots = []; +}); + +afterEach(async () => { + clearAllRecordings(); + await Promise.all(roots.map((dir) => fs.rm(dir, { recursive: true, force: true }))); + roots = []; +}); + +// ── Two recordings, one project ────────────────────────────────────── + +describe("two recordings in one project", () => { + it("keeps interleaved steps on their own files, in order", async () => { + const root = await makeRoot("one-project"); + await start(root, "alpha"); + await start(root, "beta"); + + expect( + listActiveRecordings() + .map((r) => r.name) + .sort() + ).toEqual(["alpha", "beta"]); + + // Interleave the two recordings the way two agents sharing the server would. + await addStep(root, "alpha", "a1"); + await addEcho(root, "beta", "b1"); + await addStep(root, "alpha", "a2"); + await addEcho(root, "beta", "b2"); + + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1", "tool:a2"]); + expect(await readMarkers(root, "beta")).toEqual(["echo:b1", "echo:b2"]); + }); + + it("finishing one leaves the other live and still appendable", async () => { + const root = await makeRoot("one-project-finish"); + await start(root, "alpha"); + await start(root, "beta"); + await addStep(root, "alpha", "a1"); + await addEcho(root, "beta", "b1"); + + const finished = await finish(root, "alpha"); + expect(finished.path).toBe(flowPath(root, "alpha")); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1"]); + expect(finished.steps).toBe(1); + + // Only alpha's key was cleared. + expect(getRecordingSession(root, "alpha")).toBeUndefined(); + expect(getRecordingSession(root, "beta")?.filePath).toBe(flowPath(root, "beta")); + + // beta keeps recording into its own file. + await addEcho(root, "beta", "b2"); + await addStep(root, "beta", "b3"); + const finishedB = await finish(root, "beta"); + expect(markers(parseFlow(finishedB.flowFile).steps)).toEqual(["echo:b1", "echo:b2", "tool:b3"]); + expect(await readMarkers(root, "beta")).toEqual(["echo:b1", "echo:b2", "tool:b3"]); + // alpha was never reopened by beta's appends. + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); + }); +}); + +// ── One name, two project roots ────────────────────────────────────── + +describe("the same flow name under two project roots", () => { + it("records each project's steps into that project's file only", async () => { + const rootA = await makeRoot("root-a"); + const rootB = await makeRoot("root-b"); + + await start(rootA, "checkout", "Cart has one item"); + await start(rootB, "checkout", "Cart is empty"); + + await addStep(rootA, "checkout", "a1"); + await addStep(rootB, "checkout", "b1"); + await addEcho(rootA, "checkout", "a2"); + await addStep(rootB, "checkout", "b2"); + + expect(await readMarkers(rootA, "checkout")).toEqual(["tool:a1", "echo:a2"]); + expect(await readMarkers(rootB, "checkout")).toEqual(["tool:b1", "tool:b2"]); + + // Sessions carry their own project root and prerequisite, not the other's. + expect(getRecordingSession(rootA, "checkout")?.projectRoot).toBe(rootA); + expect(getRecordingSession(rootB, "checkout")?.projectRoot).toBe(rootB); + + const finishedA = await finish(rootA, "checkout"); + expect(finishedA.path).toBe(flowPath(rootA, "checkout")); + expect(finishedA.executionPrerequisite).toBe("Cart has one item"); + + // B is untouched by A finishing, and still resolves to B's file. + const finishedB = await finish(rootB, "checkout"); + expect(finishedB.path).toBe(flowPath(rootB, "checkout")); + expect(finishedB.executionPrerequisite).toBe("Cart is empty"); + expect(markers(parseFlow(finishedB.flowFile).steps)).toEqual(["tool:b1", "tool:b2"]); + }); +}); + +// ── Addressing a key that isn't live ───────────────────────────────── + +describe("addressing an unknown recording key", () => { + async function captureFailure(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err; + } + throw new Error("expected the call to fail"); + } + + it("fails with FLOW_NO_ACTIVE_RECORDING and lists the live recordings", async () => { + const rootA = await makeRoot("unknown-a"); + const rootB = await makeRoot("unknown-b"); + await start(rootA, "alpha"); + await start(rootB, "beta"); + + const err = await captureFailure(addEcho(rootA, "never-started", "x")); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + const message = (err as Error).message; + expect(message).toContain('No active recording for flow "never-started"'); + expect(message).toContain(rootA); + // The live keys are named so the agent can self-correct. + expect(message).toContain(`"alpha" (${rootA})`); + expect(message).toContain(`"beta" (${rootB})`); + }); + + it("fails the same way for the right name under the wrong project_root", async () => { + const rootA = await makeRoot("wrong-root-a"); + const rootB = await makeRoot("wrong-root-b"); + await start(rootA, "alpha"); + + const err = await captureFailure(addStep(rootB, "alpha", "stray")); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect((err as Error).message).toContain(`"alpha" (${rootA})`); + + // The misdirected step was not recorded anywhere. + expect(await readMarkers(rootA, "alpha")).toEqual([]); + await expect(fs.stat(flowPath(rootB, "alpha"))).rejects.toThrow(); + }); + + it("reports the live recordings as none when nothing is being recorded", async () => { + const root = await makeRoot("nothing-live"); + const err = await captureFailure(finish(root, "alpha")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect((err as Error).message).toContain("Active recordings: none."); + }); +}); + +// ── Concurrent appends to one session ──────────────────────────────── + +describe("concurrent flow-add-step calls on one recording", () => { + it("loses no step when several appends are in flight at once", async () => { + const root = await makeRoot("append-race"); + await start(root, "burst"); + + const tags = ["s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7"]; + // Fire without awaiting in between: every call is past its live execution + // and inside the append phase before the first one writes. appendStep is + // read → await → write, so without the per-session mutex these would all + // read the same file and the last write would drop the others. + const inflight = tags.map((tag) => addStep(root, "burst", tag)); + await Promise.all(inflight); + + const recorded = await readMarkers(root, "burst"); + expect(recorded).toHaveLength(tags.length); + expect([...recorded].sort()).toEqual(tags.map((t) => `tool:${t}`).sort()); + + // The in-memory copy the session serves to flow-finish-recording agrees. + expect(getRecordingSession(root, "burst")?.flow.steps).toHaveLength(tags.length); + const finished = await finish(root, "burst"); + expect(finished.steps).toBe(tags.length); + }); + + it("does not serialize a second recording behind the first one's appends", async () => { + const root = await makeRoot("append-race-two"); + await start(root, "alpha"); + await start(root, "beta"); + + // Both bursts in flight together — the lock is per session, so neither + // file may pick up the other's steps. + await Promise.all([ + ...["a0", "a1", "a2", "a3"].map((tag) => addStep(root, "alpha", tag)), + ...["b0", "b1", "b2", "b3"].map((tag) => addStep(root, "beta", tag)), + ]); + + expect([...(await readMarkers(root, "alpha"))].sort()).toEqual([ + "tool:a0", + "tool:a1", + "tool:a2", + "tool:a3", + ]); + expect([...(await readMarkers(root, "beta"))].sort()).toEqual([ + "tool:b0", + "tool:b1", + "tool:b2", + "tool:b3", + ]); + }); +}); + +// ── Replaying a flow while recordings are live ─────────────────────── + +describe("running a flow in a third project while two recordings are live", () => { + it("rebinds neither recording's file path", async () => { + const rootA = await makeRoot("exec-a"); + const rootB = await makeRoot("exec-b"); + const rootC = await makeRoot("exec-c"); + + await start(rootA, "alpha"); + await start(rootB, "beta"); + await addStep(rootA, "alpha", "a1"); + await addEcho(rootB, "beta", "b1"); + + // A saved flow belonging to a third project, replayed mid-recording. + await writeSavedFlow(rootC, "standalone", { + executionPrerequisite: "App on the home screen", + steps: [{ kind: "echo", message: "replayed" }], + }); + + const prereq = await flowReadPrerequisiteTool.execute( + {}, + { name: "standalone", project_root: rootC } + ); + expect(prereq.executionPrerequisite).toBe("App on the home screen"); + + const runResult = await createRunFlowTool(registry).execute( + {}, + { + name: "standalone", + project_root: rootC, + device: IOS_DEVICE, + prerequisiteAcknowledged: true, + } + ); + expect(runResult).toHaveProperty("ok", true); + + // Both sessions still point at their own files… + expect(getRecordingSession(rootA, "alpha")?.filePath).toBe(flowPath(rootA, "alpha")); + expect(getRecordingSession(rootB, "beta")?.filePath).toBe(flowPath(rootB, "beta")); + + // …and subsequent steps still land there. + await addStep(rootA, "alpha", "a2"); + await addEcho(rootB, "beta", "b2"); + expect(await readMarkers(rootA, "alpha")).toEqual(["tool:a1", "tool:a2"]); + expect(await readMarkers(rootB, "beta")).toEqual(["echo:b1", "echo:b2"]); + + // Nothing was written into the replayed project, and the replayed flow + // did not pick up either recording's steps. + await expect(fs.stat(flowPath(rootC, "alpha"))).rejects.toThrow(); + expect(await readMarkers(rootC, "standalone")).toEqual(["echo:replayed"]); + }); +}); + +// ── Restarting one recording ───────────────────────────────────────── + +describe("restarting a recording on one key", () => { + it("resets only that flow and leaves a concurrent recording untouched", async () => { + const root = await makeRoot("restart"); + + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + await addStep(root, "alpha", "a2"); + + const startedBeta = await start(root, "beta"); + // Starting a DIFFERENT key abandons nothing — nothing to report. + expect(startedBeta.restarted).toBeUndefined(); + expect(startedBeta.discardedSteps).toBeUndefined(); + await addEcho(root, "beta", "b1"); + + const restarted = await start(root, "alpha"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(2); + expect(restarted.message).toContain("alpha"); + expect(await readMarkers(root, "alpha")).toEqual([]); + + // beta neither lost its steps nor its session. + expect(await readMarkers(root, "beta")).toEqual(["echo:b1"]); + expect(getRecordingSession(root, "beta")?.flow.steps).toHaveLength(1); + await addEcho(root, "beta", "b2"); + expect(await readMarkers(root, "beta")).toEqual(["echo:b1", "echo:b2"]); + + // The restarted take records into the reset file. + await addStep(root, "alpha", "a3"); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a3"]); + }); + + it("does not restart a same-named recording in another project", async () => { + const rootA = await makeRoot("restart-a"); + const rootB = await makeRoot("restart-b"); + + await start(rootA, "alpha"); + await addStep(rootA, "alpha", "a1"); + await start(rootB, "alpha"); + await addStep(rootB, "alpha", "b1"); + + // Same name, different root ⇒ a different key ⇒ not a restart. + const restarted = await start(rootB, "alpha"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(1); + expect(await readMarkers(rootB, "alpha")).toEqual([]); + + // The other project's recording kept its step and its session. + expect(await readMarkers(rootA, "alpha")).toEqual(["tool:a1"]); + expect(getRecordingSession(rootA, "alpha")?.flow.steps).toHaveLength(1); + }); +}); diff --git a/packages/tool-server/test/flows/flow-feature-flag-gate.test.ts b/packages/tool-server/test/flows/flow-feature-flag-gate.test.ts index 1b6a813ae..1660f8500 100644 --- a/packages/tool-server/test/flows/flow-feature-flag-gate.test.ts +++ b/packages/tool-server/test/flows/flow-feature-flag-gate.test.ts @@ -12,8 +12,8 @@ * effect never happens (store not mutated); * - flag ON → the same step runs and the side effect lands. * - * Run `run_in_band`-style serially because it relies on a shared active project - * root (the flow harness's module state), like the sibling flow tests. + * Each case gets its own temp project root, passed explicitly to `flow-execute`, + * so nothing is shared between them. */ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import * as fs from "node:fs/promises"; @@ -23,11 +23,7 @@ import { z } from "zod"; import { Registry } from "@argent/registry"; import { createRunFlowTool } from "../../src/tools/flows/flow-run"; -import { - clearActiveProjectRoot, - setActiveProjectRoot, - serializeFlow, -} from "../../src/tools/flows/flow-utils"; +import { serializeFlow } from "../../src/tools/flows/flow-utils"; let tmpDir: string; @@ -65,11 +61,9 @@ async function writeFlow(name: string): Promise { beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-flag-gate-")); - setActiveProjectRoot(tmpDir); }); afterEach(async () => { - clearActiveProjectRoot(); await fs.rm(tmpDir, { recursive: true, force: true }); }); diff --git a/packages/tool-server/test/flows/flow-record-tap.test.ts b/packages/tool-server/test/flows/flow-record-tap.test.ts index b470072b6..93b085211 100644 --- a/packages/tool-server/test/flows/flow-record-tap.test.ts +++ b/packages/tool-server/test/flows/flow-record-tap.test.ts @@ -15,14 +15,10 @@ vi.mock("../../src/tools/flows/flow-tree", () => ({ import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording"; -import { - clearActiveFlow, - clearActiveProjectRoot, - parseFlow, - setActiveProjectRoot, -} from "../../src/tools/flows/flow-utils"; +import { clearAllRecordings, parseFlow } from "../../src/tools/flows/flow-utils"; const DEVICE = "00000000-0000-0000-0000-0000000000AB"; // iOS UDID shape +const FLOW = "rec"; const PREREQ = "App on home screen"; let tmpDir: string; @@ -53,28 +49,31 @@ async function recordTap(point: { x: number; y: number }) { const tool = createFlowAddStepTool(mockRegistry()); return tool.execute( {}, - { command: "gesture-tap", args: JSON.stringify({ udid: DEVICE, ...point }) } + { + name: FLOW, + project_root: tmpDir, + command: "gesture-tap", + args: JSON.stringify({ udid: DEVICE, ...point }), + } ); } async function recordedSteps() { - const content = await fs.readFile(path.join(tmpDir, ".argent", "flows", "rec.yaml"), "utf8"); + const content = await fs.readFile(path.join(tmpDir, ".argent", "flows", `${FLOW}.yaml`), "utf8"); return parseFlow(content).steps; } beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-record-tap-")); - setActiveProjectRoot(tmpDir); - clearActiveFlow(); + clearAllRecordings(); await flowStartRecordingTool.execute( {}, - { name: "rec", project_root: tmpDir, executionPrerequisite: PREREQ } + { name: FLOW, project_root: tmpDir, executionPrerequisite: PREREQ } ); }); afterEach(async () => { - clearActiveFlow(); - clearActiveProjectRoot(); + clearAllRecordings(); await fs.rm(tmpDir, { recursive: true, force: true }); }); @@ -128,6 +127,8 @@ describe("flow-add-step tap selector capture", () => { await tool.execute( {}, { + name: FLOW, + project_root: tmpDir, command: "gesture-tap", args: JSON.stringify({ udid: DEVICE, x: 0.5, y: 0.52, clickCount: 2 }), } diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index a4befac80..8c8c71f4d 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -11,11 +11,7 @@ import { flowFinishRecordingTool } from "../../src/tools/flows/flow-finish-recor import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { createRunFlowTool, resolveFlowSource } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; -import { - clearActiveFlow, - clearActiveProjectRoot, - parseFlow, -} from "../../src/tools/flows/flow-utils"; +import { clearAllRecordings, parseFlow } from "../../src/tools/flows/flow-utils"; /** * Remote-mode flow behavior: the agent's project_root does NOT exist on this @@ -28,11 +24,16 @@ import { const CLIENT_ROOT = path.join(os.tmpdir(), "definitely-not-on-this-host", "agent-project"); const CLIENT_FLOW_PATH = path.join(CLIENT_ROOT, ".argent", "flows", "remote-flow.yaml"); -function remoteCtx(): ToolContext { +// A SECOND client project — a different agent recording a flow of the same +// name. Same host, same flow name, different project root. +const OTHER_CLIENT_ROOT = path.join(os.tmpdir(), "definitely-not-on-this-host", "other-project"); +const OTHER_CLIENT_FLOW_PATH = path.join(OTHER_CLIENT_ROOT, ".argent", "flows", "remote-flow.yaml"); + +function remoteCtx(root: string = CLIENT_ROOT): ToolContext { return { artifacts: new ArtifactStore(), fileInputs: { - project_root: { clientPath: CLIENT_ROOT, presentOnHost: false, viaUpload: false }, + project_root: { clientPath: root, presentOnHost: false, viaUpload: false }, }, }; } @@ -59,13 +60,13 @@ function createMockRegistry(tools: Record = {}) { } beforeEach(() => { - clearActiveFlow(); + clearAllRecordings(); }); afterEach(async () => { - clearActiveFlow(); - clearActiveProjectRoot(); + clearAllRecordings(); await fs.rm(CLIENT_ROOT, { recursive: true, force: true }); + await fs.rm(OTHER_CLIENT_ROOT, { recursive: true, force: true }); }); describe("flow recording with a remote client (probe miss)", () => { @@ -96,8 +97,14 @@ describe("flow recording with a remote client (probe miss)", () => { remoteCtx() ); - await flowInsertEchoTool.execute({}, { message: "label" }); - const stepResult = await addStep.execute({}, { command: "tap", args: '{"x":0.5}' }); + await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "label" } + ); + const stepResult = await addStep.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, command: "tap", args: '{"x":0.5}' } + ); const directive = stepResult.savedTo as { path: string; content: string }; expect(directive.path).toBe(CLIENT_FLOW_PATH); @@ -170,16 +177,85 @@ describe("flow recording with a remote client (probe miss)", () => { { name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" }, remoteCtx() ); - await flowInsertEchoTool.execute({}, { message: "only step" }); + await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "only step" } + ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT } + ); expect(result.steps).toBe(1); expect(result.summary).toEqual(["1. echo: only step"]); expect(result.path).toBe(CLIENT_FLOW_PATH); expect(result.savedTo).toMatchObject({ [CLIENT_FILE_MARKER]: true }); - await expect(flowFinishRecordingTool.execute({}, {})).rejects.toThrow("No active flow"); + await expect( + flowFinishRecordingTool.execute({}, { name: "remote-flow", project_root: CLIENT_ROOT }) + ).rejects.toThrow("No active recording"); + }); + + it("keeps same-named recordings under different client roots isolated", async () => { + const registry = createMockRegistry({ tap: { result: { tapped: true } } }); + const addStep = createFlowAddStepTool(registry); + + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" }, + remoteCtx() + ); + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: OTHER_CLIENT_ROOT, executionPrerequisite: "Settings" }, + remoteCtx(OTHER_CLIENT_ROOT) + ); + + await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "first client" } + ); + const otherStep = await addStep.execute( + {}, + { name: "remote-flow", project_root: OTHER_CLIENT_ROOT, command: "tap", args: '{"x":0.5}' } + ); + + // Each directive names its OWN client's file and carries only that + // recording's steps — the second agent's tap never joins the first's flow. + const otherDirective = otherStep.savedTo as { path: string; content: string }; + expect(otherDirective.path).toBe(OTHER_CLIENT_FLOW_PATH); + expect(parseFlow(otherDirective.content).steps).toEqual([ + { kind: "tool", name: "tap", args: { x: 0.5 } }, + ]); + expect(parseFlow(otherDirective.content).executionPrerequisite).toBe("Settings"); + + // Finishing one leaves the other live, with its own path and steps. + const first = await flowFinishRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT } + ); + expect(first.path).toBe(CLIENT_FLOW_PATH); + expect(first.summary).toEqual(["1. echo: first client"]); + expect(first.savedTo).toMatchObject({ + [CLIENT_FILE_MARKER]: true, + path: CLIENT_FLOW_PATH, + }); + + const other = await flowFinishRecordingTool.execute( + {}, + { name: "remote-flow", project_root: OTHER_CLIENT_ROOT } + ); + expect(other.path).toBe(OTHER_CLIENT_FLOW_PATH); + expect(other.summary).toEqual(['1. tool: tap {"x":0.5}']); + expect(other.savedTo).toMatchObject({ + [CLIENT_FILE_MARKER]: true, + path: OTHER_CLIENT_FLOW_PATH, + }); + + // Neither client's directory layout was recreated on this host. + await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow(); + await expect(fs.stat(OTHER_CLIENT_ROOT)).rejects.toThrow(); }); }); diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 05fe706de..56a3d0b0a 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -17,10 +17,9 @@ import { } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; import { - clearActiveFlow, - setActiveProjectRoot, - clearActiveProjectRoot, + clearAllRecordings, flowsDirFor, + getRecordingSession, parseFlow, serializeFlow, type FlowStep, @@ -37,6 +36,9 @@ function assertFlowRunResult( } let tmpDir: string; +// A second project root. Recordings are keyed by /, so it +// is what the cross-project cases address: same flow name, different project. +let otherDir: string; function createMockRegistry( tools: Record = {} @@ -56,8 +58,8 @@ function createMockRegistry( } as unknown as Registry; } -async function readFlowFile(name: string): Promise { - return fs.readFile(path.join(tmpDir, ".argent", "flows", `${name}.yaml`), "utf8"); +async function readFlowFile(name: string, projectRoot: string = tmpDir): Promise { + return fs.readFile(path.join(projectRoot, ".argent", "flows", `${name}.yaml`), "utf8"); } const PREREQ = "App on home screen"; @@ -66,14 +68,14 @@ const PREREQ = "App on home screen"; beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-test-")); - setActiveProjectRoot(tmpDir); - clearActiveFlow(); + otherDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-test-other-")); + clearAllRecordings(); }); afterEach(async () => { - clearActiveFlow(); - clearActiveProjectRoot(); + clearAllRecordings(); await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.rm(otherDir, { recursive: true, force: true }); }); // ── flow-start-recording ───────────────────────────────────────────── @@ -92,12 +94,15 @@ describe("flow-start-recording", () => { expect(flow.steps).toEqual([]); }); - it("sets the active flow", async () => { + it("opens a recording addressable by name + project_root", async () => { await flowStartRecordingTool.execute( {}, { name: "my-flow", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const result = await flowInsertEchoTool.execute({}, { message: "test" }); + const result = await flowInsertEchoTool.execute( + {}, + { name: "my-flow", project_root: tmpDir, message: "test" } + ); expect(result.message).toContain("my-flow"); }); @@ -106,7 +111,10 @@ describe("flow-start-recording", () => { {}, { name: "overwrite", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "line1" }); + await flowInsertEchoTool.execute( + {}, + { name: "overwrite", project_root: tmpDir, message: "line1" } + ); // Start again with same name — should reset await flowStartRecordingTool.execute( @@ -132,7 +140,7 @@ describe("flow-start-recording", () => { // ── flow-start-recording edge cases ────────────────────────────────── describe("flow-start-recording edge cases", () => { - it("starting a new flow while another is recording notifies about the switch", async () => { + it("starting a differently-named flow leaves the earlier recording live", async () => { await flowStartRecordingTool.execute( {}, { name: "first-flow", project_root: tmpDir, executionPrerequisite: PREREQ } @@ -142,52 +150,108 @@ describe("flow-start-recording edge cases", () => { { name: "second-flow", project_root: tmpDir, executionPrerequisite: "Different" } ); - // Should mention both the old and new flow - expect(result.message).toContain("first-flow"); + // A second recording abandons nothing, so there is no switch to report. expect(result.message).toContain("second-flow"); - expect(result.previousFlow).toBe("first-flow"); + expect(result.message).not.toContain("first-flow"); + expect(result.restarted).toBeUndefined(); + expect(result.discardedSteps).toBeUndefined(); + + // Both recordings still take steps, each addressed by its own name. + const secondEcho = await flowInsertEchoTool.execute( + {}, + { name: "second-flow", project_root: tmpDir, message: "goes to second" } + ); + expect(secondEcho.message).toContain("second-flow"); + const firstEcho = await flowInsertEchoTool.execute( + {}, + { name: "first-flow", project_root: tmpDir, message: "goes to first" } + ); + expect(firstEcho.message).toContain("first-flow"); + + // …and each file ends up holding only its own steps. + expect(parseFlow(await readFlowFile("first-flow")).steps).toEqual([ + { kind: "echo", message: "goes to first" }, + ]); + expect(parseFlow(await readFlowFile("second-flow")).steps).toEqual([ + { kind: "echo", message: "goes to second" }, + ]); + }); + + it("keeps same-named recordings in different projects independent", async () => { + await flowStartRecordingTool.execute( + {}, + { name: "shared-name", project_root: tmpDir, executionPrerequisite: PREREQ } + ); + const result = await flowStartRecordingTool.execute( + {}, + { name: "shared-name", project_root: otherDir, executionPrerequisite: PREREQ } + ); - // Adding a step should target second-flow, not first-flow - const echoResult = await flowInsertEchoTool.execute({}, { message: "goes to second" }); - expect(echoResult.message).toContain("second-flow"); + // Same name, other project — a different key, so nothing was restarted. + expect(result.restarted).toBeUndefined(); + expect(result.discardedSteps).toBeUndefined(); - // first-flow should still exist on disk but be empty - const firstContent = await readFlowFile("first-flow"); - const firstFlow = parseFlow(firstContent); - expect(firstFlow.steps).toEqual([]); + await flowInsertEchoTool.execute( + {}, + { name: "shared-name", project_root: tmpDir, message: "in first project" } + ); + await flowInsertEchoTool.execute( + {}, + { name: "shared-name", project_root: otherDir, message: "in second project" } + ); - // second-flow should have the echo - const secondContent = await readFlowFile("second-flow"); - const secondFlow = parseFlow(secondContent); - expect(secondFlow.steps).toEqual([{ kind: "echo", message: "goes to second" }]); + expect(parseFlow(await readFlowFile("shared-name")).steps).toEqual([ + { kind: "echo", message: "in first project" }, + ]); + expect(parseFlow(await readFlowFile("shared-name", otherDir)).steps).toEqual([ + { kind: "echo", message: "in second project" }, + ]); }); - it("restarting the same flow does not report a switch", async () => { + it("restarting the same flow reports the discarded steps and resets the file", async () => { await flowStartRecordingTool.execute( {}, { name: "same-flow", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "will be reset" }); + await flowInsertEchoTool.execute( + {}, + { name: "same-flow", project_root: tmpDir, message: "will be reset" } + ); + await flowInsertEchoTool.execute( + {}, + { name: "same-flow", project_root: tmpDir, message: "also reset" } + ); const result = await flowStartRecordingTool.execute( {}, { name: "same-flow", project_root: tmpDir, executionPrerequisite: "Updated prereq" } ); - // Should NOT mention a switch — it's the same flow being restarted - expect(result.message).not.toContain("Switched"); - expect(result.previousFlow).toBeUndefined(); + expect(result.restarted).toBe(true); + expect(result.discardedSteps).toBe(2); expect(result.message).toContain("same-flow"); + + // The earlier take is gone from the file too, prerequisite included. + const flow = parseFlow(await readFlowFile("same-flow")); + expect(flow.steps).toEqual([]); + expect(flow.executionPrerequisite).toBe("Updated prereq"); + + // The restarted recording is the live one, and it starts from empty. + const echo = await flowInsertEchoTool.execute( + {}, + { name: "same-flow", project_root: tmpDir, message: "new take" } + ); + expect(parseFlow(echo.flowFile).steps).toEqual([{ kind: "echo", message: "new take" }]); }); - it("does not report a switch when no flow was previously active", async () => { + it("does not report a restart when the flow was not already recording", async () => { const result = await flowStartRecordingTool.execute( {}, { name: "fresh-start", project_root: tmpDir, executionPrerequisite: PREREQ } ); - expect(result.message).not.toContain("Switched"); - expect(result.previousFlow).toBeUndefined(); + expect(result.restarted).toBeUndefined(); + expect(result.discardedSteps).toBeUndefined(); }); }); @@ -199,7 +263,10 @@ describe("flow-add-echo", () => { {}, { name: "echo-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const result = await flowInsertEchoTool.execute({}, { message: "Hello world" }); + const result = await flowInsertEchoTool.execute( + {}, + { name: "echo-test", project_root: tmpDir, message: "Hello world" } + ); expect(result.message).toContain("echo-test"); const flow = parseFlow(result.flowFile); @@ -211,8 +278,14 @@ describe("flow-add-echo", () => { {}, { name: "multi-echo", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "First" }); - const result = await flowInsertEchoTool.execute({}, { message: "Second" }); + await flowInsertEchoTool.execute( + {}, + { name: "multi-echo", project_root: tmpDir, message: "First" } + ); + const result = await flowInsertEchoTool.execute( + {}, + { name: "multi-echo", project_root: tmpDir, message: "Second" } + ); const flow = parseFlow(result.flowFile); expect(flow.steps).toEqual([ @@ -221,10 +294,29 @@ describe("flow-add-echo", () => { ]); }); - it("throws when no active flow", async () => { - await expect(flowInsertEchoTool.execute({}, { message: "oops" })).rejects.toThrow( - "No active flow" + it("throws when that flow has no recording in progress", async () => { + await expect( + flowInsertEchoTool.execute( + {}, + { name: "not-recording", project_root: tmpDir, message: "oops" } + ) + ).rejects.toThrow("No active recording"); + }); + + it("throws when the recording is open under a different project root", async () => { + await flowStartRecordingTool.execute( + {}, + { name: "wrong-root", project_root: tmpDir, executionPrerequisite: PREREQ } ); + + // Right name, wrong project — a different key, so no recording is found. + const err = await flowInsertEchoTool + .execute({}, { name: "wrong-root", project_root: otherDir, message: "oops" }) + .catch((e: unknown) => e as Error); + + expect(err.message).toContain("No active recording"); + // The error names what IS live, so a wrong project_root is self-correcting. + expect(err.message).toContain(`Active recordings: "wrong-root" (${tmpDir})`); }); }); @@ -241,7 +333,15 @@ describe("flow-add-step", () => { {}, { name: "step-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const result = await tool.execute({}, { command: "tap", args: '{"x":0.5,"y":0.3}' }); + const result = await tool.execute( + {}, + { + name: "step-test", + project_root: tmpDir, + command: "tap", + args: '{"x":0.5,"y":0.3}', + } + ); expect(result.toolResult).toEqual({ tapped: true }); const flow = parseFlow(result.flowFile); @@ -263,7 +363,11 @@ describe("flow-add-step", () => { {}, { name: "tele-step", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await tool.execute({}, { command: "tap", args: '{"x":0.5}' }, ctx); + await tool.execute( + {}, + { name: "tele-step", project_root: tmpDir, command: "tap", args: '{"x":0.5}' }, + ctx + ); expect(recordChildInvocation).toHaveBeenCalledOnce(); const childId = recordChildInvocation.mock.calls[0]![0]; @@ -287,9 +391,12 @@ describe("flow-add-step", () => { {}, { name: "fail-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await expect(tool.execute({}, { command: "tap", args: '{"x":0.5}' })).rejects.toThrow( - 'Tool "tap" failed' - ); + await expect( + tool.execute( + {}, + { name: "fail-test", project_root: tmpDir, command: "tap", args: '{"x":0.5}' } + ) + ).rejects.toThrow('Tool "tap" failed'); const content = await readFlowFile("fail-test"); const flow = parseFlow(content); @@ -306,7 +413,7 @@ describe("flow-add-step", () => { {}, { name: "no-args", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await tool.execute({}, { command: "screenshot" }); + await tool.execute({}, { name: "no-args", project_root: tmpDir, command: "screenshot" }); const content = await readFlowFile("no-args"); const flow = parseFlow(content); @@ -314,15 +421,20 @@ describe("flow-add-step", () => { expect(registry.invokeTool).toHaveBeenCalledWith("screenshot", {}); }); - it("throws when no active flow", async () => { + it("throws when that flow has no recording in progress", async () => { const registry = createMockRegistry({ tap: { result: { ok: true } }, }); const tool = createFlowAddStepTool(registry); - await expect(tool.execute({}, { command: "tap", args: '{"x":0.5}' })).rejects.toThrow( - "No active flow" - ); + await expect( + tool.execute( + {}, + { name: "not-recording", project_root: tmpDir, command: "tap", args: '{"x":0.5}' } + ) + ).rejects.toThrow("No active recording"); + // The step must not run either — the recording is resolved first. + expect(registry.invokeTool).not.toHaveBeenCalled(); }); it("records a restart-app as a portable launch step (device id dropped)", async () => { @@ -334,7 +446,12 @@ describe("flow-add-step", () => { await flowStartRecordingTool.execute({}, { name: "launch-rewrite", project_root: tmpDir }); const result = await tool.execute( {}, - { command: "restart-app", args: '{"udid":"ABC","bundleId":"com.acme.app"}' } + { + name: "launch-rewrite", + project_root: tmpDir, + command: "restart-app", + args: '{"udid":"ABC","bundleId":"com.acme.app"}', + } ); // Ran live with the full args… @@ -356,6 +473,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "launch-activity", + project_root: tmpDir, command: "restart-app", args: '{"udid":"ABC","bundleId":"com.acme.app","activity":".Main"}', } @@ -383,7 +502,15 @@ describe("flow-add-step", () => { { name: "contradiction", project_root: tmpDir, executionPrerequisite: PREREQ } ); await expect( - tool.execute({}, { command: "restart-app", args: '{"bundleId":"com.acme.app"}' }) + tool.execute( + {}, + { + name: "contradiction", + project_root: tmpDir, + command: "restart-app", + args: '{"bundleId":"com.acme.app"}', + } + ) ).rejects.toThrow(/must not declare executionPrerequisite/i); const flow = parseFlow(await readFlowFile("contradiction")); @@ -406,6 +533,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-test", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ name: "login", @@ -434,6 +563,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-e2e", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ name: "other-e2e", project_root: tmpDir, device: "ABC" }), } @@ -454,6 +585,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-missing", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ name: "elsewhere", project_root: tmpDir }), } @@ -1138,7 +1271,10 @@ describe("flow-add-step", () => { { name: "bad-json", project_root: tmpDir, executionPrerequisite: PREREQ } ); await expect( - tool.execute({}, { command: "tap", args: "not valid json {{{" }) + tool.execute( + {}, + { name: "bad-json", project_root: tmpDir, command: "tap", args: "not valid json {{{" } + ) ).rejects.toThrow(); // Flow file should remain unchanged (no step recorded) @@ -1155,9 +1291,12 @@ describe("flow-add-step", () => { {}, { name: "missing-tool", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await expect(tool.execute({}, { command: "nonexistent-tool", args: "{}" })).rejects.toThrow( - 'Tool "nonexistent-tool" not found' - ); + await expect( + tool.execute( + {}, + { name: "missing-tool", project_root: tmpDir, command: "nonexistent-tool", args: "{}" } + ) + ).rejects.toThrow('Tool "nonexistent-tool" not found'); // Flow file should remain unchanged const content = await readFlowFile("missing-tool"); @@ -1169,28 +1308,61 @@ describe("flow-add-step", () => { // ── flow-finish-recording ──────────────────────────────────────────── describe("flow-finish-recording", () => { - it("returns summary with prerequisite and clears active flow", async () => { + it("returns summary with prerequisite and clears that recording", async () => { await flowStartRecordingTool.execute( {}, { name: "finish-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "Step 1" }); + await flowInsertEchoTool.execute( + {}, + { name: "finish-test", project_root: tmpDir, message: "Step 1" } + ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute( + {}, + { name: "finish-test", project_root: tmpDir } + ); expect(result.message).toContain("finish-test"); expect(result.executionPrerequisite).toBe(PREREQ); expect(result.steps).toBe(1); expect(result.summary).toEqual(["1. echo: Step 1"]); - // Active flow should be cleared - await expect(flowInsertEchoTool.execute({}, { message: "after finish" })).rejects.toThrow( - "No active flow" + // The recording is gone — no more steps can be added to it. + await expect( + flowInsertEchoTool.execute( + {}, + { name: "finish-test", project_root: tmpDir, message: "after finish" } + ) + ).rejects.toThrow("No active recording"); + }); + + it("leaves other recordings in progress untouched", async () => { + await flowStartRecordingTool.execute( + {}, + { name: "finish-one", project_root: tmpDir, executionPrerequisite: PREREQ } + ); + await flowStartRecordingTool.execute( + {}, + { name: "keep-going", project_root: tmpDir, executionPrerequisite: PREREQ } + ); + + await flowFinishRecordingTool.execute({}, { name: "finish-one", project_root: tmpDir }); + + const result = await flowInsertEchoTool.execute( + {}, + { name: "keep-going", project_root: tmpDir, message: "still open" } ); + expect(result.message).toContain("keep-going"); + expect(parseFlow(await readFlowFile("keep-going")).steps).toEqual([ + { kind: "echo", message: "still open" }, + ]); }); - it("throws when no active flow", async () => { - await expect(flowFinishRecordingTool.execute({}, {})).rejects.toThrow("No active flow"); + it("throws when that flow has no recording in progress", async () => { + await expect( + flowFinishRecordingTool.execute({}, { name: "not-recording", project_root: tmpDir }) + ).rejects.toThrow("No active recording"); }); it("handles empty flow", async () => { @@ -1198,7 +1370,10 @@ describe("flow-finish-recording", () => { {}, { name: "empty", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute( + {}, + { name: "empty", project_root: tmpDir } + ); expect(result.steps).toBe(0); expect(result.summary).toEqual([]); @@ -1209,10 +1384,12 @@ describe("flow-finish-recording", () => { {}, { name: "double-finish", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowFinishRecordingTool.execute({}, {}); + await flowFinishRecordingTool.execute({}, { name: "double-finish", project_root: tmpDir }); - // Second call should fail — active flow was cleared - await expect(flowFinishRecordingTool.execute({}, {})).rejects.toThrow("No active flow"); + // Second call should fail — the recording was cleared + await expect( + flowFinishRecordingTool.execute({}, { name: "double-finish", project_root: tmpDir }) + ).rejects.toThrow("No active recording"); }); it("returns the file path so the agent knows where it was written", async () => { @@ -1220,7 +1397,10 @@ describe("flow-finish-recording", () => { {}, { name: "path-check", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute( + {}, + { name: "path-check", project_root: tmpDir } + ); expect(result.path).toContain(path.join(".argent", "flows")); expect(result.path).toContain("path-check.yaml"); @@ -1236,10 +1416,19 @@ describe("flow-finish-recording", () => { {}, { name: "summary-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "Before tap" }); - await addStep.execute({}, { command: "tap", args: '{"x":0.5}' }); + await flowInsertEchoTool.execute( + {}, + { name: "summary-test", project_root: tmpDir, message: "Before tap" } + ); + await addStep.execute( + {}, + { name: "summary-test", project_root: tmpDir, command: "tap", args: '{"x":0.5}' } + ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute( + {}, + { name: "summary-test", project_root: tmpDir } + ); expect(result.summary).toEqual(["1. echo: Before tap", '2. tool: tap {"x":0.5}']); }); @@ -1286,7 +1475,7 @@ describe("flow-finish-recording", () => { }) ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute({}, { name, project_root: tmpDir }); expect(result.summary).toEqual([ '1. await: text {"id":"status"} contains "Ready \\"now\\"\\nnext"', @@ -1346,7 +1535,7 @@ describe("flow-finish-recording", () => { }) ); - const result = await flowFinishRecordingTool.execute({}, {}); + const result = await flowFinishRecordingTool.execute({}, { name, project_root: tmpDir }); expect(result.summary).toEqual([ '1. when: text {"id":"status"} contains "Ready \\"now\\"\\nnext" (1 step)', @@ -1379,11 +1568,23 @@ describe("flow-execute", () => { {}, { name: "run-test", project_root: tmpDir, executionPrerequisite: PREREQ } ); - await flowInsertEchoTool.execute({}, { message: "Tap button" }); - await addStep.execute({}, { command: "tap", args: '{"x":0.5}' }); - await flowInsertEchoTool.execute({}, { message: "Take screenshot" }); - await addStep.execute({}, { command: "screenshot", args: "{}" }); - await flowFinishRecordingTool.execute({}, {}); + await flowInsertEchoTool.execute( + {}, + { name: "run-test", project_root: tmpDir, message: "Tap button" } + ); + await addStep.execute( + {}, + { name: "run-test", project_root: tmpDir, command: "tap", args: '{"x":0.5}' } + ); + await flowInsertEchoTool.execute( + {}, + { name: "run-test", project_root: tmpDir, message: "Take screenshot" } + ); + await addStep.execute( + {}, + { name: "run-test", project_root: tmpDir, command: "screenshot", args: "{}" } + ); + await flowFinishRecordingTool.execute({}, { name: "run-test", project_root: tmpDir }); // Reset mock call counts vi.mocked(registry.invokeTool).mockClear(); @@ -1793,28 +1994,60 @@ describe("flow-execute", () => { tap: { result: { ok: true } }, }); const runFlow = createRunFlowTool(registry); + const addStep = createFlowAddStepTool(registry); - // Write a flow to run - const dir = path.join(tmpDir, ".argent", "flows"); - await fs.mkdir(dir, { recursive: true }); + // A flow to run in the recording's own project AND one in another project — + // replay must be inert for the recording either way, and a replay under a + // different project_root is exactly what a second agent's run looks like. const content = serializeFlow({ executionPrerequisite: "", steps: [{ kind: "tool", name: "tap", args: { x: 0.1 } }], }); - await fs.writeFile(path.join(dir, "side-effect.yaml"), content); + for (const root of [tmpDir, otherDir]) { + const dir = path.join(root, ".argent", "flows"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "side-effect.yaml"), content); + } // Start recording a different flow await flowStartRecordingTool.execute( {}, { name: "recording", project_root: tmpDir, executionPrerequisite: PREREQ } ); + const before = getRecordingSession(tmpDir, "recording"); + expect(before).toBeDefined(); - // Execute a saved flow — this should NOT affect the active recording + // Execute saved flows — neither should affect the active recording await runFlow.execute({}, { name: "side-effect", project_root: tmpDir, device: DEVICE }); + await runFlow.execute({}, { name: "side-effect", project_root: otherDir, device: DEVICE }); - // We should still be able to add steps to the recording - const result = await flowInsertEchoTool.execute({}, { message: "still recording" }); + // The recording still points at the flow it was opened for, in its own + // project — a replay elsewhere must not rebind name/root/file. + const after = getRecordingSession(tmpDir, "recording"); + expect(after).toBe(before); + expect(after).toMatchObject({ + name: "recording", + projectRoot: tmpDir, + filePath: path.join(tmpDir, ".argent", "flows", "recording.yaml"), + }); + + // We should still be able to add steps to the recording… + const result = await flowInsertEchoTool.execute( + {}, + { name: "recording", project_root: tmpDir, message: "still recording" } + ); expect(result.message).toContain("recording"); + await addStep.execute( + {}, + { name: "recording", project_root: tmpDir, command: "tap", args: '{"x":0.9}' } + ); + + // …and they land in the original flow's file, not the replayed project's. + expect(parseFlow(await readFlowFile("recording")).steps).toEqual([ + { kind: "echo", message: "still recording" }, + { kind: "tool", name: "tap", args: { x: 0.9 } }, + ]); + await expect(readFlowFile("recording", otherDir)).rejects.toThrow(); }); }); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 93a5ed613..1891290f7 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -5,12 +5,13 @@ import { serializeFlow, parseFlow, describeSelector, - setActiveFlow, - getActiveFlow, - getActiveFlowOrNull, - clearActiveFlow, - setActiveProjectRoot, - clearActiveProjectRoot, + assertValidProjectRoot, + startRecordingSession, + getRecordingSession, + requireRecordingSession, + clearRecordingSession, + listActiveRecordings, + clearAllRecordings, getFlowPath, appIdForPlatform, chromiumLaunchSpec, @@ -1006,99 +1007,206 @@ describe("native launch shorthand", () => { }); }); -// ── Active flow state ──────────────────────────────────────────────── +// ── Recording sessions ─────────────────────────────────────────────── -describe("active flow state", () => { +// Recordings live in a map keyed by the resolved flow file path, so a session +// has no identity beyond (project_root, name) — two agents recording at once +// must never observe each other's state. +describe("recording sessions", () => { beforeEach(() => { - clearActiveFlow(); + clearAllRecordings(); }); - it("throws when no active flow", () => { - expect(() => getActiveFlow()).toThrow("No active flow"); + const emptyFlow = (): FlowFile => ({ executionPrerequisite: "", steps: [] }); + + const start = (projectRoot: string, name: string, flow: FlowFile = emptyFlow()) => + startRecordingSession({ + name, + projectRoot, + persist: "host", + filePath: getFlowPath(projectRoot, name), + flow, + }); + + it("throws when the key has no recording", () => { + expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + /No active recording for flow "my-flow"/ + ); + }); + + it("classifies the not-found throw as FLOW_NO_ACTIVE_RECORDING", () => { + let caught: unknown; + try { + requireRecordingSession("/tmp/proj-a", "my-flow"); + } catch (err) { + caught = err; + } + expect(getFailureSignal(caught)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); }); - it("returns the active flow after setActiveFlow", () => { - setActiveFlow("my-flow"); - expect(getActiveFlow()).toBe("my-flow"); + it("names the asked-for key and lists the live recordings in the not-found message", () => { + // With concurrent recordings the usual cause is a typo or the wrong + // project_root; the agent can only self-correct if it sees the live keys. + start("/tmp/proj-a", "checkout"); + start("/tmp/proj-b", "login"); + expect(() => requireRecordingSession("/tmp/proj-a", "chekout")).toThrow( + /No active recording for flow "chekout" in \/tmp\/proj-a\./ + ); + expect(() => requireRecordingSession("/tmp/proj-a", "chekout")).toThrow( + /Active recordings: "checkout" \(\/tmp\/proj-a\), "login" \(\/tmp\/proj-b\)\./ + ); }); - it("clears the active flow", () => { - setActiveFlow("my-flow"); - clearActiveFlow(); - expect(() => getActiveFlow()).toThrow("No active flow"); + it('reports "none" when nothing is being recorded', () => { + expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + /Active recordings: none\./ + ); }); - it("overwrites previous active flow", () => { - setActiveFlow("first"); - setActiveFlow("second"); - expect(getActiveFlow()).toBe("second"); + it("returns the session that was started for that key", () => { + start("/tmp/proj-a", "my-flow"); + const session = requireRecordingSession("/tmp/proj-a", "my-flow"); + expect(session.name).toBe("my-flow"); + expect(session.projectRoot).toBe("/tmp/proj-a"); + expect(session.persist).toBe("host"); + expect(session.filePath).toBe(getFlowPath("/tmp/proj-a", "my-flow")); }); - it("getActiveFlowOrNull returns null when no active flow", () => { - expect(getActiveFlowOrNull()).toBeNull(); + it("getRecordingSession returns undefined for a key with no recording", () => { + expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); }); - it("getActiveFlowOrNull returns the active flow name", () => { - setActiveFlow("my-flow"); - expect(getActiveFlowOrNull()).toBe("my-flow"); + it("getRecordingSession returns the live session", () => { + start("/tmp/proj-a", "my-flow"); + expect(getRecordingSession("/tmp/proj-a", "my-flow")?.name).toBe("my-flow"); }); - it("getActiveFlowOrNull returns null after clearing", () => { - setActiveFlow("my-flow"); - clearActiveFlow(); - expect(getActiveFlowOrNull()).toBeNull(); + it("clearRecordingSession removes only that key", () => { + start("/tmp/proj-a", "my-flow"); + start("/tmp/proj-a", "other-flow"); + clearRecordingSession("/tmp/proj-a", "my-flow"); + expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); + expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + /No active recording for flow "my-flow"/ + ); + // The unrelated recording is untouched. + expect(requireRecordingSession("/tmp/proj-a", "other-flow").name).toBe("other-flow"); + }); + + it("keeps same-named recordings under different project roots independent", () => { + start("/tmp/proj-a", "my-flow", { executionPrerequisite: "A", steps: [] }); + start("/tmp/proj-b", "my-flow", { executionPrerequisite: "B", steps: [] }); + expect(requireRecordingSession("/tmp/proj-a", "my-flow").flow.executionPrerequisite).toBe("A"); + expect(requireRecordingSession("/tmp/proj-b", "my-flow").flow.executionPrerequisite).toBe("B"); + // Finishing one leaves the other recording. + clearRecordingSession("/tmp/proj-a", "my-flow"); + expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); + expect(requireRecordingSession("/tmp/proj-b", "my-flow").flow.executionPrerequisite).toBe("B"); + }); + + it("returns null when starting a recording on a free key", () => { + expect(start("/tmp/proj-a", "my-flow")).toBeNull(); + // A second, unrelated recording is the common concurrent case — not a replace. + expect(start("/tmp/proj-a", "other-flow")).toBeNull(); + expect(start("/tmp/proj-b", "my-flow")).toBeNull(); + }); + + it("returns the replaced session when re-recording the same key", () => { + start("/tmp/proj-a", "my-flow", { executionPrerequisite: "first take", steps: [] }); + const replaced = start("/tmp/proj-a", "my-flow", { + executionPrerequisite: "second take", + steps: [], + }); + expect(replaced?.flow.executionPrerequisite).toBe("first take"); + // The later take wins — one key, one writer. + expect(requireRecordingSession("/tmp/proj-a", "my-flow").flow.executionPrerequisite).toBe( + "second take" + ); + }); + + it("listActiveRecordings reflects what is live", () => { + expect(listActiveRecordings()).toEqual([]); + start("/tmp/proj-a", "my-flow", { + executionPrerequisite: "", + steps: [{ kind: "echo", message: "hi" }], + }); + start("/tmp/proj-b", "my-flow"); + expect(listActiveRecordings()).toEqual([ + { name: "my-flow", projectRoot: "/tmp/proj-a", steps: 1 }, + { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, + ]); + clearRecordingSession("/tmp/proj-a", "my-flow"); + expect(listActiveRecordings()).toEqual([ + { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, + ]); + clearAllRecordings(); + expect(listActiveRecordings()).toEqual([]); + }); + + it("keys a session by the normalized flow path, so a trailing slash rejoins it", () => { + start("/tmp/proj-a", "my-flow"); + expect(requireRecordingSession("/tmp/proj-a/", "my-flow").name).toBe("my-flow"); + expect(start("/tmp/proj-a/", "my-flow")).not.toBeNull(); + expect(listActiveRecordings()).toHaveLength(1); }); }); // ── getFlowPath name validation ────────────────────────────────────── describe("getFlowPath name validation", () => { - beforeEach(() => { - clearActiveProjectRoot(); - setActiveProjectRoot("/tmp/argent-flow-name-test"); - }); + // Pure path math over two explicit inputs — the root is a parameter, never + // shared state, so two callers naming two projects can never collide. + const root = "/tmp/argent-flow-name-test"; it("accepts plain alphanumeric names", () => { - expect(getFlowPath("my-flow_1")).toBe( - path.join("/tmp/argent-flow-name-test", ".argent", "flows", "my-flow_1.yaml") + expect(getFlowPath(root, "my-flow_1")).toBe( + path.join(root, ".argent", "flows", "my-flow_1.yaml") ); }); + it("normalizes a trailing slash on the project root", () => { + // The flow path doubles as the recording-session key: a trailing slash must + // not mint a second identity for the same file. + expect(getFlowPath("/tmp/x/", "f")).toBe(getFlowPath("/tmp/x", "f")); + }); + it("rejects path-traversal segments", () => { - expect(() => getFlowPath("../../etc/passwd")).toThrow(/Invalid flow name/); - expect(() => getFlowPath("../foo")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "../../etc/passwd")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "../foo")).toThrow(/Invalid flow name/); }); it("rejects path separators", () => { - expect(() => getFlowPath("foo/bar")).toThrow(/Invalid flow name/); - expect(() => getFlowPath("/abs/path")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "foo/bar")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "/abs/path")).toThrow(/Invalid flow name/); }); it("rejects names with spaces or shell metacharacters", () => { - expect(() => getFlowPath("foo bar")).toThrow(/Invalid flow name/); - expect(() => getFlowPath("foo;bar")).toThrow(/Invalid flow name/); - expect(() => getFlowPath("foo$(id)")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "foo bar")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "foo;bar")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "foo$(id)")).toThrow(/Invalid flow name/); }); it("rejects empty names", () => { - expect(() => getFlowPath("")).toThrow(/Invalid flow name/); + expect(() => getFlowPath(root, "")).toThrow(/Invalid flow name/); }); }); // PR #194 follow-up C: project_root must be absolute AND free of ".." // segments (path.join collapses ".." and would relocate the flows dir). -describe("setActiveProjectRoot validation", () => { +describe("assertValidProjectRoot validation", () => { it("rejects a relative project_root", () => { - expect(() => setActiveProjectRoot("relative/path")).toThrow(/absolute path/); + expect(() => assertValidProjectRoot("relative/path")).toThrow(/absolute path/); }); it('rejects an absolute project_root containing ".." segments', () => { - expect(() => setActiveProjectRoot("/a/../../../etc")).toThrow(/must not contain "\.\."/); - expect(() => setActiveProjectRoot("/home/user/../../root")).toThrow(/must not contain "\.\."/); + expect(() => assertValidProjectRoot("/a/../../../etc")).toThrow(/must not contain "\.\."/); + expect(() => assertValidProjectRoot("/home/user/../../root")).toThrow( + /must not contain "\.\."/ + ); }); it("accepts a clean absolute project_root", () => { - expect(() => setActiveProjectRoot("/tmp/argent-pr194-c-test")).not.toThrow(); + expect(() => assertValidProjectRoot("/tmp/argent-pr194-c-test")).not.toThrow(); }); }); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 303c37e2b..5b7d1c045 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -141,7 +141,7 @@ describe("stop-all-simulator-servers", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, undefined); + const result = await tool.execute!({}, {}); expect(result).toEqual({ stopped: ["SimulatorServer:AAA", "SimulatorServer:BBB"], @@ -156,7 +156,7 @@ describe("stop-all-simulator-servers", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, undefined); + const result = await tool.execute!({}, {}); expect(result).toEqual({ stopped: [] }); expect(registry.disposeService).not.toHaveBeenCalled(); @@ -170,7 +170,7 @@ describe("stop-all-simulator-servers", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, undefined); + const result = await tool.execute!({}, {}); expect(result).toEqual({ stopped: ["SimulatorServer:BBB"] }); expect(registry.disposeService).toHaveBeenCalledOnce(); @@ -184,7 +184,7 @@ describe("stop-all-simulator-servers", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, undefined); + const result = await tool.execute!({}, {}); // Both get disposed (cleanup), but only the live one is reported as stopped. expect(result).toEqual({ stopped: ["SimulatorServer:BBB"] }); @@ -204,7 +204,7 @@ describe("stop-all-simulator-servers", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, undefined); + const result = await tool.execute!({}, {}); expect(result).toEqual({ stopped: ["TvControl:APPLE-TV", "AndroidTvControl:emulator-5556", "SimulatorServer:BBB"], @@ -215,6 +215,163 @@ describe("stop-all-simulator-servers", () => { }); }); +describe("stop-all-simulator-servers device scoping", () => { + // The tool-server is a host-wide singleton, so an unscoped teardown reaps + // whatever device another agent is mid-session on. `devices` narrows the + // sweep to the ids the calling session actually used. + const MINE = "AAAA-1111"; + const THEIRS = "BBBB-2222"; + + function twoAgentServices() { + return new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ["ChromiumCdp:chromium-cdp-9222", { state: ServiceState.RUNNING, dependents: [] }], + ]); + } + + it("disposes only the named device's URNs and leaves the other device live", async () => { + const registry = createMockRegistry(twoAgentServices()); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE}`], + }); + expect(registry.disposeService).toHaveBeenCalledTimes(2); + expect(registry.disposeService).not.toHaveBeenCalledWith(`SimulatorServer:${THEIRS}`); + expect(registry.disposeService).not.toHaveBeenCalledWith(`NativeDevtools:${THEIRS}`); + expect(registry.disposeService).not.toHaveBeenCalledWith("ChromiumCdp:chromium-cdp-9222"); + }); + + it("scopes across platforms when several device ids are named", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ["AndroidDevtools:emulator-5554", { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE, "emulator-5554"] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, "AndroidDevtools:emulator-5554"], + }); + expect(registry.disposeService).toHaveBeenCalledTimes(2); + }); + + it("still disposes everything when no devices are named", async () => { + const registry = createMockRegistry(twoAgentServices()); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, {}); + + expect(result).toEqual({ + stopped: [ + `SimulatorServer:${MINE}`, + `NativeDevtools:${MINE}`, + `SimulatorServer:${THEIRS}`, + `NativeDevtools:${THEIRS}`, + "ChromiumCdp:chromium-cdp-9222", + ], + }); + expect(registry.disposeService).toHaveBeenCalledTimes(5); + }); + + it("matches a transport-suffixed URN (NativeDevtools::tcp)", async () => { + const services = new Map([ + [`NativeDevtools:${MINE}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${THEIRS}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`NativeDevtools:${MINE}:tcp`] }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + expect(registry.disposeService).toHaveBeenCalledWith(`NativeDevtools:${MINE}:tcp`); + }); + + it("matches a device id that itself contains a colon (wireless adb serial)", async () => { + const wireless = "192.168.1.5:5555"; + const services = new Map([ + [`AndroidDevtools:${wireless}`, { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [wireless] }); + + expect(result).toEqual({ stopped: [`AndroidDevtools:${wireless}`] }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + }); + + it("matches the device id case-insensitively", async () => { + // iOS UDIDs are conventionally upper-case, but an agent passes through + // whatever it was handed — a case mismatch must not silently no-op. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE.toLowerCase()}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE.toLowerCase()] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE.toLowerCase()}:tcp`], + }); + expect(registry.disposeService).not.toHaveBeenCalledWith(`SimulatorServer:${THEIRS}`); + }); + + it("scopes to nothing for devices: [] rather than sweeping the machine", async () => { + // A caller that computed a device list and got none must not fall back to + // tearing down every other agent's services. + const registry = createMockRegistry(twoAgentServices()); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [] }); + + expect(result).toEqual({ stopped: [] }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + + it("does not match a device id that is a prefix of another device's id", async () => { + const services = new Map([ + ["SimulatorServer:AAAA", { state: ServiceState.RUNNING, dependents: [] }], + ["SimulatorServer:AAAA-EXTRA", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: ["AAAA"] }); + + expect(result).toEqual({ stopped: ["SimulatorServer:AAAA"] }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + }); + + it("skips an IDLE service on the named device", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.IDLE, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`NativeDevtools:${MINE}`] }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + }); +}); + describe("stop-metro", () => { it("defaults to port 8081", () => { expect(stopMetroTool.zodSchema).toBeDefined(); From f7f1a26a6451eaba522c5541122ac2ee82c06b1f Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 18:58:51 +0200 Subject: [PATCH 03/98] fix(flow): serialize every mutation of a flow file, not just appends Review of the map-keyed sessions turned up two races that survived the per-session append chain, both reproducible through the HTTP API: - flow-start-recording truncated the .yaml outside any lock and registered the replacing session two statements later, so a step from the take being discarded landed in the freshly reset file (26-29 of 200 runs) and was reported as success. - flow-finish-recording never took the lock at all; its await fs.readFile was a yield during which a concurrent append committed, leaving the returned steps/summary/flowFile disagreeing with disk (11-16 of 200). The lock is now keyed by the flow file rather than owned by the session -- a restart replaces the session, so a session-owned lock could not exclude the operation superseding it. start's truncate+register and finish's read+clear each run inside it, and an append whose session was finished, restarted or evicted meanwhile fails with FLOW_NO_ACTIVE_RECORDING instead of writing into another take. Still per file: two recordings never queue behind each other. Also from review: client-mode rollback now covers serializeFlow, not just validateFlow (a step that failed to serialize used to stay in the in-memory copy and poison every later call, unrecoverably); stop-all-simulator-servers reports unmatched device ids so a mistyped id no longer reads as a clean machine; and the test-reset export follows the house __resetXForTesting convention. --- .../test/run-flow-add-step-payload.test.ts | 108 +++- .../skills/skills/argent-create-flow/SKILL.md | 42 +- .../src/tools/flows/flow-finish-recording.ts | 44 +- .../tool-server/src/tools/flows/flow-run.ts | 6 +- .../src/tools/flows/flow-start-recording.ts | 58 ++- .../tool-server/src/tools/flows/flow-utils.ts | 131 +++-- .../simulator/stop-all-simulator-servers.ts | 48 +- .../test/failure-classification.test.ts | 73 +++ .../flows/flow-concurrent-recording.test.ts | 483 +++++++++++++++++- .../test/flows/flow-record-tap.test.ts | 6 +- .../test/flows/flow-remote-recording.test.ts | 88 +++- .../tool-server/test/flows/flow-tools.test.ts | 6 +- .../tool-server/test/flows/flow-utils.test.ts | 6 +- packages/tool-server/test/stop-tools.test.ts | 153 +++++- 14 files changed, 1088 insertions(+), 164 deletions(-) diff --git a/packages/argent-cli/test/run-flow-add-step-payload.test.ts b/packages/argent-cli/test/run-flow-add-step-payload.test.ts index 8426b0013..0579ec4f5 100644 --- a/packages/argent-cli/test/run-flow-add-step-payload.test.ts +++ b/packages/argent-cli/test/run-flow-add-step-payload.test.ts @@ -5,11 +5,13 @@ import { run, type RunCommandOptions } from "../src/run.js"; // End-to-end regression guard for issue #452 at the `run()` layer. // // The documented per-flag form -// argent run flow-add-step --command gesture-tap --args '{"udid":...}' -// must reach the tool-server with BOTH `command` AND the tool's own `args` -// field in the payload. The bug shadowed the `args` field with the -// whole-payload escape hatch, so `args` was consumed as the entire payload and -// the field arrived `undefined` (with udid/x/y hoisted to the top level). +// argent run flow-add-step --name t --project_root /p --command gesture-tap \ +// --args '{"udid":...}' +// must reach the tool-server with the recording identity (`name` + +// `project_root`), the `command`, AND the tool's own `args` field in the +// payload. The bug shadowed the `args` field with the whole-payload escape +// hatch, so `args` was consumed as the entire payload and the field arrived +// `undefined` (with udid/x/y hoisted to the top level). // // `parseFlags` is unit-tested directly, and `--help` suppression is covered in // run-help.test.ts. Neither drives the whole `run()` path through to the wire. @@ -33,14 +35,23 @@ function startServer(cap: Captured): Promise<{ url: string; close: () => Promise { name: "flow-add-step", description: "Add a step to the active flow recording", + // Mirrors what the registry advertises for the real tool — + // zodObjectToJsonSchema over the zod schema in + // packages/tool-server/src/tools/flows/flow-add-step.ts. `name` + // and `project_root` identify which open recording the step + // belongs to and are required alongside `command`; a fixture that + // still advertised the old single-required shape would let a + // regression in how those flags are parsed slip through. inputSchema: { type: "object", properties: { + name: { type: "string" }, + project_root: { type: "string" }, command: { type: "string" }, args: { type: "string" }, - delayMs: { type: "integer" }, + delayMs: { type: "integer", minimum: 0, maximum: 9007199254740991 }, }, - required: ["command"], + required: ["name", "project_root", "command"], }, }, ], @@ -88,6 +99,11 @@ describe("CLI run — flow-add-step --args reaches the payload (issue #452)", () const opts: RunCommandOptions = { paths: {} as never }; // unused: ARGENT_TOOLS_URL is set + const FLOW = "checkout-e2e"; + // A path with a space: the shell hands argv already split, so the value must + // arrive verbatim rather than being re-split or truncated by the parser. + const ROOT = "/Users/dev/My Projects/demo-app"; + beforeEach(async () => { cap = { path: null, body: null }; server = await startServer(cap); @@ -109,10 +125,23 @@ describe("CLI run — flow-add-step --args reaches the payload (issue #452)", () await server.close(); }); - it("per-flag form: --command X --args '' sends BOTH fields verbatim to the server", async () => { + it("per-flag form: every required field plus --args '' reaches the server verbatim", async () => { const stepArgs = '{"udid":"SIM-1","x":0.5,"y":0.35}'; - await run(["flow-add-step", "--command", "gesture-tap", "--args", stepArgs], opts); + await run( + [ + "flow-add-step", + "--name", + FLOW, + "--project_root", + ROOT, + "--command", + "gesture-tap", + "--args", + stepArgs, + ], + opts + ); expect(cap.path).toMatch(/^\/tools\/flow-add-step/); expect(cap.body).not.toBeNull(); @@ -120,15 +149,68 @@ describe("CLI run — flow-add-step --args reaches the payload (issue #452)", () // The exact regression from #452: `args` survives as the tool's own string // field (the raw JSON passed through untouched), and its keys are NOT // hoisted to the top level as they were when `--args` was swallowed whole. - expect(payload).toEqual({ command: "gesture-tap", args: stepArgs }); + // The recording identity rides alongside it — without both `name` and + // `project_root` the server cannot find the open recording, so a payload + // missing either is a failed step, not a mislabelled one. + expect(payload).toEqual({ + name: FLOW, + project_root: ROOT, + command: "gesture-tap", + args: stepArgs, + }); }); - it("inline --args= form also sends both fields", async () => { + it("inline --field= form sends the same payload", async () => { const stepArgs = '{"udid":"SIM-1","x":0.5,"y":0.35}'; - await run(["flow-add-step", "--command", "gesture-tap", `--args=${stepArgs}`], opts); + await run( + [ + "flow-add-step", + `--name=${FLOW}`, + `--project_root=${ROOT}`, + "--command", + "gesture-tap", + `--args=${stepArgs}`, + ], + opts + ); + + const payload = JSON.parse(cap.body!) as Record; + expect(payload).toEqual({ + name: FLOW, + project_root: ROOT, + command: "gesture-tap", + args: stepArgs, + }); + }); + + it("coerces --delayMs by its declared integer type and omits absent optionals", async () => { + // `delayMs` is the only non-string field in the schema, so it is the one + // place the payload can arrive with the wrong JSON type: a string "250" + // fails the server's zod validation. `args` is optional — omitting the flag + // must leave the key out rather than sending null/"". + await run( + [ + "flow-add-step", + "--name", + FLOW, + "--project_root", + ROOT, + "--command", + "screenshot", + "--delayMs", + "250", + ], + opts + ); const payload = JSON.parse(cap.body!) as Record; - expect(payload).toEqual({ command: "gesture-tap", args: stepArgs }); + expect(payload).toEqual({ + name: FLOW, + project_root: ROOT, + command: "screenshot", + delayMs: 250, + }); + expect(payload).not.toHaveProperty("args"); }); }); diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index bf2c0175d..bf88b6247 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -126,20 +126,22 @@ The standalone command uses only the auto-started local tool server. It is unava ## Tools -| Tool | Purpose | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------------- | -| `flow-start-recording` | Start recording — takes `name` + `project_root` and (fragments only) an optional `executionPrerequisite`; creates the file | -| `flow-add-step` | Execute a tool call live and, if it succeeds, record it into the flow named by `name` + `project_root` | -| `flow-add-echo` | Add a label/comment that prints during replay, into the flow named by `name` + `project_root` | -| `flow-finish-recording` | Stop recording the flow named by `name` + `project_root` and get a summary | -| `flow-read-prerequisite` | Read a flow's execution prerequisite without running it (same `name`/`flow_path` sources) | -| `flow-execute` | Replay a flow — a saved one by `name`, or any flow YAML by absolute `flow_path` | +| Tool | Purpose | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `flow-start-recording` | Start recording — takes `name` + `project_root` and (fragments only) an optional `executionPrerequisite`; creates the file, truncating any existing one | +| `flow-add-step` | Execute a tool call live and, if it succeeds, record it into the flow named by `name` + `project_root` | +| `flow-add-echo` | Add a label/comment that prints during replay, into the flow named by `name` + `project_root` | +| `flow-finish-recording` | Stop recording the flow named by `name` + `project_root` and get a summary | +| `flow-read-prerequisite` | Read a flow's execution prerequisite without running it (same `name`/`flow_path` sources) | +| `flow-execute` | Replay a flow — a saved one by `name`, or any flow YAML by absolute `flow_path` | Every tool during recording returns the current flow file contents, so you can track what has been recorded. Rules: - **Every step runs live.** You see the real tool result (including screenshots) — verify the step worked before continuing. **Only successful steps are recorded**: a failed call writes nothing to the flow file; fix the issue and try again. - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. -- **Concurrent recordings are isolated.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects, on one device or several — and never cross-talk. Re-calling `flow-start-recording` for the **same** name + project restarts that one and only that one: the response carries `restarted: true` and `discardedSteps`, and the `.yaml` is reset to an empty flow. Starting a _different_ flow abandons nothing. +- **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. +- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. +- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. The symptom is a step in flight failing with `Recording of "" in is no longer active — it was restarted while this step was running…`; your _next_ `flow-add-step`/`flow-add-echo` then succeeds — into the other agent's recording. Restart under a fresh name instead of re-adding the step. - **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...` — the tail lists every live recording as `"name" (project_root)`, or `none`, so a typo or a wrong `project_root` is visible in the error itself. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. @@ -155,14 +157,22 @@ command: "await-ui-element" args: "{\"udid\": \"\", \"condition\": \"visible\", \"selector\": {\"text\": \"Continue\"}}" ``` +Recording a `flow-execute` step carries **two** `name`s: the top-level `name` is the recording being appended to, `args.name` is the flow being run (captured as a `run:` step). + +``` +name: "checkout-e2e" project_root: "/Users/dev/MyApp" +command: "flow-execute" +args: "{\"name\": \"login\", \"project_root\": \"/Users/dev/MyApp\"}" +``` + Record an `await-ui-element` step to **gate** the next step on a screen transition — it blocks until the element is `visible`/`hidden` (or contains `text`), so the following step runs only once the screen has actually settled; prefer this over a fixed `delayMs`. If its condition is not met before the timeout, replay **stops at that step** (the steps after it assume the transition happened). See the `await-ui-element` section of `argent-device-interact` for the full condition/selector reference. The live call sees only the trimmed `describe` tree — if it can't find an identifier you know exists, gate on visible text to get the step recorded, then retarget the identifier in the `await:` form during polish (the directive resolves the full hierarchy — see Selectors); don't conclude the testID is unusable in the flow. ## Recording 1. **Start, then launch as the first step (e2e) or set the stage yourself (fragment).** Call `flow-start-recording` with a descriptive name and the absolute `project_root`. For an **e2e** flow, record a `restart-app` of the app under test as the **first** step — it runs live (resetting the device for the rest of the recording) and is captured as the flow's `launch` step (`restart-app` has no chromium support, so on Chromium record the flow as a fragment against the running app and add the `launch:` line to the YAML afterward, deleting the `executionPrerequisite` line if you passed one — a launch-first flow must not declare it). For a **fragment**, bring the device to the entry state _before_ recording and pass an `executionPrerequisite` describing it (e.g. "App on the login screen") to `flow-start-recording` instead. -2. **Build step-by-step**: for each action, call `flow-add-step` with the tool name and args. The tool runs immediately — check the result before moving on, and gate each navigation with an `await-ui-element` step. -3. **Add labels**: use `flow-add-echo` between steps — echo the expected state, not just the action (see _Making flows resilient_). -4. **Finish**: call `flow-finish-recording`. It returns the file path where the flow was saved and a summary of all steps. +2. **Build step-by-step**: for each action, call `flow-add-step` with the same `name` + `project_root`, plus the tool name and args. The tool runs immediately — check the result before moving on, and gate each navigation with an `await-ui-element` step. +3. **Add labels**: use `flow-add-echo` (same `name` + `project_root`) between steps — echo the expected state, not just the action (see _Making flows resilient_). +4. **Finish**: call `flow-finish-recording` with the same `name` + `project_root`. It returns the file path where the flow was saved and a summary of all steps. 5. **Polish**: **read the saved `.yaml` file** and convert the raw `tool:` steps that have a cleaner directive form (the recorder leaves these as tools): - `tool: keyboard` typing into a field → `type: { into: "", text: "…" }`, folding in the `tap` that focused the field. - `tool: await-ui-element` gating a transition → `await: { visible: "…" }` / `{ hidden: … }` / `{ text: { in: …, equals: … } }`, carrying a custom `timeoutMs` over as a `timeout` sibling key. Converting also upgrades the wait from the trimmed `describe` tree to the flow's full-hierarchy tree (see Selectors). Keep the raw `tool: await-ui-element` step only when it sets a custom `pollIntervalMs`/`bundleId` the directive can't express. @@ -191,7 +201,7 @@ Then polish the saved file: the two `await-ui-element` steps become `await:` dir ## Replaying -Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow is the exception: with no `device` it boots its own instance from the `launch` path and tears it down after, so leave `device` unset there unless you mean to attach to a running one.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. +Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow can boot its own instance from the `launch` path and tear it down after, but only with no `device` **and** an unambiguous chromium target: an explicit `platform: "chromium"`, or a single-key `launch: { chromium: … }` map. A bare-string `launch:` — what the recorder always writes — or a multi-platform map carries no hint and falls through to ordinary device auto-detection, so pass `platform: "chromium"` when you want the self-boot.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. **What each step reports.** Raw `tool:` steps include the underlying tool's full `result` (screenshots and other outputs render as usual). The directive steps are summarized: `tap`/`type`/`await`/`assert` report only `status` + `reason`, and `snapshot` adds `artifacts` only when there is something to look at — a failed comparison (baseline/current/diff paths), a missing-baseline failure (`current` only), or a baseline write; a clean pass reports just `status` + `reason`. So converting a `tool: gesture-tap` into a `tap:` directive during cleanup drops only that tap's (uninteresting) raw result — output-bearing tools like `screenshot` have no directive form and stay `tool:` steps, so their results keep flowing through. @@ -253,7 +263,7 @@ For silent misfires and partial divergence, echo annotations (see _Making flows `debugger-component-tree` is an **authoring aid only — never record a `debugger-*` step into a flow.** `device_id` is stripped at record time and re-injected at replay, but `port` is not a device-bind key, so a recorded debugger step carries whatever `port` it was given (or falls through to the 8081 default at replay) and runs against whatever Metro happens to be on that port. - When calling it directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. + When calling any `debugger-*` tool directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. That last one does not rescue `debugger-component-tree` itself: it is capability-gated off Vega, and on RN 0.72's Hermes it hangs until timeout (the binding it delivers the tree over is never installed) — use `describe` there. 4. Compare current state to what the failed step expected. Classify the root cause: @@ -278,10 +288,10 @@ Read `.argent/flows/.yaml`, update the broken step's `x`/`y`, `bundle Manually execute the failed step with corrected coordinates from the Diagnose step, then manually execute remaining steps. Does not fix the YAML — use only when re-recording is not worth it. **Strategy 3 — Re-record from failure point** (structural changes, new intermediate screens). -Navigate the app to the state just before the failure point. Call `flow-start-recording` with the same flow name (overwrites). Re-add the working prefix steps via `flow-add-step`, then continue recording new steps from the divergence point. Call `flow-finish-recording`. +Navigate the app to the state just before the failure point. Call `flow-start-recording` with the same `name` + `project_root` — the start truncates the saved `.yaml` immediately, so copy the working prefix out of the file first. Re-add that prefix via `flow-add-step` (same `name` + `project_root`), then continue recording new steps from the divergence point. Call `flow-finish-recording` with the same `name` + `project_root`. **Strategy 4 — Full re-record** (major changes, unclear diagnosis, or 3+ broken steps). -Reset the app to prerequisite state (`restart-app` + `launch-app`). Record from scratch with the same flow name. +Reset the app to prerequisite state (`restart-app` + `launch-app`). Record from scratch with the same `name` + `project_root` — the start truncates the old `.yaml`, so keep a copy if you may want to diff against it. **Decision heuristic:** diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 6730ddb00..74005f865 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -4,6 +4,7 @@ import type { ToolDefinition } from "@argent/registry"; import { requireRecordingSession, clearRecordingSession, + withFlowFileLock, clientFileDirective, parseFlow, serializeFlow, @@ -87,22 +88,33 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps zodSchema, services: () => ({}), async execute(_services, params) { - const session = requireRecordingSession(params.project_root, params.name); + // Resolve, read and clear as ONE critical section under the flow-file lock. + // Host mode's `await fs.readFile` is a yield, and an append that lands in it + // would be on disk while the summary and step count reported here — taken + // from the pre-append read — say otherwise. + const { filePath, flowFile, savedTo, flow } = await withFlowFileLock( + params.project_root, + params.name, + async () => { + const session = requireRecordingSession(params.project_root, params.name); - // Host mode re-reads the file so manual edits made during the recording - // survive into the summary; in client mode this host never has the file, - // so the in-memory copy is the truth and travels back in the directive. - const filePath = session.filePath; - let flowFile: string; - let savedTo: FlowSavedTo; - if (session.persist === "client") { - flowFile = serializeFlow(session.flow); - savedTo = clientFileDirective(filePath, flowFile); - } else { - flowFile = await fs.readFile(filePath, "utf8"); - savedTo = filePath; - } - const flow = parseFlow(flowFile); + // Host mode re-reads the file so manual edits made during the recording + // survive into the summary; in client mode this host never has the file, + // so the in-memory copy is the truth and travels back in the directive. + const filePath = session.filePath; + let flowFile: string; + let savedTo: FlowSavedTo; + if (session.persist === "client") { + flowFile = serializeFlow(session.flow); + savedTo = clientFileDirective(filePath, flowFile); + } else { + flowFile = await fs.readFile(filePath, "utf8"); + savedTo = filePath; + } + clearRecordingSession(params.project_root, params.name); + return { filePath, flowFile, savedTo, flow: parseFlow(flowFile) }; + } + ); const summary = flow.steps.map((step, i) => { const n = i + 1; @@ -159,8 +171,6 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps } }); - clearRecordingSession(params.project_root, params.name); - return { message: `Finished recording "${params.name}" flow (${flow.steps.length} steps)`, path: filePath, diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index d615a2f98..aa09ae939 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -101,12 +101,14 @@ const zodSchema = z .string() .optional() .describe( - "Device id to run against (iOS UDID, Android/Vega serial, Chromium id). Auto-detected when omitted." + "Device id to run against (iOS UDID, Android/Vega serial, Chromium id) — the id list-devices reports. Auto-detected when omitted, but only when exactly one booted device matches (optionally narrowed by `platform`); with several booted the run fails and lists them, so pass this explicitly whenever more than one device is up." ), platform: z .enum(LAUNCH_PLATFORMS) .optional() - .describe("Restrict auto-detection to this platform when several devices are booted."), + .describe( + "Restrict auto-detection to this platform when several devices are booted. `chromium` does more than filter: with no `device` it makes an e2e flow boot its own Electron instance from its `launch` step's chromium app path and tear it down after the run (a single-key `launch: { chromium: … }` map does that on its own; a bare-string `launch:` never does)." + ), updateBaselines: z .boolean() .optional() diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 60f7225e8..c4095115c 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -1,10 +1,11 @@ import { z } from "zod"; import * as fs from "node:fs/promises"; import type { FileInputSpec, ToolDefinition } from "@argent/registry"; +import * as path from "node:path"; import { - getFlowsDir, getFlowPath, startRecordingSession, + withFlowFileLock, clientFileDirective, serializeFlow, validateFlow, @@ -62,13 +63,17 @@ export const flowStartRecordingTool: ToolDefinition< }, description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory. Use when you want to capture a reusable sequence of device interactions for later replay. -Returns { message, flowFile, savedTo }, plus { restarted, discardedSteps } when this call re-started a recording of the SAME flow — the earlier take is discarded and its .yaml is reset to an empty flow, so re-record from the top rather than expecting to resume. +Returns { message, flowFile, savedTo }. +Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted, discardedSteps } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. Either way, re-record from the top rather than expecting to resume. Fails if the .argent/flows/ directory cannot be created or the flow file cannot be written. -Recordings are independent: several flows can be recorded at once (different -names, different projects, different devices) with no cross-talk. Every -subsequent recording tool takes the same \`name\` + \`project_root\` to say which -one it is addressing. +Recording state is independent: several flows can be recorded at once (different +names, different projects) and one recording's steps never land in another's +file. Steps still execute LIVE on a device, so give each concurrent recording its +own device. Every subsequent recording tool takes the same \`name\` + +\`project_root\` to say which one it is addressing — and the (project_root, name) +key has no ownership check, so pick a name unique to your task or another agent +starting the same one takes the key and your next step lands in its recording. After starting, use flow-add-step to append tool calls — each step is executed LIVE so you can verify it works before it gets recorded. For a self-contained @@ -99,21 +104,32 @@ to remove or reorder steps.`, const probe = ctx?.fileInputs?.project_root; const persist = probe && !probe.presentOnHost ? "client" : "host"; - let savedTo: FlowSavedTo; - if (persist === "host") { - await fs.mkdir(getFlowsDir(params.project_root), { recursive: true }); - await fs.writeFile(filePath, flowFile, "utf8"); - savedTo = filePath; - } else { - savedTo = clientFileDirective(filePath, flowFile); - } - const replaced = startRecordingSession({ - name: params.name, - projectRoot: params.project_root, - persist, - filePath, - flow, - }); + // Truncate-and-register is one critical section. Held under the flow-file + // lock, so a step from the take being discarded can neither slip into the + // file between the reset and the swap, nor be written after both: it finds + // its session superseded and fails instead. + const { savedTo, replaced } = await withFlowFileLock( + params.project_root, + params.name, + async () => { + let savedTo: FlowSavedTo; + if (persist === "host") { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, flowFile, "utf8"); + savedTo = filePath; + } else { + savedTo = clientFileDirective(filePath, flowFile); + } + const replaced = startRecordingSession({ + name: params.name, + projectRoot: params.project_root, + persist, + filePath, + flow, + }); + return { savedTo, replaced }; + } + ); // Only a same-key restart replaces anything — the documented "re-record it // to fix it" workflow. Starting a *different* flow abandons nothing, so diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index e21e90e49..04ec72510 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -96,10 +96,17 @@ export function assertSafeFlowName(name: string): void { /** * The flow file `/.argent/flows/.yaml`. Pure path math over - * two validated inputs — it reads no shared state, so two callers naming two - * different projects can never collide. `path.join` normalizes the root, so a - * trailing slash cannot mint a second identity for the same file (this path - * doubles as the recording-session key, see {@link startRecordingSession}). + * two validated inputs, and the recording-session key (see + * {@link startRecordingSession}). + * + * Two different projects can never collide on one key. The converse is only + * true up to `path.join`, which folds a trailing slash, `//` and `.` segments + * but NOT symlinks or case: a caller that spells one root two ways (`/tmp/p` + * vs `/private/tmp/p` on macOS, or a case-variant on APFS) gets two sessions + * writing one file, and their appends can lose each other. Callers pass a + * cwd-derived root, so this needs two callers disagreeing about the spelling of + * the same directory; resolving symlinks here is not an option because in + * "client" mode the root does not exist on this host at all. */ export function getFlowPath(projectRoot: string, name: string): string { const flowsDir = getFlowsDir(projectRoot); @@ -195,8 +202,6 @@ export interface RecordingSession { filePath: string; /** In-memory flow content — authoritative in "client" mode. */ flow: FlowFile; - /** Serializes appends to this session — see {@link appendStepToFlow}. */ - tail: Promise; /** Wall-clock of the last touch, for the LRU eviction backstop. */ lastTouchedAtMs: number; } @@ -214,11 +219,59 @@ export interface RecordingSession { */ const recordings = new Map(); +/** + * Serializes every mutation of ONE flow file: an append, the reset+register a + * `flow-start-recording` performs, and the read+clear a `flow-finish-recording` + * performs. Each of those is a read/await/write straddling at least one + * microtask, and Express dispatches tool calls concurrently, so without this + * two of them interleave and one silently loses. + * + * Keyed by the flow path, NOT by the session object: a restart *replaces* the + * session, so a lock the session owned could not exclude the very operation + * that supersedes it — the restart would truncate the file while an append from + * the discarded take was mid-flight, and that step would land in the new take. + * + * Per file, not global: two recordings write two different files and must not + * queue behind each other. + */ +const flowFileLocks = new Map>(); + +async function withFlowLock(key: string, fn: () => Promise): Promise { + const previous = flowFileLocks.get(key) ?? Promise.resolve(); + // `previous` is always an already-swallowed promise, so a failed holder can + // never wedge or reject the chain. + const run = previous.then(() => fn()); + const held = run.catch(() => {}); + flowFileLocks.set(key, held); + // Drop the entry once this holder is the last one, so the map does not grow + // by one permanent entry per flow ever recorded. + void held.then(() => { + if (flowFileLocks.get(key) === held) flowFileLocks.delete(key); + }); + return run; +} + +/** + * Run `fn` with exclusive access to one flow file. Exported so the tools whose + * critical section spans more than an append — `flow-start-recording`'s + * truncate-then-register, `flow-finish-recording`'s read-then-clear — hold the + * same lock that {@link appendStepToFlow} takes. + */ +export function withFlowFileLock( + projectRoot: string, + name: string, + fn: () => Promise +): Promise { + return withFlowLock(getFlowPath(projectRoot, name), fn); +} + /** * Leak backstop only. Sessions are small and auto-spawned servers idle out * after 30 min, but a long-lived server could accumulate recordings an agent * started and never finished. Well past any realistic concurrent-agent count, - * so evicting is never something an agent should observe. + * so evicting should never be something an agent observes — and if it ever is, + * {@link assertSessionStillLive} makes the next append fail loudly rather than + * write into a recording the server has forgotten. */ const MAX_RECORDINGS = 32; @@ -254,11 +307,7 @@ export interface RecordingSessionInit { export function startRecordingSession(init: RecordingSessionInit): RecordingSession | null { const key = getFlowPath(init.projectRoot, init.name); const previous = recordings.get(key) ?? null; - recordings.set(key, { - ...init, - tail: Promise.resolve(), - lastTouchedAtMs: Date.now(), - }); + recordings.set(key, { ...init, lastTouchedAtMs: Date.now() }); evictIfOverCapacity(); return previous; } @@ -308,9 +357,9 @@ export function clearRecordingSession(projectRoot: string, name: string): void { recordings.delete(getFlowPath(projectRoot, name)); } -/** Drop every recording — test reset. */ -export function clearAllRecordings(): void { +export function __resetRecordingsForTesting(): void { recordings.clear(); + flowFileLocks.clear(); } // ── Types ──────────────────────────────────────────────────────────── @@ -2270,22 +2319,33 @@ export function clientFileDirective(filePath: string, content: string): ClientFi export type FlowSavedTo = string | ClientFileDirective; /** - * Serialize work against one recording. `appendStep` is read → await → write, - * a lost-update window that only mattered while a single recording could have - * a single caller; now that an agent can legitimately have two `flow-add-step` - * calls in flight (and two agents can share one server), appends on a session - * are chained so the second reads what the first wrote. - * - * Per session, not global: two recordings write two different files and must - * not queue behind each other. + * A tool resolves its session up front, then runs the step LIVE — which can take + * minutes — before appending. In that window the recording it holds may have + * been finished, restarted or evicted, leaving it with a session object that is + * no longer the one registered for its key. Writing anyway is the worst outcome: + * the step lands in a file that now belongs to a *different* take and the caller + * is told it succeeded. Re-check identity at write time — inside the flow-file + * lock, so the check sees the state the write will see — and fail loudly. */ -async function withSessionLock(session: RecordingSession, fn: () => Promise): Promise { - // `.then(fn, fn)` — a prior append that rejected must not wedge the chain, - // and `session.tail` swallows the result so an unobserved rejection on the - // tail can never surface as an unhandled rejection. - const run = session.tail.then(fn, fn); - session.tail = run.catch(() => {}); - return run; +function assertSessionStillLive(session: RecordingSession): void { + const current = recordings.get(getFlowPath(session.projectRoot, session.name)); + if (current === session) return; + // A key that is occupied by a DIFFERENT session was restarted; an empty key + // was either finished or evicted by the MAX_RECORDINGS backstop, which the + // server cannot tell apart after the fact — so name both rather than guess. + const why = current + ? "it was restarted while this step was running, so the step belongs to the discarded take" + : "it was finished (or dropped by the concurrent-recording cap) while this step was running"; + throw new FailureError( + `Recording of "${session.name}" in ${session.projectRoot} is no longer active — ${why}. ` + + `Nothing was recorded. Call flow-start-recording and re-record the step.`, + { + error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, + failure_stage: "flow_session_superseded", + failure_area: "tool_server", + error_kind: "validation", + } + ); } /** @@ -2299,7 +2359,8 @@ export async function appendStepToFlow( session: RecordingSession, step: FlowStep ): Promise<{ flowFile: string; savedTo: FlowSavedTo }> { - return withSessionLock(session, async () => { + return withFlowFileLock(session.projectRoot, session.name, async () => { + assertSessionStillLive(session); session.lastTouchedAtMs = Date.now(); if (session.persist === "host") { const flowFile = await appendStep(session.filePath, step); @@ -2308,12 +2369,18 @@ export async function appendStepToFlow( } session.flow.steps.push(step); try { + // Both of these can reject on a bad step — validateFlow on a cross-field + // violation, serializeFlow on an unrepresentable one (e.g. a tap with + // un-normalized coordinates). Roll back on either: in client mode this + // in-memory copy is the ONLY copy, so leaving the rejected step in it + // poisons the recording — every later append, and the finish itself, + // would re-hit the same error with no way to recover. validateFlow(session.flow); + const flowFile = serializeFlow(session.flow); + return { flowFile, savedTo: clientFileDirective(session.filePath, flowFile) }; } catch (err) { session.flow.steps.pop(); // keep the in-memory copy consistent: nothing recorded throw err; } - const flowFile = serializeFlow(session.flow); - return { flowFile, savedTo: clientFileDirective(session.filePath, flowFile) }; }); } diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 4c1b8e2b9..5602ebaa3 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -33,29 +33,32 @@ const zodSchema = z.object({ }); /** - * Does `urn` belong to `deviceId`? Every URN in {@link PREFIXES} is - * `:`, optionally with a trailing transport discriminator - * (`NativeDevtools::tcp`). Device ids can themselves contain a colon (a - * wireless-adb serial is `192.168.1.5:5555`), so the tail is compared whole - * rather than split on ":". + * Which entry of `deviceIds` owns `urn`, if any. Every URN in {@link PREFIXES} + * is `:`, optionally with a trailing transport + * discriminator (`NativeDevtools::tcp`). Device ids can themselves contain + * a colon (a wireless-adb serial is `192.168.1.5:5555`), so the tail is compared + * whole rather than split on ":". + * + * Returns the caller's spelling of the id so the tool can report which requested + * ids matched nothing. */ -function urnTargetsDevice(urn: string, deviceIds: string[]): boolean { +function matchingDeviceId(urn: string, deviceIds: string[]): string | undefined { const prefix = PREFIXES.find((p) => urn.startsWith(p)); - if (!prefix) return false; - const tail = urn.slice(prefix.length); + if (!prefix) return undefined; // Case-insensitive: iOS UDIDs are conventionally upper-case but agents pass // through whatever they were given, and a case mismatch must not silently - // widen a scoped stop into a no-op. - return deviceIds.some((id) => { + // widen a scoped stop into a no-op. No two distinct devices can differ only + // by case in any id space we support (UUID, emulator-N, chromium-cdp-N). + const tail = urn.slice(prefix.length).toLowerCase(); + return deviceIds.find((id) => { const lower = id.toLowerCase(); - const t = tail.toLowerCase(); - return t === lower || t.startsWith(`${lower}:`); + return tail === lower || tail.startsWith(`${lower}:`); }); } export function createStopAllSimulatorServersTool( registry: Registry -): ToolDefinition, { stopped: string[] }> { +): ToolDefinition, { stopped: string[]; unmatched?: string[] }> { return { id: "stop-all-simulator-servers", interaction: { @@ -75,22 +78,23 @@ export function createStopAllSimulatorServersTool( }, description: `Stop running simulator-server processes (iOS + Android), native devtools services, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps, silently). Omit \`devices\` only when a machine-wide cleanup is what you actually want. -Returns { stopped } — an array of URNs that were shut down. Fails silently if no matching servers are running.`, +Returns { stopped } — an array of URNs that were shut down — plus { unmatched } naming any id in \`devices\` that owned no services, so a mistyped id or a device name passed where an id was expected does not read as a clean machine. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { - const devices = params?.devices; + const devices = params.devices; // Present-but-empty scopes to nothing rather than falling back to the // machine-wide sweep: a caller that computed a device list and got none // must not accidentally tear down every other agent's services. const scoped = devices !== undefined; const snapshot = registry.getSnapshot(); const stopped: string[] = []; + const matchedIds = new Set(); for (const [urn, entry] of snapshot.services) { - const matches = scoped - ? urnTargetsDevice(urn, devices) - : PREFIXES.some((p) => urn.startsWith(p)); + const matchedId = scoped ? matchingDeviceId(urn, devices) : undefined; + const matches = scoped ? matchedId !== undefined : PREFIXES.some((p) => urn.startsWith(p)); if (matches && entry.state !== ServiceState.IDLE) { + if (matchedId !== undefined) matchedIds.add(matchedId); // Dispose any non-IDLE node (this also clears ERROR/TERMINATING // nodes), but only report the ones that were actually live — an // ERROR node (e.g. a tvOS SimulatorServer that refused to start) @@ -100,7 +104,13 @@ Returns { stopped } — an array of URNs that were shut down. Fails silently if if (wasLive) stopped.push(urn); } } - return { stopped }; + if (!scoped) return { stopped }; + // A scoped stop that matched nothing is indistinguishable from a clean + // machine unless we say so — and the ids that miss are exactly the ones + // whose simulator-server, devtools and (on tvOS) two --timeout 3600 + // daemons are being left running. + const unmatched = devices.filter((id) => !matchedIds.has(id)); + return unmatched.length > 0 ? { stopped, unmatched } : { stopped }; }, }; } diff --git a/packages/tool-server/test/failure-classification.test.ts b/packages/tool-server/test/failure-classification.test.ts index bb6a4c234..1929d0e95 100644 --- a/packages/tool-server/test/failure-classification.test.ts +++ b/packages/tool-server/test/failure-classification.test.ts @@ -3,6 +3,9 @@ import { describe, it, expect, afterEach } from "vitest"; import { FAILURE_CODES, getFailureSignal, type FailureCode } from "@argent/registry"; import { assertValidProjectRoot, assertSafeFlowName } from "../src/tools/flows/flow-utils"; +import { createFlowAddStepTool } from "../src/tools/flows/flow-add-step"; +import { flowInsertEchoTool } from "../src/tools/flows/flow-insert-echo"; +import { createRunFlowTool } from "../src/tools/flows/flow-run"; import type { DeviceInfo, Registry } from "@argent/registry"; import { makeChromiumImpl } from "../src/tools/keyboard/platforms/chromium"; import { chromiumCdpBlueprint } from "../src/blueprints/chromium-cdp"; @@ -99,6 +102,76 @@ describe("flow-utils classifications", () => { }); }); +describe("flow tool project_root classifications", () => { + // Every flow tool takes `project_root` from the caller, so the invalid-root + // throw is reachable from a tool's execute — not just from the helper above. + // Root validation happens inside getFlowPath, which runs BEFORE the recording + // is looked up (`requireRecordingSession` keys the session map by that path), + // so a bad root reports FLOW_PROJECT_ROOT_INVALID rather than the + // FLOW_NO_ACTIVE_RECORDING it would earn if the lookup came first — even + // though no recording was ever started here. + const RELATIVE_ROOT = "relative/project"; + const DOTDOT_ROOT = "/tmp/project/../../etc"; + + // Both tools throw on the root before any tool dispatch or file read, so a + // bare stub registry is never actually used. + const registry = {} as unknown as Registry; + + it("classifies flow-add-step with a relative project_root as FLOW_PROJECT_ROOT_INVALID", async () => { + const addStep = createFlowAddStepTool(registry); + const err = await captureError( + addStep.execute( + {}, + { name: "some-flow", project_root: RELATIVE_ROOT, command: "gesture-tap" } + ) + ); + expectCode(err, FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID); + // The root is judged first: no "No active recording" for an unstarted flow. + expect((err as Error).message).not.toMatch(/No active recording/); + }); + + it("classifies flow-add-step with a '..'-bearing project_root as FLOW_PROJECT_ROOT_INVALID", async () => { + const addStep = createFlowAddStepTool(registry); + expectCode( + await captureError( + addStep.execute( + {}, + { name: "some-flow", project_root: DOTDOT_ROOT, command: "gesture-tap" } + ) + ), + FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID + ); + }); + + it("classifies flow-add-echo with a relative project_root as FLOW_PROJECT_ROOT_INVALID", async () => { + expectCode( + await captureError( + flowInsertEchoTool.execute( + {}, + { name: "some-flow", project_root: RELATIVE_ROOT, message: "label" } + ) + ), + FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID + ); + }); + + it("classifies flow-execute with a relative project_root as FLOW_PROJECT_ROOT_INVALID", async () => { + const runFlow = createRunFlowTool(registry); + expectCode( + await captureError(runFlow.execute({}, { name: "some-flow", project_root: RELATIVE_ROOT })), + FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID + ); + }); + + it("classifies flow-execute with a '..'-bearing project_root as FLOW_PROJECT_ROOT_INVALID", async () => { + const runFlow = createRunFlowTool(registry); + expectCode( + await captureError(runFlow.execute({}, { name: "some-flow", project_root: DOTDOT_ROOT })), + FAILURE_CODES.FLOW_PROJECT_ROOT_INVALID + ); + }); +}); + describe("keyboard classifications", () => { // The chromium typing path lives in makeChromiumImpl's handler, which resolves // the CDP service then validates the key/char before touching it. A stub diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 80c288eb2..b9251e418 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { readFileSync } from "node:fs"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; @@ -12,11 +13,12 @@ import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { createRunFlowTool } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; import { - clearAllRecordings, + __resetRecordingsForTesting, getRecordingSession, listActiveRecordings, parseFlow, serializeFlow, + withFlowFileLock, type FlowFile, type FlowStep, } from "../../src/tools/flows/flow-utils"; @@ -30,10 +32,34 @@ import { * one recording's steps never land in another's file, addressing a key that * isn't live fails loudly (naming the ones that are), replaying a flow * elsewhere rebinds nothing, and appends to one session can't lose each other. + * + * The second half pins the *mutual exclusion* that makes the above hold when + * the tools genuinely overlap. Every recording tool's critical section straddles + * an await — a restart's truncate-then-register, a finish's read-then-clear, an + * append's read-then-write — so each is covered by the per-flow-file lock, and a + * step that resolved its session before some other tool superseded it must fail + * rather than write into a file that now belongs to a different take. */ const IOS_DEVICE = "00000000-0000-0000-0000-0000000000ab"; +/** + * The concurrent-recording cap is internal to flow-utils, and a copy of it here + * would silently stop testing the real backstop the day it changes. Read it out + * of the source instead. + */ +function readMaxRecordings(): number { + const source = readFileSync( + path.resolve(__dirname, "../../src/tools/flows/flow-utils.ts"), + "utf8" + ); + const match = /^const MAX_RECORDINGS = (\d+);$/m.exec(source); + if (!match) throw new Error("could not read MAX_RECORDINGS out of flow-utils.ts"); + return Number(match[1]); +} + +const MAX_RECORDINGS = readMaxRecordings(); + // ── Harness ────────────────────────────────────────────────────────── let roots: string[] = []; @@ -45,6 +71,39 @@ async function makeRoot(label: string): Promise { return dir; } +/** A promise plus the function that resolves it. */ +function openGate(): { promise: Promise; open: () => void } { + let open!: () => void; + const promise = new Promise((resolve) => { + open = () => resolve(); + }); + return { promise, open }; +} + +/** Installed by {@link gateNextSubTool}; consumed by the mock registry. */ +let subToolGate: (() => Promise) | null = null; + +/** + * Suspend the NEXT live sub-tool execution and report when it is reached. + * + * flow-add-step resolves its recording session, runs the step LIVE (which can + * take minutes on a device), and only then appends. Parking a step inside that + * window is what puts an append genuinely in flight across a concurrent + * restart / finish / eviction — deterministically, with no timing guesses. + */ +function gateNextSubTool(): { reached: Promise; release: () => void } { + const arrived = openGate(); + const held = openGate(); + subToolGate = async () => { + // One-shot: later calls (including the ones asserting the recording still + // works afterwards) run straight through. + subToolGate = null; + arrived.open(); + await held.promise; + }; + return { reached: arrived.promise, release: held.open }; +} + function createMockRegistry(): Registry { return { invokeTool: vi.fn(async (id: string) => { @@ -53,6 +112,7 @@ function createMockRegistry(): Registry { // so this is what lets several calls issued without an await in between // reach the append phase concurrently (see the lost-update test). await new Promise((resolve) => setTimeout(resolve, 0)); + if (subToolGate) await subToolGate(); return { ok: true }; }), getTool: vi.fn(() => ({ inputSchema: { properties: { udid: {} } } })), @@ -69,17 +129,13 @@ function start(root: string, name: string, executionPrerequisite?: string) { return flowStartRecordingTool.execute({}, { name, project_root: root, executionPrerequisite }); } +function addRawStep(root: string, name: string, command: string, args: Record) { + return addStepTool.execute({}, { name, project_root: root, command, args: JSON.stringify(args) }); +} + /** Record a `tool` step tagged with `marker`, so its file of origin is provable. */ function addStep(root: string, name: string, marker: string) { - return addStepTool.execute( - {}, - { - name, - project_root: root, - command: "keyboard", - args: JSON.stringify({ text: marker }), - } - ); + return addRawStep(root, name, "keyboard", { text: marker }); } function addEcho(root: string, name: string, message: string) { @@ -104,17 +160,68 @@ function markers(steps: FlowStep[]): string[] { }); } +async function readSteps(root: string, name: string): Promise { + return parseFlow(await fs.readFile(flowPath(root, name), "utf8")).steps; +} + async function readMarkers(root: string, name: string): Promise { - return markers(parseFlow(await fs.readFile(flowPath(root, name), "utf8")).steps); + return markers(await readSteps(root, name)); +} + +async function captureFailure(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err; + } + throw new Error("expected the call to fail"); +} + +/** Let real timers and in-flight fs I/O drain, so "still blocked" means blocked. */ +function settle(ms = 25): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Resolve to `label` if `promise` settles in time, else to "timed-out". */ +async function within(promise: Promise, label: string, ms = 2000): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve("timed-out"), ms); + }); + try { + return await Promise.race([promise.then(() => label), timeout]); + } finally { + clearTimeout(timer); + } +} + +/** + * The LRU backstop compares `Date.now()` stamps, and dozens of recordings can be + * started and touched inside one millisecond — so drive its clock explicitly + * rather than depending on wall-clock resolution. + */ +function useMonotonicClock(): { restore: () => void } { + let ticks = Date.now(); + const spy = vi.spyOn(Date, "now").mockImplementation(() => ++ticks); + return { restore: () => spy.mockRestore() }; +} + +/** Fill the recording table exactly to its cap; returns the names, oldest first. */ +async function fillRecordings(root: string): Promise { + const names = Array.from({ length: MAX_RECORDINGS }, (_, i) => `rec-${i}`); + for (const name of names) await start(root, name); + return names; } beforeEach(() => { - clearAllRecordings(); + __resetRecordingsForTesting(); + subToolGate = null; roots = []; }); afterEach(async () => { - clearAllRecordings(); + __resetRecordingsForTesting(); + subToolGate = null; await Promise.all(roots.map((dir) => fs.rm(dir, { recursive: true, force: true }))); roots = []; }); @@ -207,15 +314,6 @@ describe("the same flow name under two project roots", () => { // ── Addressing a key that isn't live ───────────────────────────────── describe("addressing an unknown recording key", () => { - async function captureFailure(promise: Promise): Promise { - try { - await promise; - } catch (err) { - return err; - } - throw new Error("expected the call to fail"); - } - it("fails with FLOW_NO_ACTIVE_RECORDING and lists the live recordings", async () => { const rootA = await makeRoot("unknown-a"); const rootB = await makeRoot("unknown-b"); @@ -266,7 +364,7 @@ describe("concurrent flow-add-step calls on one recording", () => { const tags = ["s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7"]; // Fire without awaiting in between: every call is past its live execution // and inside the append phase before the first one writes. appendStep is - // read → await → write, so without the per-session mutex these would all + // read → await → write, so without the per-file lock these would all // read the same file and the last write would drop the others. const inflight = tags.map((tag) => addStep(root, "burst", tag)); await Promise.all(inflight); @@ -281,13 +379,11 @@ describe("concurrent flow-add-step calls on one recording", () => { expect(finished.steps).toBe(tags.length); }); - it("does not serialize a second recording behind the first one's appends", async () => { + it("keeps two concurrent bursts on their own files", async () => { const root = await makeRoot("append-race-two"); await start(root, "alpha"); await start(root, "beta"); - // Both bursts in flight together — the lock is per session, so neither - // file may pick up the other's steps. await Promise.all([ ...["a0", "a1", "a2", "a3"].map((tag) => addStep(root, "alpha", tag)), ...["b0", "b1", "b2", "b3"].map((tag) => addStep(root, "beta", tag)), @@ -308,6 +404,52 @@ describe("concurrent flow-add-step calls on one recording", () => { }); }); +// ── The lock is per flow file, not one global mutex ────────────────── + +describe("the flow-file lock", () => { + it("lets one recording append while another recording's file is locked", async () => { + const root = await makeRoot("per-file-lock"); + await start(root, "alpha"); + await start(root, "beta"); + + // Hold alpha's file lock — this is exactly the state an alpha append is in + // while it is mid read-modify-write. Whether beta can make progress *during* + // that window is the property under test: a single global lock passes any + // assertion about final file contents, and fails this one. + const order: string[] = []; + const alphaLock = openGate(); + const alphaHeld = withFlowFileLock(root, "alpha", () => alphaLock.promise); + + // A second append to alpha must queue behind the holder… + const alphaAppend = addStep(root, "alpha", "a1").then((r) => { + order.push("alpha-appended"); + return r; + }); + const betaAppend = addStep(root, "beta", "b1").then((r) => { + order.push("beta-appended"); + return r; + }); + + // …while beta's append, on a different file, runs to completion inside it. + expect(await within(betaAppend, "beta-appended")).toBe("beta-appended"); + await settle(); + expect(order).toEqual(["beta-appended"]); + expect(await readMarkers(root, "beta")).toEqual(["tool:b1"]); + expect(await readMarkers(root, "alpha")).toEqual([]); + + order.push("alpha-lock-released"); + alphaLock.open(); + await alphaHeld; + expect(await within(alphaAppend, "alpha-appended")).toBe("alpha-appended"); + + // beta finished strictly inside alpha's critical section — real overlap, + // not a serialization that happened to be fast. + expect(order).toEqual(["beta-appended", "alpha-lock-released", "alpha-appended"]); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); + expect(await readMarkers(root, "beta")).toEqual(["tool:b1"]); + }); +}); + // ── Replaying a flow while recordings are live ─────────────────────── describe("running a flow in a third project while two recordings are live", () => { @@ -414,3 +556,292 @@ describe("restarting a recording on one key", () => { expect(getRecordingSession(rootA, "alpha")?.flow.steps).toHaveLength(1); }); }); + +// ── A restart landing on top of an in-flight append ────────────────── + +describe("a restart that lands while a step is still running", () => { + it("rejects the in-flight step instead of writing it into the new take", async () => { + const root = await makeRoot("restart-inflight"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + // The step resolves its session, then parks in its LIVE execution. + const gate = gateNextSubTool(); + const appending = addStep(root, "alpha", "a2"); + await gate.reached; + + // The take that step belongs to is discarded while it is still running. + const restarted = await start(root, "alpha"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(1); + + gate.release(); + const err = await captureFailure(appending); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); + expect((err as Error).message).toContain("restarted while this step was running"); + expect((err as Error).message).toContain("Nothing was recorded"); + + // The new take is empty — no step from the discarded one leaked into it. + expect(await readMarkers(root, "alpha")).toEqual([]); + expect(getRecordingSession(root, "alpha")?.flow.steps).toHaveLength(0); + + // …and the restarted recording still works. + await addStep(root, "alpha", "a3"); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a3"]); + const finished = await finish(root, "alpha"); + expect(finished.steps).toBe(1); + }); + + it("truncates and re-registers only once the flow's lock is free", async () => { + const root = await makeRoot("restart-lock"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + const firstSession = getRecordingSession(root, "alpha"); + + // Stand in for an append that is mid read-modify-write on alpha's file. + const order: string[] = []; + const lock = openGate(); + const held = withFlowFileLock(root, "alpha", () => lock.promise); + + const restarting = start(root, "alpha").then((r) => { + order.push("restart-returned"); + return r; + }); + + await settle(); + // The restart is a truncate AND a session swap; neither half may happen + // while another writer holds the file. + expect(order).toEqual([]); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); + expect(getRecordingSession(root, "alpha")).toBe(firstSession); + + order.push("lock-released"); + lock.open(); + await held; + const restarted = await restarting; + + expect(order).toEqual(["lock-released", "restart-returned"]); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(1); + expect(await readMarkers(root, "alpha")).toEqual([]); + expect(getRecordingSession(root, "alpha")).not.toBe(firstSession); + }); +}); + +// ── A finish landing on top of an in-flight append ─────────────────── + +describe("a finish that lands while a step is still running", () => { + it("never reports steps, a summary or YAML the file disagrees with", async () => { + // Vary how far the append has progressed when the finish arrives, so both + // outcomes are exercised: the append wins the lock (and must be included in + // what finish reports) or the finish wins it (and the append must fail). + for (const microtasks of [0, 1, 2, 3, 4, 6, 8]) { + const root = await makeRoot(`finish-inflight-${microtasks}`); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + const gate = gateNextSubTool(); + const appending = addStep(root, "alpha", "a2"); + await gate.reached; + gate.release(); + for (let i = 0; i < microtasks; i++) await Promise.resolve(); + + const [appended, finished] = await Promise.allSettled([appending, finish(root, "alpha")]); + + if (finished.status === "rejected") throw finished.reason; + const report = finished.value; + const onDisk = await readMarkers(root, "alpha"); + + // The whole report is one snapshot of one file state. + expect(markers(parseFlow(report.flowFile).steps)).toEqual(onDisk); + expect(report.steps).toBe(onDisk.length); + expect(report.summary).toHaveLength(onDisk.length); + expect(report.path).toBe(flowPath(root, "alpha")); + expect(report.savedTo).toBe(flowPath(root, "alpha")); + + if (appended.status === "rejected") { + expect(getFailureSignal(appended.reason)?.error_code).toBe( + FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING + ); + expect(onDisk).toEqual(["tool:a1"]); + } else { + expect(onDisk).toEqual(["tool:a1", "tool:a2"]); + } + + // Either way the recording is gone, and nothing can be appended to it. + expect(getRecordingSession(root, "alpha")).toBeUndefined(); + } + }); + + it("reads the file back and clears the session only once the lock is free", async () => { + const root = await makeRoot("finish-lock"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + const order: string[] = []; + const lock = openGate(); + const held = withFlowFileLock(root, "alpha", () => lock.promise); + + const finishing = finish(root, "alpha").then((r) => { + order.push("finish-returned"); + return r; + }); + + await settle(); + expect(order).toEqual([]); + // The session is still live: resolve-read-clear is one critical section. + expect(getRecordingSession(root, "alpha")).toBeDefined(); + + order.push("lock-released"); + lock.open(); + await held; + const finished = await finishing; + + expect(order).toEqual(["lock-released", "finish-returned"]); + expect(finished.steps).toBe(1); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1"]); + expect(getRecordingSession(root, "alpha")).toBeUndefined(); + }); + + it("rejects a step whose recording was already finished", async () => { + const root = await makeRoot("append-after-finish"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + const gate = gateNextSubTool(); + const appending = addStep(root, "alpha", "a2"); + await gate.reached; + + // The finish completes end-to-end before the step comes back. + const finished = await finish(root, "alpha"); + expect(finished.steps).toBe(1); + + gate.release(); + const err = await captureFailure(appending); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); + expect((err as Error).message).toContain("no longer active"); + + // The finished file is exactly what the finish reported. + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1"]); + }); +}); + +// ── The concurrent-recording cap ───────────────────────────────────── + +describe("the concurrent-recording cap", () => { + it("evicts the least recently touched recording and keeps the rest", async () => { + const root = await makeRoot("evict"); + const clock = useMonotonicClock(); + try { + const names = await fillRecordings(root); + expect(listActiveRecordings()).toHaveLength(MAX_RECORDINGS); + + // Touch everything except the first, so `rec-0` is unambiguously the + // least recently touched recording. + for (const name of names.slice(1)) await addEcho(root, name, "touch"); + + await start(root, "overflow"); + + const live = listActiveRecordings() + .map((r) => r.name) + .sort(); + expect(live).toHaveLength(MAX_RECORDINGS); + expect(live).toEqual([...names.slice(1), "overflow"].sort()); + expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + // The survivors are still usable — eviction dropped one, not the table. + expect(getRecordingSession(root, names[1])).toBeDefined(); + await addEcho(root, names[1], "still-live"); + expect(await readMarkers(root, names[1])).toEqual(["echo:touch", "echo:still-live"]); + } finally { + clock.restore(); + } + }); + + it("rejects an append whose recording was evicted while the step ran", async () => { + const root = await makeRoot("evict-inflight"); + const clock = useMonotonicClock(); + try { + const names = await fillRecordings(root); + + // The step resolves rec-0's session (touching it) and parks. + const gate = gateNextSubTool(); + const appending = addStep(root, "rec-0", "victim"); + await gate.reached; + + // Every other recording is touched, then one more overflows the cap — + // rec-0 is now the LRU and gets dropped out from under the running step. + for (const name of names.slice(1)) await addEcho(root, name, "touch"); + await start(root, "overflow"); + expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + + gate.release(); + const err = await captureFailure(appending); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); + expect((err as Error).message).toContain("concurrent-recording cap"); + expect(await readMarkers(root, "rec-0")).toEqual([]); + + // A fresh call on the evicted key fails the ordinary not-live way. + const late = await captureFailure(addEcho(root, "rec-0", "late")); + expect(getFailureSignal(late)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(late)?.failure_stage).toBe("flow_require_recording"); + } finally { + clock.restore(); + } + }); +}); + +// ── Recording a flow-execute step ──────────────────────────────────── + +describe("recording a flow-execute step while several projects are in play", () => { + const fragment: FlowFile = { + executionPrerequisite: "", + steps: [{ kind: "echo", message: "helper" }], + }; + + it("keeps the raw step when the target is not a sibling of the RECORDING", async () => { + const recordingRoot = await makeRoot("run-target-recording"); + const executedRoot = await makeRoot("run-target-executed"); + + // The fragment exists in the project the nested flow-execute ran in, but + // NOT next to the flow being recorded — so `run: helper` would be a + // dangling reference at replay, which resolves siblings of the recording. + await writeSavedFlow(executedRoot, "helper", fragment); + + await start(recordingRoot, "wrapper"); + const res = await addRawStep(recordingRoot, "wrapper", "flow-execute", { + name: "helper", + project_root: executedRoot, + udid: IOS_DEVICE, + }); + + expect(res.message).toContain('could not resolve "helper" as a sibling fragment'); + expect(res.message).toContain("kept the raw flow-execute step"); + expect(await readSteps(recordingRoot, "wrapper")).toEqual([ + { kind: "tool", name: "flow-execute", args: { name: "helper", project_root: executedRoot } }, + ]); + }); + + it("records run: when the target sits next to the recording", async () => { + const recordingRoot = await makeRoot("run-target-sibling"); + const executedRoot = await makeRoot("run-target-elsewhere"); + + // Mirror image: the fragment is a sibling of the flow being recorded and is + // absent from the executed project. + await writeSavedFlow(recordingRoot, "helper", fragment); + await fs.mkdir(path.join(executedRoot, ".argent", "flows"), { recursive: true }); + + await start(recordingRoot, "wrapper"); + const res = await addRawStep(recordingRoot, "wrapper", "flow-execute", { + name: "helper", + project_root: executedRoot, + udid: IOS_DEVICE, + }); + + expect(res.message).not.toContain("kept the raw flow-execute step"); + expect(await readSteps(recordingRoot, "wrapper")).toEqual([{ kind: "run", flow: "helper" }]); + }); +}); diff --git a/packages/tool-server/test/flows/flow-record-tap.test.ts b/packages/tool-server/test/flows/flow-record-tap.test.ts index 93b085211..8729efe03 100644 --- a/packages/tool-server/test/flows/flow-record-tap.test.ts +++ b/packages/tool-server/test/flows/flow-record-tap.test.ts @@ -15,7 +15,7 @@ vi.mock("../../src/tools/flows/flow-tree", () => ({ import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording"; -import { clearAllRecordings, parseFlow } from "../../src/tools/flows/flow-utils"; +import { __resetRecordingsForTesting, parseFlow } from "../../src/tools/flows/flow-utils"; const DEVICE = "00000000-0000-0000-0000-0000000000AB"; // iOS UDID shape const FLOW = "rec"; @@ -65,7 +65,7 @@ async function recordedSteps() { beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-record-tap-")); - clearAllRecordings(); + __resetRecordingsForTesting(); await flowStartRecordingTool.execute( {}, { name: FLOW, project_root: tmpDir, executionPrerequisite: PREREQ } @@ -73,7 +73,7 @@ beforeEach(async () => { }); afterEach(async () => { - clearAllRecordings(); + __resetRecordingsForTesting(); await fs.rm(tmpDir, { recursive: true, force: true }); }); diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index 8c8c71f4d..c5bfc4a8c 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -11,7 +11,7 @@ import { flowFinishRecordingTool } from "../../src/tools/flows/flow-finish-recor import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { createRunFlowTool, resolveFlowSource } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; -import { clearAllRecordings, parseFlow } from "../../src/tools/flows/flow-utils"; +import { __resetRecordingsForTesting, parseFlow } from "../../src/tools/flows/flow-utils"; /** * Remote-mode flow behavior: the agent's project_root does NOT exist on this @@ -60,11 +60,11 @@ function createMockRegistry(tools: Record = {}) { } beforeEach(() => { - clearAllRecordings(); + __resetRecordingsForTesting(); }); afterEach(async () => { - clearAllRecordings(); + __resetRecordingsForTesting(); await fs.rm(CLIENT_ROOT, { recursive: true, force: true }); await fs.rm(OTHER_CLIENT_ROOT, { recursive: true, force: true }); }); @@ -197,6 +197,88 @@ describe("flow recording with a remote client (probe miss)", () => { ).rejects.toThrow("No active recording"); }); + it("a rejected append leaves the session usable instead of poisoning it", async () => { + // In client mode the in-memory flow is the ONLY copy, so a step the append + // refuses must not stay in it — every later append, and the finish itself, + // would re-hit the same error with no way to recover. Both gates are + // exercised: serializeFlow (an unrepresentable step) and validateFlow (a + // cross-field violation). + const registry = createMockRegistry({ + "gesture-tap": { result: { tapped: true } }, + "restart-app": { result: { restarted: true } }, + }); + const addStep = createFlowAddStepTool(registry); + const device = "00000000-0000-0000-0000-0000000000ab"; + + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" }, + remoteCtx() + ); + await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "before" } + ); + + // serializeFlow rejects: a tap carrying pixel coordinates, not the + // normalized 0–1 fractions a YAML gesture target can represent. (Selector + // capture can't reach a device here, so the coordinates are kept as-is.) + await expect( + addStep.execute( + {}, + { + name: "remote-flow", + project_root: CLIENT_ROOT, + command: "gesture-tap", + args: JSON.stringify({ udid: device, x: 250, y: 400 }), + } + ) + ).rejects.toThrow("not pixels"); + + // validateFlow rejects: a `restart-app` is recorded as a `launch`, and this + // recording declared an executionPrerequisite — a flow that begins by + // launching controls its own start state and must not declare one. + await expect( + addStep.execute( + {}, + { + name: "remote-flow", + project_root: CLIENT_ROOT, + command: "restart-app", + args: JSON.stringify({ udid: device, bundleId: "com.example.app" }), + } + ) + ).rejects.toThrow("must not declare executionPrerequisite"); + + // The session survived both rejections: the next append succeeds, and its + // directive carries only the accepted steps — neither rejected step is in + // the flow, and neither error is replayed. + const after = await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "after" } + ); + const directive = after.savedTo as { path: string; content: string }; + expect(directive.path).toBe(CLIENT_FLOW_PATH); + expect(parseFlow(directive.content).steps).toEqual([ + { kind: "echo", message: "before" }, + { kind: "echo", message: "after" }, + ]); + expect(parseFlow(directive.content).executionPrerequisite).toBe("Home"); + + // And the recording still finishes — the whole point of rolling back. + const finished = await flowFinishRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT } + ); + expect(finished.steps).toBe(2); + expect(finished.summary).toEqual(["1. echo: before", "2. echo: after"]); + expect(finished.savedTo).toMatchObject({ + [CLIENT_FILE_MARKER]: true, + path: CLIENT_FLOW_PATH, + }); + await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow(); + }); + it("keeps same-named recordings under different client roots isolated", async () => { const registry = createMockRegistry({ tap: { result: { tapped: true } } }); const addStep = createFlowAddStepTool(registry); diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 56a3d0b0a..9c078da31 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -17,7 +17,7 @@ import { } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; import { - clearAllRecordings, + __resetRecordingsForTesting, flowsDirFor, getRecordingSession, parseFlow, @@ -69,11 +69,11 @@ const PREREQ = "App on home screen"; beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-test-")); otherDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-test-other-")); - clearAllRecordings(); + __resetRecordingsForTesting(); }); afterEach(async () => { - clearAllRecordings(); + __resetRecordingsForTesting(); await fs.rm(tmpDir, { recursive: true, force: true }); await fs.rm(otherDir, { recursive: true, force: true }); }); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 1891290f7..761065d05 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -11,7 +11,7 @@ import { requireRecordingSession, clearRecordingSession, listActiveRecordings, - clearAllRecordings, + __resetRecordingsForTesting, getFlowPath, appIdForPlatform, chromiumLaunchSpec, @@ -1014,7 +1014,7 @@ describe("native launch shorthand", () => { // must never observe each other's state. describe("recording sessions", () => { beforeEach(() => { - clearAllRecordings(); + __resetRecordingsForTesting(); }); const emptyFlow = (): FlowFile => ({ executionPrerequisite: "", steps: [] }); @@ -1139,7 +1139,7 @@ describe("recording sessions", () => { expect(listActiveRecordings()).toEqual([ { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, ]); - clearAllRecordings(); + __resetRecordingsForTesting(); expect(listActiveRecordings()).toEqual([]); }); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 5b7d1c045..4b622e2ab 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -215,13 +215,13 @@ describe("stop-all-simulator-servers", () => { }); }); -describe("stop-all-simulator-servers device scoping", () => { - // The tool-server is a host-wide singleton, so an unscoped teardown reaps - // whatever device another agent is mid-session on. `devices` narrows the - // sweep to the ids the calling session actually used. - const MINE = "AAAA-1111"; - const THEIRS = "BBBB-2222"; +// The tool-server is a host-wide singleton, so an unscoped teardown reaps +// whatever device another agent is mid-session on. `devices` narrows the sweep +// to the ids the calling session actually used. +const MINE = "AAAA-1111"; +const THEIRS = "BBBB-2222"; +describe("stop-all-simulator-servers device scoping", () => { function twoAgentServices() { return new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], @@ -280,6 +280,34 @@ describe("stop-all-simulator-servers device scoping", () => { ], }); expect(registry.disposeService).toHaveBeenCalledTimes(5); + // Nothing was requested, so there is nothing that could have missed. + expect(result).not.toHaveProperty("unmatched"); + }); + + it("scopes the non-simulator namespaces too (ChromiumCdp / TvControl / AndroidTvControl)", async () => { + // Every namespace in PREFIXES must honour `devices`, not just + // SimulatorServer/NativeDevtools: a TvControl daemon left running holds two + // spawned --timeout 3600 processes, and reaping another agent's is exactly + // the cross-session damage scoping exists to prevent. + const chromium = "chromium-cdp-9222"; + const appleTv = "APPLE-TV-UDID"; + const androidTv = "emulator-5556"; + const services = new Map([ + [`ChromiumCdp:${chromium}`, { state: ServiceState.RUNNING, dependents: [] }], + [`TvControl:${appleTv}`, { state: ServiceState.RUNNING, dependents: [] }], + [`AndroidTvControl:${androidTv}`, { state: ServiceState.RUNNING, dependents: [] }], + [`TvControl:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [chromium, appleTv, androidTv] }); + + expect(result).toEqual({ + stopped: [`ChromiumCdp:${chromium}`, `TvControl:${appleTv}`, `AndroidTvControl:${androidTv}`], + }); + expect(registry.disposeService).toHaveBeenCalledTimes(3); + expect(registry.disposeService).not.toHaveBeenCalledWith(`TvControl:${THEIRS}`); }); it("matches a transport-suffixed URN (NativeDevtools::tcp)", async () => { @@ -340,6 +368,9 @@ describe("stop-all-simulator-servers device scoping", () => { const result = await tool.execute!({}, { devices: [] }); expect(result).toEqual({ stopped: [] }); + // No id was requested, so nothing missed: an empty `unmatched` would read + // as a warning where there is nothing to warn about. + expect(result).not.toHaveProperty("unmatched"); expect(registry.disposeService).not.toHaveBeenCalled(); }); @@ -372,6 +403,116 @@ describe("stop-all-simulator-servers device scoping", () => { }); }); +describe("stop-all-simulator-servers unmatched ids", () => { + // A scoped stop whose ids owned nothing used to return a bare `{ stopped: [] }` + // — byte-identical to the answer on a genuinely clean machine. So a mistyped + // id, a device *name* passed where an id was expected, or an empty string all + // read as success while the services they were meant to reap (on tvOS, two + // spawned --timeout 3600 daemons) stayed running. `unmatched` names them. + + it("names an unknown id in unmatched while still stopping the live device", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE, "GHOST-9999"] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE}`], + unmatched: ["GHOST-9999"], + }); + expect(registry.disposeService).toHaveBeenCalledTimes(2); + }); + + it("reports a typo, a device name, and an empty-string id — the shapes that used to look clean", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const typo = `${MINE}0`; + const deviceName = "iPhone 15 Pro"; + const result = await tool.execute!({}, { devices: [MINE, typo, deviceName, ""] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`], + unmatched: [typo, deviceName, ""], + }); + }); + + it("omits unmatched entirely when every requested id matched something", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ["AndroidDevtools:emulator-5554", { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE, "emulator-5554"] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, "AndroidDevtools:emulator-5554"], + }); + // Absent, not an empty array — a clean scoped stop must carry no warning. + expect(result).not.toHaveProperty("unmatched"); + }); + + it("counts an id whose only service is IDLE as unmatched — nothing was stopped for it", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.IDLE, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [], unmatched: [MINE] }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + + it("does not report an ERROR-only device as unmatched — its dead node was cleaned up", async () => { + // The boundary against the IDLE case above: an ERROR node is never reported + // as `stopped` (it never ran), but it IS disposed, so the id did own + // something and calling it unmatched would be a false alarm. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.ERROR, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [] }); + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${MINE}`); + }); + + it("counts a case-differing id as matched and echoes the caller's own spelling for the miss", async () => { + // The registry holds the upper-case UDID; the caller passes lower-case. + // The hit must not be reported as a miss (matching is case-insensitive), + // and the miss must come back spelled exactly as the caller typed it so the + // agent can find it in its own device list. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE.toLowerCase(), "Mine-Typo"] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`], + unmatched: ["Mine-Typo"], + }); + }); +}); + describe("stop-metro", () => { it("defaults to port 8081", () => { expect(stopMetroTool.zodSchema).toBeDefined(); From 903b45b9708000b03e93383769a162fdcde5fe0a Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 19:33:59 +0200 Subject: [PATCH 04/98] fix(flow): keep a failed finish recoverable, and scope what errors disclose Three findings from the second review pass: - flow-finish-recording cleared the session before parseFlow, so a botched hand-edit of the .yaml (a workflow the tool descriptions invite) destroyed the recording on the way out: the error told the agent to call flow-start-recording, which truncates the very take it would recover. Parse first, clear after. - stop-all-simulator-servers counted a device as unmatched unless it owned a non-IDLE service, but disposeService returns a node to IDLE and keeps it, so the routine stop-one-then-stop-the-rest sequence reported the device the session had just stopped as a mistyped id. Ownership now counts regardless of state, and unmatched is de-duplicated and case-folded to match the lookup. - the not-found error enumerated every live recording's flow name and absolute project root. A tool-server bound beyond loopback serves unrelated callers, so other projects are now counted rather than named; a typo in your own project, the case worth recovering from, still is. Also corrects agent-facing prose that a first round of doc fixes overshot, most importantly the chromium self-boot advice: passing platform=chromium does not fall through to device auto-detection, it selects the self-boot branch and treats the launch string as an Electron app path -- so on a recorder-written flow, whose launch holds a bundle id, it failed the whole run. --- packages/argent-cli/test/flag-parser.test.ts | 46 ++++++- .../test/run-flow-add-step-payload.test.ts | 4 +- packages/argent-cli/test/run-help.test.ts | 40 +++++- .../skills/skills/argent-create-flow/SKILL.md | 8 +- .../src/tools/flows/flow-add-step.ts | 4 +- .../src/tools/flows/flow-finish-recording.ts | 9 +- .../src/tools/flows/flow-insert-echo.ts | 2 +- .../tool-server/src/tools/flows/flow-run.ts | 2 +- .../src/tools/flows/flow-start-recording.ts | 2 +- .../tool-server/src/tools/flows/flow-utils.ts | 21 +++- .../simulator/stop-all-simulator-servers.ts | 21 ++-- .../flows/flow-concurrent-recording.test.ts | 117 ++++++++++++++++-- .../tool-server/test/flows/flow-tools.test.ts | 8 +- .../tool-server/test/flows/flow-utils.test.ts | 30 ++++- packages/tool-server/test/stop-tools.test.ts | 91 ++++++++++++-- 15 files changed, 355 insertions(+), 50 deletions(-) diff --git a/packages/argent-cli/test/flag-parser.test.ts b/packages/argent-cli/test/flag-parser.test.ts index aff0c06ff..fcb1d9ed7 100644 --- a/packages/argent-cli/test/flag-parser.test.ts +++ b/packages/argent-cli/test/flag-parser.test.ts @@ -114,15 +114,20 @@ describe("flag-parser array + -json interleave never throws a raw error", () => }); // A tool (like flow-add-step) whose schema declares its own `args` field — a -// JSON string holding the recorded step's tool arguments. +// JSON string holding the recorded step's tool arguments. Mirrors the schema the +// registry advertises for the real tool (zodObjectToJsonSchema over +// packages/tool-server/src/tools/flows/flow-add-step.ts): recordings are keyed by +// `name` + `project_root`, so both are required alongside `command`. const flowAddStepSchema: JsonSchema = { type: "object", properties: { + name: { type: "string" }, + project_root: { type: "string" }, command: { type: "string" }, args: { type: "string" }, delayMs: { type: "integer" }, }, - required: ["command"], + required: ["name", "project_root", "command"], }; // A tool (like gesture-tap) with NO `args` field — here `--args` must stay the @@ -138,6 +143,43 @@ const gestureTapSchema: JsonSchema = { }; describe("parseFlags — schema-aware --args", () => { + it("routes the recording identity through the plain scalar path", () => { + const result = parseFlags( + [ + "--name", + "checkout-e2e", + "--project_root", + "/Users/dev/My Projects/demo-app", + "--command", + "gesture-tap", + "--args", + '{"udid":"X"}', + ], + flowAddStepSchema + ); + expect(result.args.name).toBe("checkout-e2e"); + // `project_root` is the only schema field carrying an underscore, so it pins + // that flag names reach the payload verbatim — a parser that normalised them + // to camel/kebab case would file the value under the wrong key and the server + // would reject the step for a missing `project_root`. The value also holds a + // space: argv arrives already split, so it must survive whole. + expect(result.args.project_root).toBe("/Users/dev/My Projects/demo-app"); + expect(result.args.command).toBe("gesture-tap"); + expect(result.args.args).toBe('{"udid":"X"}'); + expect(result.rawArgs).toBeNull(); + }); + + it("routes the recording identity through the inline --field= form too", () => { + const result = parseFlags( + ["--name=checkout-e2e", "--project_root=/Users/dev/demo-app", "--command=screenshot"], + flowAddStepSchema + ); + expect(result.args.name).toBe("checkout-e2e"); + expect(result.args.project_root).toBe("/Users/dev/demo-app"); + expect(result.args.command).toBe("screenshot"); + expect(result.rawArgs).toBeNull(); + }); + it("treats --args as the tool's own string field (space-separated form)", () => { const result = parseFlags( ["--command", "gesture-tap", "--args", '{"udid":"X","x":0.5}'], diff --git a/packages/argent-cli/test/run-flow-add-step-payload.test.ts b/packages/argent-cli/test/run-flow-add-step-payload.test.ts index 0579ec4f5..a5a552354 100644 --- a/packages/argent-cli/test/run-flow-add-step-payload.test.ts +++ b/packages/argent-cli/test/run-flow-add-step-payload.test.ts @@ -34,7 +34,9 @@ function startServer(cap: Captured): Promise<{ url: string; close: () => Promise tools: [ { name: "flow-add-step", - description: "Add a step to the active flow recording", + // Leading sentence of the real tool description, verbatim. + description: + "Execute a tool call and record it as a step in the flow named by `name` + `project_root` (the recording must already be open — see flow-start-recording).", // Mirrors what the registry advertises for the real tool — // zodObjectToJsonSchema over the zod schema in // packages/tool-server/src/tools/flows/flow-add-step.ts. `name` diff --git a/packages/argent-cli/test/run-help.test.ts b/packages/argent-cli/test/run-help.test.ts index 3ec7addd8..05d31a2e2 100644 --- a/packages/argent-cli/test/run-help.test.ts +++ b/packages/argent-cli/test/run-help.test.ts @@ -24,18 +24,29 @@ vi.mock("@argent/tools-client", () => ({ vi.mock("@argent/telemetry", () => telemetryMock); -// A tool (like flow-add-step) that owns its `args` field. +// A tool (like flow-add-step) that owns its `args` field. Schema and +// description mirror what the registry advertises for the real tool - +// zodObjectToJsonSchema over the zod schema in +// packages/tool-server/src/tools/flows/flow-add-step.ts. Recordings are keyed +// by `name` + `project_root`, so both are required alongside `command` and only +// `args` / `delayMs` are optional; a fixture still describing a single "active" +// recording with one required field would render help for a tool that no longer +// exists. const flowAddStepMeta = { name: "flow-add-step", - description: "Add a step to the active flow recording", + // Leading sentence of the real tool description, verbatim. + description: + "Execute a tool call and record it as a step in the flow named by `name` + `project_root` (the recording must already be open — see flow-start-recording).", inputSchema: { type: "object", properties: { + name: { type: "string" }, + project_root: { type: "string" }, command: { type: "string" }, args: { type: "string" }, - delayMs: { type: "integer" }, + delayMs: { type: "integer", minimum: 0, maximum: 9007199254740991 }, }, - required: ["command"], + required: ["name", "project_root", "command"], }, }; @@ -100,4 +111,25 @@ describe("argent run --help — whole-payload --args advertisement", () => { expect(help).toContain("--args "); expect(toolsClientMock.callTool).not.toHaveBeenCalled(); }); + + it("renders each required flag with the (required) marker and leaves the optionals unmarked", async () => { + toolsClientMock.fetchTool.mockResolvedValue(flowAddStepMeta); + + await run(["flow-add-step", "--help"], { paths: {} as never }); + + const help = capturedHelp(); + // The tool's own prose is printed above the flag block, so the help names + // the recording a step is being added to rather than an implicit active one. + expect(help).toContain(flowAddStepMeta.description); + // The recording identity is required alongside `command`: omitting either + // flag fails the server's zod validation, so the help has to say so up front + // instead of presenting them as optional extras. + expect(help).toMatch(/--name \s+string \(required\)/); + expect(help).toMatch(/--project_root \s+string \(required\)/); + expect(help).toMatch(/--command \s+string \(required\)/); + // ...while the two genuinely optional fields must NOT carry the marker. + expect(help).toMatch(/--args \s+string(?! \(required\))/); + expect(help).toMatch(/--delayMs \s+integer(?! \(required\))/); + expect(toolsClientMock.callTool).not.toHaveBeenCalled(); + }); }); diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index bf88b6247..5a8f13ee9 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -141,7 +141,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. - **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. -- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. The symptom is a step in flight failing with `Recording of "" in is no longer active — it was restarted while this step was running…`; your _next_ `flow-add-step`/`flow-add-echo` then succeeds — into the other agent's recording. Restart under a fresh name instead of re-adding the step. +- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. Either way, restart under a fresh name instead of re-adding the step. - **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...` — the tail lists every live recording as `"name" (project_root)`, or `none`, so a typo or a wrong `project_root` is visible in the error itself. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. @@ -165,6 +165,8 @@ command: "flow-execute" args: "{\"name\": \"login\", \"project_root\": \"/Users/dev/MyApp\"}" ``` +Caveat to the "only successful steps are recorded" rule: if that sibling is a fragment with an `executionPrerequisite`, `flow-execute` returns its prerequisite **notice** instead of running - still a successful return, so `run: login` is recorded even though nothing executed. Add `"prerequisiteAcknowledged": true` to `args` to actually run it. + Record an `await-ui-element` step to **gate** the next step on a screen transition — it blocks until the element is `visible`/`hidden` (or contains `text`), so the following step runs only once the screen has actually settled; prefer this over a fixed `delayMs`. If its condition is not met before the timeout, replay **stops at that step** (the steps after it assume the transition happened). See the `await-ui-element` section of `argent-device-interact` for the full condition/selector reference. The live call sees only the trimmed `describe` tree — if it can't find an identifier you know exists, gate on visible text to get the step recorded, then retarget the identifier in the `await:` form during polish (the directive resolves the full hierarchy — see Selectors); don't conclude the testID is unusable in the flow. ## Recording @@ -201,7 +203,7 @@ Then polish the saved file: the two `await-ui-element` steps become `await:` dir ## Replaying -Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow can boot its own instance from the `launch` path and tear it down after, but only with no `device` **and** an unambiguous chromium target: an explicit `platform: "chromium"`, or a single-key `launch: { chromium: … }` map. A bare-string `launch:` — what the recorder always writes — or a multi-platform map carries no hint and falls through to ordinary device auto-detection, so pass `platform: "chromium"` when you want the self-boot.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. +Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow can boot its own instance and tear it down after, but only when all of these hold: no `device`, the launch resolves to chromium (an explicit `platform: "chromium"`, or a single-key `launch: { chromium: … }` map), and that launch value is a real Electron app path on the tool-server host. With no chromium hint - a bare-string or multi-platform `launch:` and no `platform` - the run auto-detects a booted device instead. **Don't reach for `platform: "chromium"` to force the self-boot on a recorded flow:** it does not fall through, it selects the boot branch, and a bare-string `launch:` - what the recorder always writes - holds an installed-app _bundle id_, which that branch reads as an app path. The whole `flow-execute` call then fails with `Electron boot: path does not exist: …`. Hand-edit the launch to `{ chromium: }` first.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. **What each step reports.** Raw `tool:` steps include the underlying tool's full `result` (screenshots and other outputs render as usual). The directive steps are summarized: `tap`/`type`/`await`/`assert` report only `status` + `reason`, and `snapshot` adds `artifacts` only when there is something to look at — a failed comparison (baseline/current/diff paths), a missing-baseline failure (`current` only), or a baseline write; a clean pass reports just `status` + `reason`. So converting a `tool: gesture-tap` into a `tap:` directive during cleanup drops only that tap's (uninteresting) raw result — output-bearing tools like `screenshot` have no directive form and stay `tool:` steps, so their results keep flowing through. @@ -263,7 +265,7 @@ For silent misfires and partial divergence, echo annotations (see _Making flows `debugger-component-tree` is an **authoring aid only — never record a `debugger-*` step into a flow.** `device_id` is stripped at record time and re-injected at replay, but `port` is not a device-bind key, so a recorded debugger step carries whatever `port` it was given (or falls through to the 8081 default at replay) and runs against whatever Metro happens to be on that port. - When calling any `debugger-*` tool directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. That last one does not rescue `debugger-component-tree` itself: it is capability-gated off Vega, and on RN 0.72's Hermes it hangs until timeout (the binding it delivers the tree over is never installed) — use `describe` there. + When calling any `debugger-*` tool directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. That last one does not rescue `debugger-component-tree` itself: it is capability-gated off Vega, and on a legacy-inspector RN 0.72 Hermes it fails fast with a coded error pointing you at `describe` (the binding it delivers the tree over is ACKed but never installed, and a probe at connect catches that) - so use `describe` there. 4. Compare current state to what the failed step expected. Classify the root cause: diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index ec146671e..7ea75f785 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -36,7 +36,7 @@ const zodSchema = z.object({ .describe( "Absolute path to the project root of the flow being recorded — the same value passed to flow-start-recording. Together with `name` it identifies which recording this step belongs to." ), - command: z.string().describe('MCP tool name (e.g. "tap", "screenshot", "launch-app")'), + command: z.string().describe('MCP tool name (e.g. "gesture-tap", "screenshot", "launch-app")'), args: z .string() .optional() @@ -450,7 +450,7 @@ export function createFlowAddStepTool( failedMsg: ({ params, failureSignal }) => `Failed to add ${params.command} step to flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Execute a tool call and record it as a step in the flow named by \`name\` + \`project_root\` (the recording must already be open — see flow-start-recording). Use when recording a flow and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile } on success. If it fails an error is returned and nothing is recorded. + description: `Execute a tool call and record it as a step in the flow named by \`name\` + \`project_root\` (the recording must already be open — see flow-start-recording). Use when recording a flow and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile, savedTo } on success - \`savedTo\` is where the YAML landed: a host path, or, against a remote client, the directive that has the client write it (the only field naming the destination in that mode). If it fails an error is returned and nothing is recorded. If a step was recorded by mistake, edit the .yaml file directly to remove it.`, zodSchema, services: () => ({}), diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 74005f865..1ae1ae554 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -111,8 +111,15 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps flowFile = await fs.readFile(filePath, "utf8"); savedTo = filePath; } + // Parse BEFORE clearing. Hand-editing the .yaml mid-recording is a + // documented workflow, so parseFlow can legitimately throw here on a + // botched edit — and clearing first would destroy the session on the + // way out, leaving the agent unable to retry the finish after repairing + // the file (the only tool that re-establishes the key, + // flow-start-recording, truncates the take it would be recovering). + const flow = parseFlow(flowFile); clearRecordingSession(params.project_root, params.name); - return { filePath, flowFile, savedTo, flow: parseFlow(flowFile) }; + return { filePath, flowFile, savedTo, flow }; } ); diff --git a/packages/tool-server/src/tools/flows/flow-insert-echo.ts b/packages/tool-server/src/tools/flows/flow-insert-echo.ts index e70af3cc1..4075ebb4e 100644 --- a/packages/tool-server/src/tools/flows/flow-insert-echo.ts +++ b/packages/tool-server/src/tools/flows/flow-insert-echo.ts @@ -29,7 +29,7 @@ export const flowInsertEchoTool: ToolDefinition< }, description: `Record an echo step in the flow named by \`name\` + \`project_root\`. Echo steps print a message when the flow is replayed — useful as labels between tool calls. Use when you want to annotate a recorded flow with a human-readable label or checkpoint message. -Returns { message, flowFile }. Fails if that flow has no recording in progress.`, +Returns { message, flowFile, savedTo } - \`savedTo\` is where the YAML landed: a host path, or, against a remote client, the directive that has the client write it (the only field naming the destination in that mode). Fails if that flow has no recording in progress.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index aa09ae939..da99d4cab 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -107,7 +107,7 @@ const zodSchema = z .enum(LAUNCH_PLATFORMS) .optional() .describe( - "Restrict auto-detection to this platform when several devices are booted. `chromium` does more than filter: with no `device` it makes an e2e flow boot its own Electron instance from its `launch` step's chromium app path and tear it down after the run (a single-key `launch: { chromium: … }` map does that on its own; a bare-string `launch:` never does)." + "Restrict auto-detection to this platform when several devices are booted. `chromium` does more than filter: with no `device` it SELECTS the self-boot branch for an e2e flow - the runner boots an Electron instance from the `launch` step's chromium value and tears it down after the run (a single-key `launch: { chromium: … }` map selects it on its own, without this parameter). It never falls back to device auto-detection, and the launch value must be a real Electron app path on the tool-server host: a bare-string `launch:` - what the recorder writes - holds an installed-app bundle id, so passing `chromium` for one fails the whole run with `Electron boot: path does not exist`. Edit the launch to `{ chromium: }` first." ), updateBaselines: z .boolean() diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index c4095115c..b274d388c 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -65,7 +65,7 @@ export const flowStartRecordingTool: ToolDefinition< Use when you want to capture a reusable sequence of device interactions for later replay. Returns { message, flowFile, savedTo }. Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted, discardedSteps } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. Either way, re-record from the top rather than expecting to resume. -Fails if the .argent/flows/ directory cannot be created or the flow file cannot be written. +Fails before anything is written on a \`project_root\` that is not absolute or contains a ".." segment, or a \`name\` outside letters/digits/underscore/hyphen. It can also fail on the .argent/flows/ directory not being creatable or the file not being writable - but only when the project root is on the tool-server host; against a remote client the YAML travels back in \`savedTo\` for the client to write and no host filesystem access happens. Recording state is independent: several flows can be recorded at once (different names, different projects) and one recording's steps never land in another's diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 04ec72510..459394e40 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -331,13 +331,22 @@ export function listActiveRecordings(): { name: string; projectRoot: string; ste export function requireRecordingSession(projectRoot: string, name: string): RecordingSession { const session = getRecordingSession(projectRoot, name); if (!session) { - // Name what was asked for AND what is live: with concurrent recordings the - // usual cause is a typo or the wrong project_root, and the agent can only - // self-correct if it can see the keys that do exist. + // Name what was asked for AND what is live, so the agent can self-correct: + // with concurrent recordings the usual cause is a typo in `name` or the + // wrong `project_root`. + // + // Only this project's recordings are named. The others are counted, not + // listed: a tool-server bound beyond loopback is shared by unrelated + // callers (that is what "client" persist mode exists for), and their flow + // names and absolute project paths are not this caller's to see. A typo in + // your own project — the case worth recovering from — is still spelled out. const active = listActiveRecordings(); - const activeList = active.length - ? active.map((r) => `"${r.name}" (${r.projectRoot})`).join(", ") - : "none"; + const here = active.filter((r) => r.projectRoot === projectRoot); + const elsewhere = active.length - here.length; + const others = elsewhere > 0 ? ` (plus ${elsewhere} in other projects)` : ""; + const activeList = here.length + ? `${here.map((r) => `"${r.name}"`).join(", ")}${others}` + : `none in this project${others}`; throw new FailureError( `No active recording for flow "${name}" in ${projectRoot}. ` + `Call flow-start-recording first. Active recordings: ${activeList}.`, diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 5602ebaa3..9587c9eef 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -78,7 +78,7 @@ export function createStopAllSimulatorServersTool( }, description: `Stop running simulator-server processes (iOS + Android), native devtools services, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps, silently). Omit \`devices\` only when a machine-wide cleanup is what you actually want. -Returns { stopped } — an array of URNs that were shut down — plus { unmatched } naming any id in \`devices\` that owned no services, so a mistyped id or a device name passed where an id was expected does not read as a clean machine. Never throws.`, +Returns { stopped } - the URNs of the services that were actually live and got shut down; ERROR/TERMINATING nodes are disposed too but never appear there, so an empty \`stopped\` only means nothing was still running. { unmatched } is present ONLY when \`devices\` was supplied AND at least one of its ids owns no service at all in these namespaces - absent on an unscoped call and when every id matched - so a mistyped id, or a device NAME passed where an id was expected, does not read as a clean machine. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { @@ -93,8 +93,13 @@ Returns { stopped } — an array of URNs that were shut down — plus { unmatche for (const [urn, entry] of snapshot.services) { const matchedId = scoped ? matchingDeviceId(urn, devices) : undefined; const matches = scoped ? matchedId !== undefined : PREFIXES.some((p) => urn.startsWith(p)); + // Ownership is recorded regardless of state. `disposeService` moves a + // node to IDLE without removing it, so a device this session already + // stopped would otherwise be reported as unmatched by the next scoped + // call — turning the routine "stop one, then stop the rest" sequence + // into a false alarm about a mistyped id. + if (matchedId !== undefined) matchedIds.add(matchedId.toLowerCase()); if (matches && entry.state !== ServiceState.IDLE) { - if (matchedId !== undefined) matchedIds.add(matchedId); // Dispose any non-IDLE node (this also clears ERROR/TERMINATING // nodes), but only report the ones that were actually live — an // ERROR node (e.g. a tvOS SimulatorServer that refused to start) @@ -105,11 +110,13 @@ Returns { stopped } — an array of URNs that were shut down — plus { unmatche } } if (!scoped) return { stopped }; - // A scoped stop that matched nothing is indistinguishable from a clean - // machine unless we say so — and the ids that miss are exactly the ones - // whose simulator-server, devtools and (on tvOS) two --timeout 3600 - // daemons are being left running. - const unmatched = devices.filter((id) => !matchedIds.has(id)); + // A scoped stop that named an id owning nothing is indistinguishable from + // a clean machine unless we say so — and that id is usually a typo, or a + // device NAME passed where an id belongs, in which case its + // simulator-server, devtools and (on tvOS) two --timeout 3600 daemons are + // being left running. Compared case-insensitively to match the lookup, + // and de-duplicated so a repeated id is reported once. + const unmatched = [...new Set(devices)].filter((id) => !matchedIds.has(id.toLowerCase())); return unmatched.length > 0 ? { stopped, unmatched } : { stopped }; }, }; diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index b9251e418..67ec97047 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -38,7 +38,9 @@ import { * an await — a restart's truncate-then-register, a finish's read-then-clear, an * append's read-then-write — so each is covered by the per-flow-file lock, and a * step that resolved its session before some other tool superseded it must fail - * rather than write into a file that now belongs to a different take. + * rather than write into a file that now belongs to a different take. A finish + * that fails inside its critical section must also leave the recording live, so + * the take survives the failure and can be finished on a retry. */ const IOS_DEVICE = "00000000-0000-0000-0000-0000000000ab"; @@ -314,7 +316,7 @@ describe("the same flow name under two project roots", () => { // ── Addressing a key that isn't live ───────────────────────────────── describe("addressing an unknown recording key", () => { - it("fails with FLOW_NO_ACTIVE_RECORDING and lists the live recordings", async () => { + it("fails with FLOW_NO_ACTIVE_RECORDING and names this project's live recordings", async () => { const rootA = await makeRoot("unknown-a"); const rootB = await makeRoot("unknown-b"); await start(rootA, "alpha"); @@ -326,9 +328,11 @@ describe("addressing an unknown recording key", () => { const message = (err as Error).message; expect(message).toContain('No active recording for flow "never-started"'); expect(message).toContain(rootA); - // The live keys are named so the agent can self-correct. - expect(message).toContain(`"alpha" (${rootA})`); - expect(message).toContain(`"beta" (${rootB})`); + // This project's live keys are named so the agent can self-correct… + expect(message).toContain('Active recordings: "alpha" (plus 1 in other projects)'); + // …while another caller's flow name and project path stay theirs. + expect(message).not.toContain('"beta"'); + expect(message).not.toContain(rootB); }); it("fails the same way for the right name under the wrong project_root", async () => { @@ -339,7 +343,9 @@ describe("addressing an unknown recording key", () => { const err = await captureFailure(addStep(rootB, "alpha", "stray")); expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); - expect((err as Error).message).toContain(`"alpha" (${rootA})`); + const message = (err as Error).message; + expect(message).toContain("Active recordings: none in this project (plus 1 in other projects)"); + expect(message).not.toContain(rootA); // The misdirected step was not recorded anywhere. expect(await readMarkers(rootA, "alpha")).toEqual([]); @@ -350,7 +356,8 @@ describe("addressing an unknown recording key", () => { const root = await makeRoot("nothing-live"); const err = await captureFailure(finish(root, "alpha")); expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); - expect((err as Error).message).toContain("Active recordings: none."); + // No parenthetical: there is nothing elsewhere to count either. + expect((err as Error).message).toContain("Active recordings: none in this project."); }); }); @@ -729,6 +736,102 @@ describe("a finish that lands while a step is still running", () => { }); }); +// ── A finish whose file a hand-edit broke ──────────────────────────── + +describe("a finish on a flow file that no longer parses", () => { + // Hand-editing the .yaml mid-recording is a documented workflow, so parseFlow + // can legitimately throw inside flow-finish-recording's critical section. The + // session must survive that: clearing the key first leaves the agent unable to + // retry the finish after repairing the file — flow-finish-recording answers + // "No active recording", and the only tool that re-establishes the key, + // flow-start-recording, truncates the very take it would be recovering. + + /** `steps` present but not a list — a shape parseFlow rejects outright. */ + const NOT_A_LIST = 'executionPrerequisite: ""\nsteps: oops\n'; + + it("keeps the recording live and finishable once the file is repaired", async () => { + const root = await makeRoot("finish-unparseable"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + await addEcho(root, "alpha", "a2"); + const session = getRecordingSession(root, "alpha"); + const repaired = await fs.readFile(flowPath(root, "alpha"), "utf8"); + + await fs.writeFile(flowPath(root, "alpha"), NOT_A_LIST, "utf8"); + + const err = await captureFailure(finish(root, "alpha")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_INVALID); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_file_parse"); + + // The finish reads and clears; it never writes — the botched edit is still + // on disk byte for byte, so the agent can diff it against what it typed. + expect(await fs.readFile(flowPath(root, "alpha"), "utf8")).toBe(NOT_A_LIST); + + // The take survived the failure, as the same session object. + expect(getRecordingSession(root, "alpha")).toBe(session); + expect(session?.flow.steps).toHaveLength(2); + + // A retry while the file is still broken fails the same way — the recording + // is live (the call got past requireRecordingSession), the FILE is at fault. + const again = await captureFailure(finish(root, "alpha")); + expect(getFailureSignal(again)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_INVALID); + + // Repair the file the way the agent would, then carry on recording… + await fs.writeFile(flowPath(root, "alpha"), repaired, "utf8"); + await addEcho(root, "alpha", "a3"); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1", "echo:a2", "echo:a3"]); + + // …and the retried finish succeeds, reporting the repaired file. + const finished = await finish(root, "alpha"); + expect(finished.steps).toBe(3); + expect(finished.summary).toHaveLength(3); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1", "echo:a2", "echo:a3"]); + expect(getRecordingSession(root, "alpha")).toBeUndefined(); + }); + + it("leaves a concurrent recording — and its own key — exactly as they were", async () => { + const root = await makeRoot("finish-unparseable-step"); + await start(root, "alpha"); + await start(root, "beta"); + await addStep(root, "alpha", "a1"); + await addEcho(root, "beta", "b1"); + + // A second botched-edit shape: a step whose directive key is a typo. + await fs.writeFile( + flowPath(root, "alpha"), + 'executionPrerequisite: ""\nsteps:\n - ecko: oops\n', + "utf8" + ); + + const err = await captureFailure(finish(root, "alpha")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_ENTRY_UNRECOGNIZED); + + // Both keys are still live, each bound to its own file. + expect( + listActiveRecordings() + .map((r) => r.name) + .sort() + ).toEqual(["alpha", "beta"]); + expect(getRecordingSession(root, "alpha")?.filePath).toBe(flowPath(root, "alpha")); + + // beta neither lost its file nor its ability to finish. + expect(await readMarkers(root, "beta")).toEqual(["echo:b1"]); + const finishedBeta = await finish(root, "beta"); + expect(markers(parseFlow(finishedBeta.flowFile).steps)).toEqual(["echo:b1"]); + + // alpha outlived beta's finish too, and finishes on the repaired file. + expect(getRecordingSession(root, "alpha")).toBeDefined(); + await fs.writeFile( + flowPath(root, "alpha"), + 'executionPrerequisite: ""\nsteps:\n - echo: repaired\n', + "utf8" + ); + const finishedAlpha = await finish(root, "alpha"); + expect(markers(parseFlow(finishedAlpha.flowFile).steps)).toEqual(["echo:repaired"]); + expect(listActiveRecordings()).toEqual([]); + }); +}); + // ── The concurrent-recording cap ───────────────────────────────────── describe("the concurrent-recording cap", () => { diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 9c078da31..96a3a485c 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -315,8 +315,12 @@ describe("flow-add-echo", () => { .catch((e: unknown) => e as Error); expect(err.message).toContain("No active recording"); - // The error names what IS live, so a wrong project_root is self-correcting. - expect(err.message).toContain(`Active recordings: "wrong-root" (${tmpDir})`); + // The error names the key that was asked for, and counts — without naming — + // the recordings live under other roots, so a wrong project_root is + // recognizable without disclosing another project's flows. + expect(err.message).toContain(`No active recording for flow "wrong-root" in ${otherDir}`); + expect(err.message).toContain("Active recordings: none in this project (plus 1 in other"); + expect(err.message).not.toContain(tmpDir); }); }); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 761065d05..7affd6e03 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1044,7 +1044,7 @@ describe("recording sessions", () => { expect(getFailureSignal(caught)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); }); - it("names the asked-for key and lists the live recordings in the not-found message", () => { + it("names the asked-for key and this project's live recordings in the not-found message", () => { // With concurrent recordings the usual cause is a typo or the wrong // project_root; the agent can only self-correct if it sees the live keys. start("/tmp/proj-a", "checkout"); @@ -1053,13 +1053,35 @@ describe("recording sessions", () => { /No active recording for flow "chekout" in \/tmp\/proj-a\./ ); expect(() => requireRecordingSession("/tmp/proj-a", "chekout")).toThrow( - /Active recordings: "checkout" \(\/tmp\/proj-a\), "login" \(\/tmp\/proj-b\)\./ + /Active recordings: "checkout" \(plus 1 in other projects\)\./ ); }); - it('reports "none" when nothing is being recorded', () => { + it("counts other projects' recordings without naming them", () => { + // A tool-server bound beyond loopback serves unrelated callers; another + // project's flow names and absolute paths are not this caller's to see. + start("/tmp/proj-b", "login"); + start("/tmp/proj-c", "secret-onboarding"); + const message = (() => { + try { + requireRecordingSession("/tmp/proj-a", "my-flow"); + } catch (err) { + return (err as Error).message; + } + throw new Error("expected a throw"); + })(); + expect(message).toMatch( + /Active recordings: none in this project \(plus 2 in other projects\)\./ + ); + expect(message).not.toContain("login"); + expect(message).not.toContain("secret-onboarding"); + expect(message).not.toContain("/tmp/proj-b"); + expect(message).not.toContain("/tmp/proj-c"); + }); + + it('reports "none in this project" when nothing is being recorded', () => { expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( - /Active recordings: none\./ + /Active recordings: none in this project\./ ); }); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 4b622e2ab..13441acf7 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -11,7 +11,15 @@ function createMockRegistry(services: Map {}), + // The real `disposeService` returns the node to IDLE and LEAVES IT IN the + // map (Registry._teardown), rather than removing it — so a second stop of + // the same device still sees its URNs, in IDLE. Mirror that here: a mock + // that forgets disposed nodes would hide exactly the sequence the + // stop-one-then-stop-the-rest tests below exist to pin. + disposeService: vi.fn(async (urn: string) => { + const node = services.get(urn); + if (node) node.state = ServiceState.IDLE; + }), } as unknown as Registry; } @@ -462,34 +470,101 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(result).not.toHaveProperty("unmatched"); }); - it("counts an id whose only service is IDLE as unmatched — nothing was stopped for it", async () => { + it("does not report an all-IDLE device as unmatched — it still owns those nodes", async () => { + // `disposeService` returns a node to IDLE without removing it, so this is + // precisely the state a device is left in by a stop THIS session already + // performed. `unmatched` means "this id owns nothing on the machine, look + // for a typo"; saying it about a device we just tore down ourselves is a + // false alarm on the routine stop-one-then-stop-the-rest sequence. const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.IDLE, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.IDLE, dependents: [] }], [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, { devices: [MINE] }); + const result = await tool.execute!({}, { devices: [MINE, "GHOST-9999"] }); - expect(result).toEqual({ stopped: [], unmatched: [MINE] }); + // Nothing left to stop for MINE, but only the id that owns no node at all + // is a miss. + expect(result).toEqual({ stopped: [], unmatched: ["GHOST-9999"] }); expect(registry.disposeService).not.toHaveBeenCalled(); }); + it("reports nothing unmatched when the same device is stopped twice in a row", async () => { + // The session-end sequence the argent rules prescribe: stop the device you + // finished with, then sweep the rest. The second call finds every URN the + // first one left behind in IDLE, and must not read that as a mistyped id. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const first = await tool.execute!({}, { devices: [MINE] }); + expect(first).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE}`], + }); + expect(first).not.toHaveProperty("unmatched"); + + const second = await tool.execute!({}, { devices: [MINE] }); + expect(second).toEqual({ stopped: [] }); + expect(second).not.toHaveProperty("unmatched"); + // The second call had nothing live to tear down. + expect(registry.disposeService).toHaveBeenCalledTimes(2); + }); + + it("names a repeated missing id only once", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + // A device list assembled from several sources can repeat an id; the + // warning is about the id, not about how many times it was passed. + const result = await tool.execute!({}, { devices: [MINE, "GHOST-9999", MINE, "GHOST-9999"] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`], + unmatched: ["GHOST-9999"], + }); + }); + + it("reports neither spelling when one device is named twice in different cases", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + // Matching is case-insensitive, so both spellings name the same device — + // and the device matched. Neither is a miss. + const result = await tool.execute!({}, { devices: [MINE, MINE.toLowerCase()] }); + + expect(result).toEqual({ stopped: [`SimulatorServer:${MINE}`] }); + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).toHaveBeenCalledOnce(); + }); + it("does not report an ERROR-only device as unmatched — its dead node was cleaned up", async () => { - // The boundary against the IDLE case above: an ERROR node is never reported - // as `stopped` (it never ran), but it IS disposed, so the id did own - // something and calling it unmatched would be a false alarm. + // The other side of the IDLE case above: neither state is a miss (both own + // nodes), but an ERROR node is still DISPOSED — it never ran, so it never + // shows up in `stopped`, yet the dead node has to be cleared. const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.ERROR, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.IDLE, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, { devices: [MINE] }); + const result = await tool.execute!({}, { devices: [MINE, THEIRS] }); expect(result).toEqual({ stopped: [] }); expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).toHaveBeenCalledOnce(); expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${MINE}`); }); From 451cc84cc465ef088b75a126dd39ef5a97c2d57d Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 20:08:34 +0200 Subject: [PATCH 05/98] fix(flow): hold the invariant for the whole critical section, and pin it Third review pass. Nothing blocking in the lock itself, but three edges: - flow-finish-recording rendered the step summary AFTER clearing the session, and that renderer walks step bodies the parser does not fully constrain, so a throw there destroyed the recording the previous commit had just made recoverable. Summary now renders inside the lock, before the clear, so nothing that can throw runs after a destructive step. - the not-found message compared project roots as raw strings while the map key is path.join-normalized, so a caller spelling its own root with a trailing slash was told its recording lived in another project -- degrading the message in the case it exists to diagnose. - stop-all-simulator-servers de-duplicated unmatched ids by exact string though it matches case-insensitively, so two spellings of one wrong id were reported as two mistakes. The start side of the lock was also load-bearing but unpinned: moving startRecordingSession out of the critical section passed the entire suite. It is now covered by a test that queues an append from the discarded take directly on the flow file's lock, behind the restart -- proven to fail against that mutation, 15/15 runs. Plus the agent-facing prose for the error format the previous commit changed, and three descriptions that over-claimed. --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-finish-recording.ts | 125 ++++++++++-------- .../tool-server/src/tools/flows/flow-run.ts | 2 +- .../tool-server/src/tools/flows/flow-utils.ts | 12 +- .../simulator/stop-all-simulator-servers.ts | 13 +- .../flows/flow-concurrent-recording.test.ts | 61 ++++++++- 6 files changed, 147 insertions(+), 68 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 5a8f13ee9..9753d1e29 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -142,7 +142,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. - **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. - **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. Either way, restart under a fresh name instead of re-adding the step. -- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...` — the tail lists every live recording as `"name" (project_root)`, or `none`, so a typo or a wrong `project_root` is visible in the error itself. +- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...`. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. ### flow-add-step arguments diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 1ae1ae554..4a5b45e14 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -9,6 +9,7 @@ import { parseFlow, serializeFlow, selectorToYaml, + type FlowFile, type FlowSavedTo, type FlowSelector, } from "./flow-utils"; @@ -83,7 +84,7 @@ export const flowFinishRecordingTool: ToolDefinition< failedMsg: ({ params, failureSignal }) => `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving any other recordings in progress untouched. Returns a summary of all recorded steps and the final YAML content. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. + description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving any other recordings in progress untouched. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. You can still edit the .yaml file directly afterwards to remove or reorder steps.`, zodSchema, services: () => ({}), @@ -92,7 +93,7 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps // Host mode's `await fs.readFile` is a yield, and an append that lands in it // would be on disk while the summary and step count reported here — taken // from the pre-append read — say otherwise. - const { filePath, flowFile, savedTo, flow } = await withFlowFileLock( + const { filePath, flowFile, savedTo, flow, summary } = await withFlowFileLock( params.project_root, params.name, async () => { @@ -118,66 +119,16 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps // the file (the only tool that re-establishes the key, // flow-start-recording, truncates the take it would be recovering). const flow = parseFlow(flowFile); + // Render the summary before clearing too, for the same reason: it walks + // step bodies the parser does not fully constrain (`JSON.stringify` on + // a tool step's args can throw on a cyclic YAML anchor), and nothing + // that can throw may run after the session is destroyed. + const summary = summarizeSteps(flow); clearRecordingSession(params.project_root, params.name); - return { filePath, flowFile, savedTo, flow }; + return { filePath, flowFile, savedTo, flow, summary }; } ); - const summary = flow.steps.map((step, i) => { - const n = i + 1; - switch (step.kind) { - case "echo": - return `${n}. echo: ${step.message}`; - case "launch": - return `${n}. launch: ${typeof step.app === "string" ? step.app : JSON.stringify(step.app)}`; - case "run": - return `${n}. run: ${step.flow}`; - case "tap": - case "long-press": - return `${n}. ${step.kind}: ${step.selector ? selectorLabel(step.selector) : `(${step.x}, ${step.y})`}`; - case "type": - return `${n}. type: ${selectorLabel(step.into)} ← "${step.text}"`; - case "await": - case "assert": { - const tail = - step.condition === "text" - ? textConditionLabel(step.selector, step.expectedText, step.textMatch) - : `${step.condition} ${selectorLabel(step.selector)}`; - return `${n}. ${step.kind}: ${tail}`; - } - case "wait": - return `${n}. wait: ${step.ms}ms`; - case "when": { - // Mirror the await/assert rendering above — selectorLabel spelling, - // same comparator tail for text guards. - const cond = - step.condition.kind === "platform" - ? `platform ${step.condition.platform}` - : step.condition.condition === "text" - ? textConditionLabel( - step.condition.selector, - step.condition.expectedText, - step.condition.textMatch - ) - : `${step.condition.condition} ${selectorLabel(step.condition.selector)}`; - // Pluralize like flow-run's skip reason so the two surfaces agree. - const count = step.steps.length; - return `${n}. when: ${cond} (${count} step${count === 1 ? "" : "s"})`; - } - case "scroll-to": - return `${n}. scroll-to: ${selectorLabel(step.target)} (${step.direction})`; - case "pinch": - return `${n}. pinch: scale ${step.scale}${step.selector ? ` on ${selectorLabel(step.selector)}` : ""}`; - case "rotate": - return `${n}. rotate: by ${step.by}°${step.selector ? ` on ${selectorLabel(step.selector)}` : ""}`; - case "snapshot": - return `${n}. snapshot: ${step.name}`; - case "tool": - default: - return `${n}. tool: ${step.name} ${JSON.stringify(step.args)}`; - } - }); - return { message: `Finished recording "${params.name}" flow (${flow.steps.length} steps)`, path: filePath, @@ -189,3 +140,61 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps }; }, }; + +/** One human-readable line per recorded step, in the flow file's own spellings. */ +function summarizeSteps(flow: FlowFile): string[] { + return flow.steps.map((step, i) => { + const n = i + 1; + switch (step.kind) { + case "echo": + return `${n}. echo: ${step.message}`; + case "launch": + return `${n}. launch: ${typeof step.app === "string" ? step.app : JSON.stringify(step.app)}`; + case "run": + return `${n}. run: ${step.flow}`; + case "tap": + case "long-press": + return `${n}. ${step.kind}: ${step.selector ? selectorLabel(step.selector) : `(${step.x}, ${step.y})`}`; + case "type": + return `${n}. type: ${selectorLabel(step.into)} ← "${step.text}"`; + case "await": + case "assert": { + const tail = + step.condition === "text" + ? textConditionLabel(step.selector, step.expectedText, step.textMatch) + : `${step.condition} ${selectorLabel(step.selector)}`; + return `${n}. ${step.kind}: ${tail}`; + } + case "wait": + return `${n}. wait: ${step.ms}ms`; + case "when": { + // Mirror the await/assert rendering above — selectorLabel spelling, + // same comparator tail for text guards. + const cond = + step.condition.kind === "platform" + ? `platform ${step.condition.platform}` + : step.condition.condition === "text" + ? textConditionLabel( + step.condition.selector, + step.condition.expectedText, + step.condition.textMatch + ) + : `${step.condition.condition} ${selectorLabel(step.condition.selector)}`; + // Pluralize like flow-run's skip reason so the two surfaces agree. + const count = step.steps.length; + return `${n}. when: ${cond} (${count} step${count === 1 ? "" : "s"})`; + } + case "scroll-to": + return `${n}. scroll-to: ${selectorLabel(step.target)} (${step.direction})`; + case "pinch": + return `${n}. pinch: scale ${step.scale}${step.selector ? ` on ${selectorLabel(step.selector)}` : ""}`; + case "rotate": + return `${n}. rotate: by ${step.by}°${step.selector ? ` on ${selectorLabel(step.selector)}` : ""}`; + case "snapshot": + return `${n}. snapshot: ${step.name}`; + case "tool": + default: + return `${n}. tool: ${step.name} ${JSON.stringify(step.args)}`; + } + }); +} diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index da99d4cab..f025fbaa3 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -107,7 +107,7 @@ const zodSchema = z .enum(LAUNCH_PLATFORMS) .optional() .describe( - "Restrict auto-detection to this platform when several devices are booted. `chromium` does more than filter: with no `device` it SELECTS the self-boot branch for an e2e flow - the runner boots an Electron instance from the `launch` step's chromium value and tears it down after the run (a single-key `launch: { chromium: … }` map selects it on its own, without this parameter). It never falls back to device auto-detection, and the launch value must be a real Electron app path on the tool-server host: a bare-string `launch:` - what the recorder writes - holds an installed-app bundle id, so passing `chromium` for one fails the whole run with `Electron boot: path does not exist`. Edit the launch to `{ chromium: }` first." + "Restrict auto-detection to this platform when several devices are booted. `chromium` does more than filter: with no `device` it SELECTS the self-boot branch for an e2e flow - the runner boots an Electron instance from the `launch` step's chromium value and tears it down after the run (a single-key `launch: { chromium: … }` map selects it on its own, without this parameter). When it selects that branch it never falls back to device auto-detection (a fragment, or an e2e launch map with no `chromium` key, still does), and the launch value must be a real Electron app path on the tool-server host: a bare-string `launch:` - what the recorder writes - holds an installed-app bundle id, so passing `chromium` for one fails the whole run with `Electron boot: path does not exist`. Edit the launch to `{ chromium: }` first." ), updateBaselines: z .boolean() diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 459394e40..c315aeec1 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -319,7 +319,10 @@ export function getRecordingSession( return recordings.get(getFlowPath(projectRoot, name)); } -/** Every live recording, for diagnostics and the not-found error message. */ +/** + * Every live recording. Feeds the not-found error message, which names only + * the caller's own project; `steps` is carried for tests and diagnostics. + */ export function listActiveRecordings(): { name: string; projectRoot: string; steps: number }[] { return [...recordings.values()].map((s) => ({ name: s.name, @@ -341,7 +344,12 @@ export function requireRecordingSession(projectRoot: string, name: string): Reco // names and absolute project paths are not this caller's to see. A typo in // your own project — the case worth recovering from — is still spelled out. const active = listActiveRecordings(); - const here = active.filter((r) => r.projectRoot === projectRoot); + // Compare roots the way the key does (path.join-normalized), or a caller + // that spells its own root with a trailing slash would be told its live + // recording is in "another project" — degrading the message in exactly the + // wrong-project_root case it exists to diagnose. + const hereDir = getFlowsDir(projectRoot); + const here = active.filter((r) => getFlowsDir(r.projectRoot) === hereDir); const elsewhere = active.length - here.length; const others = elsewhere > 0 ? ` (plus ${elsewhere} in other projects)` : ""; const activeList = here.length diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 9587c9eef..984a95a36 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -114,9 +114,16 @@ Returns { stopped } - the URNs of the services that were actually live and got s // a clean machine unless we say so — and that id is usually a typo, or a // device NAME passed where an id belongs, in which case its // simulator-server, devtools and (on tvOS) two --timeout 3600 daemons are - // being left running. Compared case-insensitively to match the lookup, - // and de-duplicated so a repeated id is reported once. - const unmatched = [...new Set(devices)].filter((id) => !matchedIds.has(id.toLowerCase())); + // being left running. Compared AND de-duplicated case-insensitively, to + // match the lookup: two spellings of one id are one mistake, reported in + // the caller's first spelling. + const seen = new Set(); + const unmatched = devices.filter((id) => { + const key = id.toLowerCase(); + if (matchedIds.has(key) || seen.has(key)) return false; + seen.add(key); + return true; + }); return unmatched.length > 0 ? { stopped, unmatched } : { stopped }; }, }; diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 67ec97047..de276e46e 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -543,7 +543,7 @@ describe("restarting a recording on one key", () => { expect(await readMarkers(root, "alpha")).toEqual(["tool:a3"]); }); - it("does not restart a same-named recording in another project", async () => { + it("restarts this project's take and leaves the same name elsewhere alone", async () => { const rootA = await makeRoot("restart-a"); const rootB = await makeRoot("restart-b"); @@ -552,13 +552,14 @@ describe("restarting a recording on one key", () => { await start(rootB, "alpha"); await addStep(rootB, "alpha", "b1"); - // Same name, different root ⇒ a different key ⇒ not a restart. + // Same name AND same root ⇒ the same key ⇒ this take is restarted… const restarted = await start(rootB, "alpha"); expect(restarted.restarted).toBe(true); expect(restarted.discardedSteps).toBe(1); expect(await readMarkers(rootB, "alpha")).toEqual([]); - // The other project's recording kept its step and its session. + // …while the same name under the other root — a different key — is not + // touched: that recording kept its step and its session. expect(await readMarkers(rootA, "alpha")).toEqual(["tool:a1"]); expect(getRecordingSession(rootA, "alpha")?.flow.steps).toHaveLength(1); }); @@ -634,6 +635,60 @@ describe("a restart that lands while a step is still running", () => { expect(await readMarkers(root, "alpha")).toEqual([]); expect(getRecordingSession(root, "alpha")).not.toBe(firstSession); }); + + it("keeps a step queued behind the restart out of the new take", async () => { + const root = await makeRoot("restart-queued-append"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + const discarded = getRecordingSession(root, "alpha"); + + // Park a holder on alpha's file lock. Everything issued below queues behind + // it, so the interleaving is fixed by the lock's arrival order rather than + // by how long any I/O happens to take. + const holder = openGate(); + const held = withFlowFileLock(root, "alpha", () => holder.promise); + + // Second in the queue: the restart — truncate the file, swap the session. + const restarting = start(root, "alpha"); + // Third: a step for the take the restart is discarding. flow-add-echo + // resolves its session and takes the lock in one synchronous block, so this + // append is bound to the OLD session and enters the lock the instant the + // restart's critical section ends — the window a truncate that is not fused + // to the session swap leaves open, onto a file that is already empty. + const appending = addEcho(root, "alpha", "stray"); + expect(getRecordingSession(root, "alpha")).toBe(discarded); + + holder.open(); + const [restartResult, appendResult] = await Promise.allSettled([restarting, appending]); + await held; + + if (restartResult.status === "rejected") throw restartResult.reason; + expect(restartResult.value.restarted).toBe(true); + expect(restartResult.value.discardedSteps).toBe(1); + + // The step belongs to a take that no longer exists: it must be reported as + // rejected, never as recorded. + expect(appendResult.status).toBe("rejected"); + const failure = + appendResult.status === "rejected" ? getFailureSignal(appendResult.reason) : undefined; + expect(failure?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(failure?.failure_stage).toBe("flow_session_superseded"); + + // The invariant: the new take's file is what the new take says it is, and + // carries nothing from the discarded one. + const session = getRecordingSession(root, "alpha"); + expect(session).toBeDefined(); + expect(session).not.toBe(discarded); + const onDisk = await readMarkers(root, "alpha"); + expect(onDisk).toEqual(markers(session!.flow.steps)); + expect(onDisk).not.toContain("echo:stray"); + + // …and the new take records from there as a fresh recording. + await addStep(root, "alpha", "a2"); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a2"]); + const finished = await finish(root, "alpha"); + expect(finished.steps).toBe(1); + }); }); // ── A finish landing on top of an in-flight append ─────────────────── From 611038241d90445b067f3fc812c9377d0c3e06bf Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 20:38:52 +0200 Subject: [PATCH 06/98] fix(flow): make the recording cap evict the least recently USED session The eviction backstop stamped sessions with Date.now(), which has millisecond resolution, so recordings touched inside one millisecond tied and the scan fell back to map insertion order. Observed directly: fill to the cap, touch the first-registered key, overflow -- and the session just touched is the one evicted while the untouched next-oldest survives. It takes 33 recordings inside a millisecond to hit, so this is a backstop edge rather than a live bug, but the entry it drops is the one whose steps would be stranded. A counter cannot tie, so least-recently-used now means that. The test pins LRU against FIFO, which the previous stamps could not distinguish. --- .../tool-server/src/tools/flows/flow-utils.ts | 29 ++++++++++++++----- .../tool-server/test/flows/flow-utils.test.ts | 21 ++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index c315aeec1..93ba063a9 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -202,8 +202,8 @@ export interface RecordingSession { filePath: string; /** In-memory flow content — authoritative in "client" mode. */ flow: FlowFile; - /** Wall-clock of the last touch, for the LRU eviction backstop. */ - lastTouchedAtMs: number; + /** Order of the last touch, for the LRU eviction backstop. See {@link touch}. */ + lastTouchedSeq: number; } /** @@ -275,13 +275,26 @@ export function withFlowFileLock( */ const MAX_RECORDINGS = 32; +/** + * Stamp a session as most-recently-used. A counter rather than `Date.now()`: + * wall-clock has millisecond resolution, so sessions touched inside one + * millisecond tie, and the eviction scan's tie-break is map insertion order — + * which can drop the session that was touched most recently while keeping one + * that was not touched at all. A counter cannot tie, so "least recently used" + * means exactly that. + */ +let touchSeq = 0; +function touch(): number { + return ++touchSeq; +} + function evictIfOverCapacity(): void { while (recordings.size > MAX_RECORDINGS) { let oldestKey: string | undefined; - let oldestAt = Infinity; + let oldestSeq = Infinity; for (const [key, session] of recordings) { - if (session.lastTouchedAtMs < oldestAt) { - oldestAt = session.lastTouchedAtMs; + if (session.lastTouchedSeq < oldestSeq) { + oldestSeq = session.lastTouchedSeq; oldestKey = key; } } @@ -307,7 +320,7 @@ export interface RecordingSessionInit { export function startRecordingSession(init: RecordingSessionInit): RecordingSession | null { const key = getFlowPath(init.projectRoot, init.name); const previous = recordings.get(key) ?? null; - recordings.set(key, { ...init, lastTouchedAtMs: Date.now() }); + recordings.set(key, { ...init, lastTouchedSeq: touch() }); evictIfOverCapacity(); return previous; } @@ -366,7 +379,7 @@ export function requireRecordingSession(projectRoot: string, name: string): Reco } ); } - session.lastTouchedAtMs = Date.now(); + session.lastTouchedSeq = touch(); return session; } @@ -2378,7 +2391,7 @@ export async function appendStepToFlow( ): Promise<{ flowFile: string; savedTo: FlowSavedTo }> { return withFlowFileLock(session.projectRoot, session.name, async () => { assertSessionStillLive(session); - session.lastTouchedAtMs = Date.now(); + session.lastTouchedSeq = touch(); if (session.persist === "host") { const flowFile = await appendStep(session.filePath, step); session.flow = parseFlow(flowFile); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 7affd6e03..1fef22113 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1146,6 +1146,27 @@ describe("recording sessions", () => { ); }); + it("evicts the least recently USED recording, not the oldest one", () => { + // The cap is a leak backstop, but which entry it drops matters: evicting a + // recording an agent is actively using would strand its steps. Fill past + // the cap, touching the first-registered key just before the overflow — it + // must survive and the untouched next-oldest must go. A FIFO eviction, or + // an LRU keyed on a millisecond clock (every one of these registers inside + // the same millisecond, so they would all tie), fails this. + const cap = 32; + for (let i = 0; i < cap; i++) start("/tmp/proj-a", `flow-${i}`); + expect(listActiveRecordings()).toHaveLength(cap); + + requireRecordingSession("/tmp/proj-a", "flow-0"); // now most-recently-used + start("/tmp/proj-a", "overflow"); + + const live = new Set(listActiveRecordings().map((r) => r.name)); + expect(live.size).toBe(cap); + expect(live.has("flow-0")).toBe(true); // touched, so kept + expect(live.has("flow-1")).toBe(false); // untouched and now the oldest use + expect(live.has("overflow")).toBe(true); + }); + it("listActiveRecordings reflects what is live", () => { expect(listActiveRecordings()).toEqual([]); start("/tmp/proj-a", "my-flow", { From af14e43a0819827438ad8d77bcdd2c9b6be22663 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 27 Jul 2026 20:59:27 +0200 Subject: [PATCH 07/98] fix(flow): stop a bare IP from claiming every wireless device at that address matchingDeviceId treated anything after a colon as the transport discriminator, but only NativeDevtools appends one (:tcp) and an adb serial over wifi is itself ip:port. So `devices: ["192.168.1.5"]` matched AndroidDevtools:192.168.1.5:5555 AND SimulatorServer:192.168.1.5:5556 -- tearing down a second agent's device and reporting nothing unmatched, which is the exact failure the devices scope exists to prevent. The suffix is now enumerated rather than wildcarded, so that id reports itself unmatched and disposes nothing, while the full serial and the :tcp form still match. --- .../simulator/stop-all-simulator-servers.ts | 20 +++++++++++++------ packages/tool-server/test/stop-tools.test.ts | 17 ++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 984a95a36..24bb83ff9 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -33,11 +33,19 @@ const zodSchema = z.object({ }); /** - * Which entry of `deviceIds` owns `urn`, if any. Every URN in {@link PREFIXES} - * is `:`, optionally with a trailing transport - * discriminator (`NativeDevtools::tcp`). Device ids can themselves contain - * a colon (a wireless-adb serial is `192.168.1.5:5555`), so the tail is compared - * whole rather than split on ":". + * The only discriminator any URN in {@link PREFIXES} appends after the device id + * (`NativeDevtools::tcp` — every other namespace is a bare + * `:`). Enumerated rather than matched as "anything after + * a colon", because a device id can itself end in `:`: an adb serial + * over wifi is `192.168.1.5:5555`, so a suffix wildcard would let the bare + * `192.168.1.5` claim every device at that address and tear down another + * agent's — while reporting nothing unmatched. + */ +const URN_SUFFIXES = ["", ":tcp"] as const; + +/** + * Which entry of `deviceIds` owns `urn`, if any. The tail after the namespace + * is compared whole (never split on ":", see {@link URN_SUFFIXES}). * * Returns the caller's spelling of the id so the tool can report which requested * ids matched nothing. @@ -52,7 +60,7 @@ function matchingDeviceId(urn: string, deviceIds: string[]): string | undefined const tail = urn.slice(prefix.length).toLowerCase(); return deviceIds.find((id) => { const lower = id.toLowerCase(); - return tail === lower || tail.startsWith(`${lower}:`); + return URN_SUFFIXES.some((suffix) => tail === `${lower}${suffix}`); }); } diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 13441acf7..c8d3f8ee6 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -348,6 +348,23 @@ describe("stop-all-simulator-servers device scoping", () => { expect(registry.disposeService).toHaveBeenCalledOnce(); }); + it("does not let a bare IP claim every wireless device at that address", async () => { + // An adb serial is `ip:port`, so treating "anything after a colon" as the + // transport discriminator would let a caller who dropped the port tear down + // a second agent's device — and report nothing unmatched while doing it. + const services = new Map([ + ["AndroidDevtools:192.168.1.5:5555", { state: ServiceState.RUNNING, dependents: [] }], + ["SimulatorServer:192.168.1.5:5556", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: ["192.168.1.5"] }); + + expect(result).toEqual({ stopped: [], unmatched: ["192.168.1.5"] }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + it("matches the device id case-insensitively", async () => { // iOS UDIDs are conventionally upper-case, but an agent passes through // whatever it was handed — a case mismatch must not silently no-op. From 6f9adc23169325372230c9f128a4b43e20c6f023 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 28 Jul 2026 11:24:52 +0200 Subject: [PATCH 08/98] test(flow): make the eviction test prove recency, and drop the inert clock mock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LRU case filled the table, touched everything but `rec-0`, and asserted `rec-0` was evicted — but `rec-0` was also the first-registered key, so an insertion-order eviction picks the same victim and the assertion passes under either policy. It now leaves an entry in the middle of the table untouched, so the least-recently-used key and the first-registered one differ; degrading `touch()` to a constant fails it, which it did not before. `useMonotonicClock` dated from when the backstop stamped with `Date.now()`. The eviction path reads `lastTouchedSeq` from a counter now and never calls the wall clock, so the spy had no effect on either case it wrapped. --- .../flows/flow-concurrent-recording.test.ts | 108 ++++++++---------- 1 file changed, 47 insertions(+), 61 deletions(-) diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index de276e46e..88639f0aa 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -197,17 +197,6 @@ async function within(promise: Promise, label: string, ms = 2000): Promise } } -/** - * The LRU backstop compares `Date.now()` stamps, and dozens of recordings can be - * started and touched inside one millisecond — so drive its clock explicitly - * rather than depending on wall-clock resolution. - */ -function useMonotonicClock(): { restore: () => void } { - let ticks = Date.now(); - const spy = vi.spyOn(Date, "now").mockImplementation(() => ++ticks); - return { restore: () => spy.mockRestore() }; -} - /** Fill the recording table exactly to its cap; returns the names, oldest first. */ async function fillRecordings(root: string): Promise { const names = Array.from({ length: MAX_RECORDINGS }, (_, i) => `rec-${i}`); @@ -892,63 +881,60 @@ describe("a finish on a flow file that no longer parses", () => { describe("the concurrent-recording cap", () => { it("evicts the least recently touched recording and keeps the rest", async () => { const root = await makeRoot("evict"); - const clock = useMonotonicClock(); - try { - const names = await fillRecordings(root); - expect(listActiveRecordings()).toHaveLength(MAX_RECORDINGS); - - // Touch everything except the first, so `rec-0` is unambiguously the - // least recently touched recording. - for (const name of names.slice(1)) await addEcho(root, name, "touch"); - - await start(root, "overflow"); - - const live = listActiveRecordings() - .map((r) => r.name) - .sort(); - expect(live).toHaveLength(MAX_RECORDINGS); - expect(live).toEqual([...names.slice(1), "overflow"].sort()); - expect(getRecordingSession(root, "rec-0")).toBeUndefined(); - // The survivors are still usable — eviction dropped one, not the table. - expect(getRecordingSession(root, names[1])).toBeDefined(); - await addEcho(root, names[1], "still-live"); - expect(await readMarkers(root, names[1])).toEqual(["echo:touch", "echo:still-live"]); - } finally { - clock.restore(); - } + const names = await fillRecordings(root); + expect(listActiveRecordings()).toHaveLength(MAX_RECORDINGS); + + // Touch everything EXCEPT one entry in the middle of the table, so the + // least-recently-used entry and the first-registered one are different + // keys: `rec-7` is the only one never touched since it was started, while + // `rec-0` was registered first but has since been used. An insertion-order + // eviction would drop `rec-0`, so the assertions below separate the two + // policies rather than passing under either. + const untouched = names[7]; + for (const name of names.filter((n) => n !== untouched)) await addEcho(root, name, "touch"); + + await start(root, "overflow"); + + const live = listActiveRecordings() + .map((r) => r.name) + .sort(); + expect(live).toHaveLength(MAX_RECORDINGS); + expect(live).toEqual([...names.filter((n) => n !== untouched), "overflow"].sort()); + expect(getRecordingSession(root, untouched)).toBeUndefined(); + // The oldest registration survived, because it was still being used. + expect(getRecordingSession(root, names[0])).toBeDefined(); + // The survivors are still usable — eviction dropped one, not the table. + await addEcho(root, names[0], "still-live"); + expect(await readMarkers(root, names[0])).toEqual(["echo:touch", "echo:still-live"]); }); it("rejects an append whose recording was evicted while the step ran", async () => { const root = await makeRoot("evict-inflight"); - const clock = useMonotonicClock(); - try { - const names = await fillRecordings(root); + const names = await fillRecordings(root); - // The step resolves rec-0's session (touching it) and parks. - const gate = gateNextSubTool(); - const appending = addStep(root, "rec-0", "victim"); - await gate.reached; + // The step resolves rec-0's session (touching it) and parks. + const gate = gateNextSubTool(); + const appending = addStep(root, "rec-0", "victim"); + await gate.reached; - // Every other recording is touched, then one more overflows the cap — - // rec-0 is now the LRU and gets dropped out from under the running step. - for (const name of names.slice(1)) await addEcho(root, name, "touch"); - await start(root, "overflow"); - expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + // Every other recording is touched afterwards, so rec-0's use is the oldest + // one on the table; the next start overflows the cap and drops it out from + // under the running step. + for (const name of names.slice(1)) await addEcho(root, name, "touch"); + await start(root, "overflow"); + expect(getRecordingSession(root, "rec-0")).toBeUndefined(); - gate.release(); - const err = await captureFailure(appending); - expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); - expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); - expect((err as Error).message).toContain("concurrent-recording cap"); - expect(await readMarkers(root, "rec-0")).toEqual([]); - - // A fresh call on the evicted key fails the ordinary not-live way. - const late = await captureFailure(addEcho(root, "rec-0", "late")); - expect(getFailureSignal(late)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); - expect(getFailureSignal(late)?.failure_stage).toBe("flow_require_recording"); - } finally { - clock.restore(); - } + gate.release(); + const err = await captureFailure(appending); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); + expect((err as Error).message).toContain("concurrent-recording cap"); + expect(await readMarkers(root, "rec-0")).toEqual([]); + + // A fresh call on the evicted key fails the ordinary not-live way. + const late = await captureFailure(addEcho(root, "rec-0", "late")); + expect(getFailureSignal(late)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(getFailureSignal(late)?.failure_stage).toBe("flow_require_recording"); }); }); From 746585e9a49d8679ac42202c400ffeb957d0a034 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 28 Jul 2026 11:44:16 +0200 Subject: [PATCH 09/98] fix(flow): stop the superseded-step error from advising a destructive restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assertSessionStillLive` closed with "Call flow-start-recording and re-record the step." That truncates unconditionally, so every branch of the message sent the agent at something worth keeping: on "restarted", the live take that had just claimed the key (which restarting both wipes and steals back); on "finished", the completed flow now sitting on disk. This is the defect 38995cad fixed one file over, still spelled out in the error the other site emits. It now points at a fresh name, and says the step already ran on the device — `flow-add-step` invokes the sub-tool before it appends, so "nothing was recorded" was true of the file and false of the phone, and an agent re-issuing a tap on "Place order" would order twice. `flow-add-echo` runs nothing live and gets the shorter wording. Two more from the same sweep: `summarizeSteps` stringified a tool step's `args` unguarded. That is the one step body the parser does not constrain, so a cyclic YAML anchor in a hand-edited file — a documented workflow — reached it as a cyclic object and threw a raw TypeError naming neither the flow nor the step. `parseFlow` already falls back to a marker for the same input class; do the same here. `captureRunTarget` resolved the `run:` target by name alone, so running project B's `login` while recording in project A recorded `run: login` and replayed A's copy — a different file than the one that ran, silently. Generic fragment names are exactly what collides now that recordings span projects. Still recorded as composition, but the substitution is stated; the resolved-target branch was also dropping warnings on the floor rather than surfacing them. --- .../skills/skills/argent-create-flow/SKILL.md | 4 +- .../src/tools/flows/flow-add-step.ts | 4 ++ .../src/tools/flows/flow-finish-recording.ts | 27 +++++-- .../tool-server/src/tools/flows/flow-utils.ts | 18 ++++- .../flows/flow-concurrent-recording.test.ts | 71 ++++++++++++++++++- 5 files changed, 115 insertions(+), 9 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 9753d1e29..6edb48a46 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -141,7 +141,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. - **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. -- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. Either way, restart under a fresh name instead of re-adding the step. +- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. - **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...`. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. @@ -150,9 +150,11 @@ Every tool during recording returns the current flow file contents, so you can t The `command` parameter is the MCP tool name; `args` is a **JSON string** (not an object), omitted entirely for tools with no arguments: ``` +name: "checkout-e2e" project_root: "/Users/dev/MyApp" command: "gesture-tap" args: "{\"udid\": \"\", \"x\": 0.5, \"y\": 0.35}" +name: "checkout-e2e" project_root: "/Users/dev/MyApp" command: "await-ui-element" args: "{\"udid\": \"\", \"condition\": \"visible\", \"selector\": {\"text\": \"Continue\"}}" ``` diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index 7ea75f785..01ebd2b78 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -526,6 +526,10 @@ If a step was recorded by mistake, edit the .yaml file directly to remove it.`, step = { kind: "launch", app: strippedArgs.bundleId as string }; } else if (runTarget?.flow) { step = { kind: "run", flow: runTarget.flow }; + // A resolved target can still carry a warning (a same-named sibling in + // another project), so this branch surfaces it too — not only the + // kept-the-raw-step one below. + warning = runTarget.warning; } else { warning = runTarget?.warning; // The step ran live with the full args (incl. the device id), but the diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 4a5b45e14..97f7fd376 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -120,9 +120,11 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps // flow-start-recording, truncates the take it would be recovering). const flow = parseFlow(flowFile); // Render the summary before clearing too, for the same reason: it walks - // step bodies the parser does not fully constrain (`JSON.stringify` on - // a tool step's args can throw on a cyclic YAML anchor), and nothing - // that can throw may run after the session is destroyed. + // step bodies the parser does not fully constrain, and nothing that can + // throw may run after the session is destroyed. The one known thrower + // there — `JSON.stringify` on a cyclic `args` anchor — is guarded in + // {@link renderToolArgs}; keeping the order is what makes the next one + // recoverable rather than fatal. const summary = summarizeSteps(flow); clearRecordingSession(params.project_root, params.name); return { filePath, flowFile, savedTo, flow, summary }; @@ -141,6 +143,23 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps }, }; +/** + * A `tool:` step's `args` is the one step body the parser does not constrain, so + * a cyclic YAML alias in a hand-edited file reaches here as a cyclic object and + * `JSON.stringify` throws on it. Fall back to a marker, the way `parseFlow` + * already does for the same input class (see `badEntry` in flow-utils) — the + * summary of a recording that is otherwise fine should not fail on one + * unrenderable step. Interpolation matches the previous inline spelling, so an + * absent `args` still renders "undefined". + */ +function renderToolArgs(args: unknown): string { + try { + return `${JSON.stringify(args)}`; + } catch { + return "[cyclic args]"; + } +} + /** One human-readable line per recorded step, in the flow file's own spellings. */ function summarizeSteps(flow: FlowFile): string[] { return flow.steps.map((step, i) => { @@ -194,7 +213,7 @@ function summarizeSteps(flow: FlowFile): string[] { return `${n}. snapshot: ${step.name}`; case "tool": default: - return `${n}. tool: ${step.name} ${JSON.stringify(step.args)}`; + return `${n}. tool: ${step.name} ${renderToolArgs(step.args)}`; } }); } diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 93ba063a9..155a7d11b 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2357,7 +2357,7 @@ export type FlowSavedTo = string | ClientFileDirective; * is told it succeeded. Re-check identity at write time — inside the flow-file * lock, so the check sees the state the write will see — and fail loudly. */ -function assertSessionStillLive(session: RecordingSession): void { +function assertSessionStillLive(session: RecordingSession, step: FlowStep): void { const current = recordings.get(getFlowPath(session.projectRoot, session.name)); if (current === session) return; // A key that is occupied by a DIFFERENT session was restarted; an empty key @@ -2366,9 +2366,21 @@ function assertSessionStillLive(session: RecordingSession): void { const why = current ? "it was restarted while this step was running, so the step belongs to the discarded take" : "it was finished (or dropped by the concurrent-recording cap) while this step was running"; + // Do NOT send the agent to flow-start-recording here. It truncates + // unconditionally, and on every branch there is now something to lose: the + // live take that just claimed this key (which restarting would both wipe and + // steal), or the finished flow sitting on disk. Recording under a fresh name + // is the only recovery that destroys nothing. + const recovery = + `Nothing was added to the flow file` + + (step.kind === "echo" + ? ". " + : ", but the step itself already ran on the device — repeating it repeats that action. ") + + `This key now belongs to another take and flow-start-recording truncates, so re-record ` + + `under a fresh name rather than restarting this one.`; throw new FailureError( `Recording of "${session.name}" in ${session.projectRoot} is no longer active — ${why}. ` + - `Nothing was recorded. Call flow-start-recording and re-record the step.`, + recovery, { error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, failure_stage: "flow_session_superseded", @@ -2390,7 +2402,7 @@ export async function appendStepToFlow( step: FlowStep ): Promise<{ flowFile: string; savedTo: FlowSavedTo }> { return withFlowFileLock(session.projectRoot, session.name, async () => { - assertSessionStillLive(session); + assertSessionStillLive(session, step); session.lastTouchedSeq = touch(); if (session.persist === "host") { const flowFile = await appendStep(session.filePath, step); diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 88639f0aa..35599a7d1 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -577,7 +577,15 @@ describe("a restart that lands while a step is still running", () => { expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); expect((err as Error).message).toContain("restarted while this step was running"); - expect((err as Error).message).toContain("Nothing was recorded"); + expect((err as Error).message).toContain("Nothing was added to the flow file"); + // The recovery advice must not send the agent back to flow-start-recording: + // it truncates, so on this branch it would wipe the live take that just took + // the key (and take it back again). A fresh name is the only safe recovery. + expect((err as Error).message).toContain("fresh name"); + expect((err as Error).message).not.toMatch(/Call flow-start-recording/); + // The step already ran live before the append was rejected, so an agent that + // simply retries it would repeat the device action. + expect((err as Error).message).toContain("already ran on the device"); // The new take is empty — no step from the discarded one leaked into it. expect(await readMarkers(root, "alpha")).toEqual([]); @@ -988,4 +996,65 @@ describe("recording a flow-execute step while several projects are in play", () expect(res.message).not.toContain("kept the raw flow-execute step"); expect(await readSteps(recordingRoot, "wrapper")).toEqual([{ kind: "run", flow: "helper" }]); }); + + it("warns when a same-named fragment exists in BOTH projects", async () => { + const recordingRoot = await makeRoot("run-target-both"); + const executedRoot = await makeRoot("run-target-both-other"); + + // The ambiguous case concurrent recording makes routine: a generic fragment + // name that exists in two projects. `run: helper` resolves against the + // recording, so replay runs a DIFFERENT file than the one that just ran. + await writeSavedFlow(recordingRoot, "helper", fragment); + await writeSavedFlow(executedRoot, "helper", { + executionPrerequisite: "", + steps: [{ kind: "echo", message: "the other project's helper" }], + }); + + await start(recordingRoot, "wrapper"); + const res = await addRawStep(recordingRoot, "wrapper", "flow-execute", { + name: "helper", + project_root: executedRoot, + udid: IOS_DEVICE, + }); + + // Still recorded as composition — that is what `run:` means — but the + // substitution is stated rather than silent. + expect(await readSteps(recordingRoot, "wrapper")).toEqual([{ kind: "run", flow: "helper" }]); + expect(res.message).toContain("replays THIS project's helper.yaml"); + expect(res.message).toContain(executedRoot); + + // Same project on both sides is the unambiguous case and stays quiet. + await start(recordingRoot, "quiet"); + const same = await addRawStep(recordingRoot, "quiet", "flow-execute", { + name: "helper", + project_root: recordingRoot, + udid: IOS_DEVICE, + }); + expect(same.message).toBe('Step added to "quiet" flow'); + }); +}); + +// ── Summarizing a hand-edited file that the parser cannot fully constrain ── + +describe("finishing a recording whose YAML was hand-edited into an unrenderable step", () => { + it("summarizes a cyclic tool-args anchor instead of throwing", async () => { + const root = await makeRoot("cyclic-args"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + // Hand-editing mid-recording is a documented workflow, and `args:` is the + // one step body the parser does not constrain — a cyclic YAML anchor + // reaches the summarizer as a cyclic object, which JSON.stringify throws on. + await fs.writeFile( + flowPath(root, "alpha"), + 'executionPrerequisite: ""\nsteps:\n - tool: keyboard\n args: &a\n self: *a\n', + "utf8" + ); + + const finished = await finish(root, "alpha"); + expect(finished.steps).toBe(1); + expect(finished.summary).toEqual(["1. tool: keyboard [cyclic args]"]); + // The recording is properly closed, not left dangling by a thrown summary. + expect(getRecordingSession(root, "alpha")).toBeUndefined(); + }); }); From dc7722bf963f6bf4469bc0ab1ece39e50ff0bad2 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 28 Jul 2026 11:52:11 +0200 Subject: [PATCH 10/98] test(flow): pin the lock's self-cleanup and the append's disk re-read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation sweep over the new suite found two guarantees that no test held. Replacing `if (flowFileLocks.get(key) === held) delete` with an unconditional delete passed all 639 tests. It is not harmless: once the first holder finishes while the second is still inside its critical section, the key is gone from the map, so a third acquirer finds no predecessor and runs concurrently with the second — the lost update the lock exists to prevent. Every existing case used two parties on two different files, which is exactly where the identity guard does not matter. Now pinned three-deep on one key. Serializing the in-memory take on a host-mode append instead of re-reading the file also passed everything, and silently resurrects a step the agent had just hand-deleted. Editing the .yaml mid-recording is what both tools' descriptions tell the agent to do, and the finish path's re-read is covered while the append path's was not. Also softened the LRU comment in flow-utils.test.ts. It claimed the case fails an LRU keyed on a millisecond clock; that only holds when the fill and the touch land in one millisecond, which is true for the file alone and false under full-suite load — it survived 2 of 4 runs. FIFO is what this case rules out deterministically, so that is what it now claims. --- .../flows/flow-concurrent-recording.test.ts | 74 +++++++++++++++++++ .../tool-server/test/flows/flow-utils.test.ts | 10 ++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 35599a7d1..4ee08ba21 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -444,6 +444,80 @@ describe("the flow-file lock", () => { expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); expect(await readMarkers(root, "beta")).toEqual(["tool:b1"]); }); + + it("still excludes a third acquirer after the first one has released", async () => { + const root = await makeRoot("lock-three-deep"); + + // Three acquirers on ONE key, which is where the lock map's self-cleanup + // has to be careful: the entry may only be dropped by the holder that is + // still the tail. Deleting it unconditionally looks harmless — every + // two-party test still passes — but once A finishes while B is holding, the + // key is gone from the map, so C finds no predecessor to queue behind and + // runs *concurrently with B*. That is the lost update this lock exists to + // prevent, so pin three-deep contention explicitly. + const order: string[] = []; + const gateA = openGate(); + const gateB = openGate(); + + const heldA = withFlowFileLock(root, "alpha", async () => { + order.push("a-enter"); + await gateA.promise; + order.push("a-exit"); + }); + const heldB = withFlowFileLock(root, "alpha", async () => { + order.push("b-enter"); + await gateB.promise; + order.push("b-exit"); + }); + + // A is holding, B is queued. Release A so B takes the lock and A's cleanup + // runs while B is still inside its critical section. + gateA.open(); + await heldA; + await settle(); + expect(order).toEqual(["a-enter", "a-exit", "b-enter"]); + + // C arrives now — after A's cleanup, while B holds. + const heldC = withFlowFileLock(root, "alpha", async () => { + order.push("c-enter"); + }); + expect(await within(heldC, "c-done", 200)).toBe("timed-out"); + expect(order).toEqual(["a-enter", "a-exit", "b-enter"]); + + gateB.open(); + await heldB; + await heldC; + expect(order).toEqual(["a-enter", "a-exit", "b-enter", "b-exit", "c-enter"]); + }); +}); + +// ── The append path's source of truth ──────────────────────────────── + +describe("appending to a recording whose file was hand-edited", () => { + it("re-reads the file, so an edit made mid-recording survives the next append", async () => { + const root = await makeRoot("append-rereads"); + await start(root, "alpha"); + await addEcho(root, "alpha", "s1"); + await addEcho(root, "alpha", "s2"); + expect(await readMarkers(root, "alpha")).toEqual(["echo:s1", "echo:s2"]); + + // Removing a bad step by editing the .yaml is what both recording tools' + // descriptions tell the agent to do. It only survives because the host-mode + // append re-reads from disk; serializing the in-memory copy instead would + // silently resurrect the deleted step on the very next append. + await fs.writeFile( + flowPath(root, "alpha"), + 'executionPrerequisite: ""\nsteps:\n - echo: s2\n', + "utf8" + ); + + await addEcho(root, "alpha", "s3"); + expect(await readMarkers(root, "alpha")).toEqual(["echo:s2", "echo:s3"]); + + // …and the finish reports the file, not the take as it was recorded. + const finished = await finish(root, "alpha"); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["echo:s2", "echo:s3"]); + }); }); // ── Replaying a flow while recordings are live ─────────────────────── diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 1fef22113..f0389d796 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1150,9 +1150,13 @@ describe("recording sessions", () => { // The cap is a leak backstop, but which entry it drops matters: evicting a // recording an agent is actively using would strand its steps. Fill past // the cap, touching the first-registered key just before the overflow — it - // must survive and the untouched next-oldest must go. A FIFO eviction, or - // an LRU keyed on a millisecond clock (every one of these registers inside - // the same millisecond, so they would all tie), fails this. + // must survive and the untouched next-oldest must go. A FIFO eviction fails + // this deterministically. + // + // It does NOT reliably catch an LRU keyed on a millisecond clock: that only + // ties when the whole fill and the touch land inside one millisecond, which + // holds when this file runs alone but not under full-suite load. The + // counter's tie-freedom is argued at `touch()` rather than pinned here. const cap = 32; for (let i = 0; i < cap; i++) start("/tmp/proj-a", `flow-${i}`); expect(listActiveRecordings()).toHaveLength(cap); From 5bc82cf6c6ee2e1032b26c4c183b686a4ad90acb Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 28 Jul 2026 12:10:49 +0200 Subject: [PATCH 11/98] =?UTF-8?q?fix(flow):=20make=20flow-file=20writes=20?= =?UTF-8?q?atomic,=20and=20share=20the=20device=E2=86=92services=20matcher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from review. The flow-file lock serializes writers, but every reader stays outside it — flow-execute's own load, its `run:` fragment load, flow-read-prerequisite, flow-add-step's sibling check, and the CLI reading from another process, where an in-process lock cannot reach. `appendStep` and flow-start-recording both persisted with a plain `fs.writeFile`, which opens O_TRUNC, so a reader could land between the truncate and the write. That case is silent: parseFlow("") returns { steps: [] } with no error, and summarize derives `ok` from "no failures", so a flow-execute racing an append reports a top-level PASS having replayed zero steps. Both writes now go through a sibling temp file and a rename, so a reader sees either the whole old file or the whole new one. The added reader test observes zero-step reads against the old write and none against the new one. `unmatched` diagnosed real device ids as typos whenever their services lived outside PREFIXES. An iOS session that only ran boot/launch/describe owns `AXService:` and nothing else, so the mandated session-end call both left the in-sim ax daemon (spawned --timeout 3600) running and reported a correct UDID as a mistyped one. AXService joins the namespace set — its `:tcp` shape was already covered — and the description now admits that a serviceless device (Vega, driven entirely by CLI/adb shell-outs) lands in `unmatched` legitimately. The device→services mapping had become two implementations that disagreed: stop-simulator-server looked URNs up with an exact case-sensitive `services.get()` and no `:tcp` handling, so a lower-cased UDID silently no-op'd there while the scoped stop-all reaped it. Both now share one matcher in device-services.ts. The namespace sets stay deliberately different, documented where they are defined: stop-simulator-server is the recovery path for a wedged transport, and widening it to devtools/AX would make a routine retry drop the native-devtools connection another agent's recording depends on. --- .../src/tools/flows/flow-start-recording.ts | 6 +- .../tool-server/src/tools/flows/flow-utils.ts | 54 ++++++++- .../src/tools/simulator/device-services.ts | 105 +++++++++++++++++ .../simulator/stop-all-simulator-servers.ts | 92 +++++---------- .../tools/simulator/stop-simulator-server.ts | 31 +++-- .../flows/flow-concurrent-recording.test.ts | 106 ++++++++++++++++++ packages/tool-server/test/stop-tools.test.ts | 104 +++++++++++++++++ 7 files changed, 410 insertions(+), 88 deletions(-) create mode 100644 packages/tool-server/src/tools/simulator/device-services.ts diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index b274d388c..3bb55d105 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -1,11 +1,10 @@ import { z } from "zod"; -import * as fs from "node:fs/promises"; import type { FileInputSpec, ToolDefinition } from "@argent/registry"; -import * as path from "node:path"; import { getFlowPath, startRecordingSession, withFlowFileLock, + writeNewFlowFile, clientFileDirective, serializeFlow, validateFlow, @@ -114,8 +113,7 @@ to remove or reorder steps.`, async () => { let savedTo: FlowSavedTo; if (persist === "host") { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, flowFile, "utf8"); + await writeNewFlowFile(filePath, flowFile); savedTo = filePath; } else { savedTo = clientFileDirective(filePath, flowFile); diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 155a7d11b..6b7bccc41 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2322,6 +2322,58 @@ export function parseFlow(content: string): FlowFile { // ── File helpers ───────────────────────────────────────────────────── +/** + * Suffix counter for {@link writeFlowFile}'s scratch file. Paired with the pid, + * this keeps two concurrent writers — in this process or in a CLI sharing the + * directory — off each other's temp file. + */ +let flowWriteSeq = 0; + +/** + * Replace a flow file's contents so no reader can ever observe it half-written. + * + * {@link withFlowFileLock} serializes WRITERS, but every reader of a flow YAML + * stays outside it — `flow-execute`'s own load, its `run:` fragment load, + * `flow-read-prerequisite`, `flow-add-step`'s sibling-fragment check — and the + * `argent` CLI reads these files from another process entirely, where an + * in-process lock cannot reach. A plain `fs.writeFile` opens with O_TRUNC, so + * such a reader could land in the window between the truncate and the write and + * parse a truncated file, or an empty one — and `parseFlow("")` yields + * `{ steps: [] }` with no error, which replays as a top-level PASS over zero + * steps. + * + * Writing to a sibling temp file and renaming makes the swap atomic: a reader + * sees either the whole previous file or the whole new one. The temp name is + * dotted and `.tmp`-suffixed so a half-written scratch file can never be + * mistaken for a flow (`getFlowPath` only ever produces `.yaml`, and + * nothing enumerates the flows directory). + */ +async function writeFlowFile(filePath: string, content: string): Promise { + const tmpPath = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${++flowWriteSeq}.tmp` + ); + await fs.writeFile(tmpPath, content, "utf8"); + try { + // Atomic within a filesystem, and the temp file is a sibling of the target, + // so it is always the same one. + await fs.rename(tmpPath, filePath); + } catch (err) { + // Leave no scratch file behind on a failed swap (e.g. a read-only dir). + await fs.rm(tmpPath, { force: true }).catch(() => {}); + throw err; + } +} + +/** + * Create or reset a flow file with `content`, making the parent directory if + * needed. Atomic (see {@link writeFlowFile}). + */ +export async function writeNewFlowFile(filePath: string, content: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await writeFlowFile(filePath, content); +} + /** Read and parse the flow file, append a step, write it back. */ export async function appendStep(filePath: string, step: FlowStep): Promise { const content = await fs.readFile(filePath, "utf8"); @@ -2332,7 +2384,7 @@ export async function appendStep(filePath: string, step: FlowStep): Promise:tcp` and `AXService::tcp`; every other URN in + * {@link DEVICE_OWNED_NAMESPACES} is a bare `:`). + * Enumerated rather than matched as "anything after a colon", because a device + * id can itself end in `:`: an adb serial over wifi is + * `192.168.1.5:5555`, so a suffix wildcard would let the bare `192.168.1.5` + * claim every device at that address and tear down another agent's — while + * reporting nothing unmatched. + */ +export const URN_SUFFIXES = ["", ":tcp"] as const; + +/** + * Every namespace whose service belongs to exactly one device, and whose + * `dispose()` frees something worth freeing. A device owning none of these is + * not a bad id — Vega is driven entirely by CLI/adb shell-outs and registers no + * service at all. + * + * `AXService` is here because its `dispose()` is the only thing that reaps the + * in-sim ax daemon (spawned `--timeout 3600`) and unlinks its socket; nothing + * cascades from `SimulatorServer`, so an iOS session that only ran + * boot/launch/describe owns this and nothing else. `TvControl` likewise owns two + * spawned `--timeout 3600` daemons. (`AndroidTvControl` is stateless adb + * shell-outs with a no-op dispose, but is included for symmetry so the snapshot + * is fully drained.) + */ +export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ + SIMULATOR_SERVER_NAMESPACE, + NATIVE_DEVTOOLS_NAMESPACE, + ANDROID_DEVTOOLS_NAMESPACE, + CHROMIUM_CDP_NAMESPACE, + TV_CONTROL_NAMESPACE, + ANDROID_TV_CONTROL_NAMESPACE, + AX_SERVICE_NAMESPACE, +]; + +/** + * The subset `stop-simulator-server` disposes: the device's transport session, + * plus the TV-control daemons a tvOS udid may own alongside it. + * + * Deliberately narrower than {@link DEVICE_OWNED_NAMESPACES}. That tool is also + * the documented recovery for a wedged transport ("stop it and retry"), and + * widening it to devtools/AX would make a routine retry silently drop the + * native-devtools connection another agent's in-progress recording depends on — + * degrading that flow to coordinate taps, which is the exact hazard + * `stop-all-simulator-servers`' `devices` scope exists to prevent. Agents + * finishing a session call `stop-all-simulator-servers` instead, which drains + * everything. + */ +export function transportNamespacesForPlatform(platform: string): readonly string[] { + if (platform === "chromium") return [CHROMIUM_CDP_NAMESPACE]; + if (platform === "android") return [SIMULATOR_SERVER_NAMESPACE, ANDROID_TV_CONTROL_NAMESPACE]; + // A tvOS UDID is iOS-shaped and can't be told apart from a phone here without + // an async probe, so cover both. + return [SIMULATOR_SERVER_NAMESPACE, TV_CONTROL_NAMESPACE]; +} + +/** + * Which entry of `deviceIds` owns `urn` within `namespaces`, if any. The tail + * after the namespace is compared whole (never split on ":", see + * {@link URN_SUFFIXES}). + * + * Matching is case-insensitive: iOS UDIDs are conventionally upper-case but + * agents pass through whatever they were given, and a case mismatch must not + * silently turn a scoped stop into a no-op. No two distinct devices can differ + * only by case in any id space we support (UUID, emulator-N, chromium-cdp-N). + * + * Returns the caller's spelling of the id, so a tool can report which of the + * ids it was given matched nothing. + */ +export function deviceIdOwningUrn( + urn: string, + namespaces: readonly string[], + deviceIds: readonly string[] +): string | undefined { + const namespace = namespaces.find((ns) => urn.startsWith(`${ns}:`)); + if (namespace === undefined) return undefined; + const tail = urn.slice(namespace.length + 1).toLowerCase(); + return deviceIds.find((id) => { + const lower = id.toLowerCase(); + return URN_SUFFIXES.some((suffix) => tail === `${lower}${suffix}`); + }); +} + +/** Whether `urn` belongs to any of `namespaces`, regardless of which device. */ +export function isDeviceServiceUrn(urn: string, namespaces: readonly string[]): boolean { + return namespaces.some((ns) => urn.startsWith(`${ns}:`)); +} diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 24bb83ff9..805e031ef 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -1,27 +1,7 @@ import { z } from "zod"; import { ServiceState, isLiveServiceState } from "@argent/registry"; import type { Registry, ToolDefinition } from "@argent/registry"; -import { SIMULATOR_SERVER_NAMESPACE } from "../../blueprints/simulator-server"; -import { NATIVE_DEVTOOLS_NAMESPACE } from "../../blueprints/native-devtools"; -import { ANDROID_DEVTOOLS_NAMESPACE } from "../../blueprints/android-devtools"; -import { CHROMIUM_CDP_NAMESPACE } from "../../blueprints/chromium-cdp"; -import { TV_CONTROL_NAMESPACE } from "../../blueprints/tv-control"; -import { ANDROID_TV_CONTROL_NAMESPACE } from "../../blueprints/android-tv-control"; - -const PREFIXES = [ - `${SIMULATOR_SERVER_NAMESPACE}:`, - `${NATIVE_DEVTOOLS_NAMESPACE}:`, - `${ANDROID_DEVTOOLS_NAMESPACE}:`, - `${CHROMIUM_CDP_NAMESPACE}:`, - // The Apple TV service owns two spawned daemons (in-sim tvos-ax-service + - // host-side tvos-hid-daemon, both --timeout 3600); only its dispose() reaps - // them and unlinks the sockets. Without this prefix a session-end stop leaves - // them running for up to an hour. (AndroidTvControl is stateless adb shell-outs - // with a no-op dispose, but include it for symmetry so the snapshot is fully - // drained.) - `${TV_CONTROL_NAMESPACE}:`, - `${ANDROID_TV_CONTROL_NAMESPACE}:`, -]; +import { DEVICE_OWNED_NAMESPACES, deviceIdOwningUrn, isDeviceServiceUrn } from "./device-services"; const zodSchema = z.object({ devices: z @@ -32,38 +12,6 @@ const zodSchema = z.object({ ), }); -/** - * The only discriminator any URN in {@link PREFIXES} appends after the device id - * (`NativeDevtools::tcp` — every other namespace is a bare - * `:`). Enumerated rather than matched as "anything after - * a colon", because a device id can itself end in `:`: an adb serial - * over wifi is `192.168.1.5:5555`, so a suffix wildcard would let the bare - * `192.168.1.5` claim every device at that address and tear down another - * agent's — while reporting nothing unmatched. - */ -const URN_SUFFIXES = ["", ":tcp"] as const; - -/** - * Which entry of `deviceIds` owns `urn`, if any. The tail after the namespace - * is compared whole (never split on ":", see {@link URN_SUFFIXES}). - * - * Returns the caller's spelling of the id so the tool can report which requested - * ids matched nothing. - */ -function matchingDeviceId(urn: string, deviceIds: string[]): string | undefined { - const prefix = PREFIXES.find((p) => urn.startsWith(p)); - if (!prefix) return undefined; - // Case-insensitive: iOS UDIDs are conventionally upper-case but agents pass - // through whatever they were given, and a case mismatch must not silently - // widen a scoped stop into a no-op. No two distinct devices can differ only - // by case in any id space we support (UUID, emulator-N, chromium-cdp-N). - const tail = urn.slice(prefix.length).toLowerCase(); - return deviceIds.find((id) => { - const lower = id.toLowerCase(); - return URN_SUFFIXES.some((suffix) => tail === `${lower}${suffix}`); - }); -} - export function createStopAllSimulatorServersTool( registry: Registry ): ToolDefinition, { stopped: string[]; unmatched?: string[] }> { @@ -79,14 +27,24 @@ export function createStopAllSimulatorServersTool( ? `Stopping simulator servers for ${devices.length} ${devices.length === 1 ? "device" : "devices"}` : "Stopping all simulator servers"; }, - completedMsg: ({ result }) => - `Stopped ${result.stopped.length} simulator ${result.stopped.length === 1 ? "server" : "servers"}`, + completedMsg: ({ result }) => { + const n = result.stopped.length; + const base = `Stopped ${n} simulator ${n === 1 ? "server" : "servers"}`; + // `unmatched` is the whole point of the scoped stop: a mistyped id must + // not read as a clean machine. Omitting it here would report exactly + // that — "Stopped 0 simulator servers" for a teardown that reaped + // nothing because every id was wrong. + const unmatched = result.unmatched; + return unmatched?.length + ? `${base} (${unmatched.length} supplied ${unmatched.length === 1 ? "id" : "ids"} matched no service)` + : base; + }, failedMsg: ({ failureSignal }) => `Failed to stop simulator servers: ${failureSignal.error_code}`, }, - description: `Stop running simulator-server processes (iOS + Android), native devtools services, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. + description: `Stop running simulator-server processes (iOS + Android), native devtools and accessibility services, TV-control daemons, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps, silently). Omit \`devices\` only when a machine-wide cleanup is what you actually want. -Returns { stopped } - the URNs of the services that were actually live and got shut down; ERROR/TERMINATING nodes are disposed too but never appear there, so an empty \`stopped\` only means nothing was still running. { unmatched } is present ONLY when \`devices\` was supplied AND at least one of its ids owns no service at all in these namespaces - absent on an unscoped call and when every id matched - so a mistyped id, or a device NAME passed where an id was expected, does not read as a clean machine. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, +Returns { stopped } - the URNs of the services that were actually live and got shut down; ERROR/TERMINATING nodes are disposed too but never appear there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a device driven only through CLI/adb shell-outs (Vega) registers no service and always lands here, and so does a real device this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { @@ -99,8 +57,12 @@ Returns { stopped } - the URNs of the services that were actually live and got s const stopped: string[] = []; const matchedIds = new Set(); for (const [urn, entry] of snapshot.services) { - const matchedId = scoped ? matchingDeviceId(urn, devices) : undefined; - const matches = scoped ? matchedId !== undefined : PREFIXES.some((p) => urn.startsWith(p)); + const matchedId = scoped + ? deviceIdOwningUrn(urn, DEVICE_OWNED_NAMESPACES, devices) + : undefined; + const matches = scoped + ? matchedId !== undefined + : isDeviceServiceUrn(urn, DEVICE_OWNED_NAMESPACES); // Ownership is recorded regardless of state. `disposeService` moves a // node to IDLE without removing it, so a device this session already // stopped would otherwise be reported as unmatched by the next scoped @@ -119,12 +81,12 @@ Returns { stopped } - the URNs of the services that were actually live and got s } if (!scoped) return { stopped }; // A scoped stop that named an id owning nothing is indistinguishable from - // a clean machine unless we say so — and that id is usually a typo, or a - // device NAME passed where an id belongs, in which case its - // simulator-server, devtools and (on tvOS) two --timeout 3600 daemons are - // being left running. Compared AND de-duplicated case-insensitively, to - // match the lookup: two spellings of one id are one mistake, reported in - // the caller's first spelling. + // a clean machine unless we say so — and when that id is a typo, or a + // device NAME passed where an id belongs, its simulator-server, devtools + // and (on tvOS) two --timeout 3600 daemons are being left running. + // Compared AND de-duplicated case-insensitively, to match the lookup: two + // spellings of one id are one mistake, reported in the caller's first + // spelling. const seen = new Set(); const unmatched = devices.filter((id) => { const key = id.toLowerCase(); diff --git a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts index 532ed9cc5..95ea6911f 100644 --- a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts +++ b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts @@ -1,11 +1,8 @@ import { z } from "zod"; import { ServiceState, isLiveServiceState } from "@argent/registry"; import type { Registry, ToolDefinition } from "@argent/registry"; -import { SIMULATOR_SERVER_NAMESPACE } from "../../blueprints/simulator-server"; -import { CHROMIUM_CDP_NAMESPACE } from "../../blueprints/chromium-cdp"; -import { TV_CONTROL_NAMESPACE } from "../../blueprints/tv-control"; -import { ANDROID_TV_CONTROL_NAMESPACE } from "../../blueprints/android-tv-control"; import { resolveDevice } from "../../utils/device-info"; +import { deviceIdOwningUrn, transportNamespacesForPlatform } from "./device-services"; const zodSchema = z.object({ udid: z @@ -33,24 +30,22 @@ export function createStopSimulatorServerTool( const udid = (params as { udid: string }).udid; // A single device id can back more than one service: the transport // (SimulatorServer / ChromiumCdp) and — for a TV target — the focus-driven - // TvControl daemon, which owns the spawned tvos-ax/tvos-hid processes. A - // tvOS UDID is iOS-shaped, so we can't tell it apart from a phone here - // without an async probe; instead, dispose every namespace this id could - // own and report `stopped` if any of them was live. Shape narrows the set: - // chromium ids only have a CDP session; everything else can be a simulator - // server and/or a TV-control service. + // TvControl daemon, which owns the spawned tvos-ax/tvos-hid processes. + // Shape narrows the set; see `transportNamespacesForPlatform` for why it + // stops there rather than draining everything this device owns. const platform = resolveDevice(udid).platform; - const namespaces = - platform === "chromium" - ? [CHROMIUM_CDP_NAMESPACE] - : platform === "android" - ? [SIMULATOR_SERVER_NAMESPACE, ANDROID_TV_CONTROL_NAMESPACE] - : [SIMULATOR_SERVER_NAMESPACE, TV_CONTROL_NAMESPACE]; + const namespaces = transportNamespacesForPlatform(platform); const snapshot = registry.getSnapshot(); let stopped = false; - for (const namespace of namespaces) { - const urn = `${namespace}:${udid}`; + // Scanned rather than looked up by exact URN, so this agrees with + // `stop-all-simulator-servers` on which services a device id owns: the + // match is case-insensitive and covers the `:tcp` transport suffix, both + // of which an exact `services.get()` silently missed. + const urns = [...snapshot.services.keys()].filter( + (urn) => deviceIdOwningUrn(urn, namespaces, [udid]) !== undefined + ); + for (const urn of urns) { const entry = snapshot.services.get(urn); if (!entry || entry.state === ServiceState.IDLE) continue; // A non-live node (ERROR / TERMINATING) holds no running process — e.g. diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 4ee08ba21..3e8c2f10b 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -491,6 +491,112 @@ describe("the flow-file lock", () => { }); }); +// ── What a READER of the flow file can observe ─────────────────────── + +describe("flow-file writes as seen by a concurrent reader", () => { + // The lock serializes writers, but no reader of a flow YAML joins it — + // `flow-execute`'s own load, its `run:` fragment load, flow-read-prerequisite, + // flow-add-step's sibling-fragment check, and the `argent` CLI reading from + // another process, where an in-process lock cannot reach at all. A plain + // `fs.writeFile` opens O_TRUNC, so such a reader could observe the file empty + // or half-written — and `parseFlow("")` returns `{ steps: [] }` with no error, + // which `flow-execute` summarizes as a top-level PASS over zero steps. So the + // writes must be atomic swaps rather than in-place truncations. + + /** The file's identity on disk. A rename replaces it; a truncate does not. */ + async function inode(root: string, name: string): Promise { + return (await fs.stat(flowPath(root, name))).ino; + } + + /** Anything the writer left behind next to the flow file. */ + async function strayFiles(root: string, name: string): Promise { + const entries = await fs.readdir(path.dirname(flowPath(root, name))); + return entries.filter((entry) => entry !== `${name}.yaml`); + } + + it("replaces the file on append instead of truncating it in place", async () => { + const root = await makeRoot("append-atomic"); + await start(root, "alpha"); + const before = await inode(root, "alpha"); + + await addStep(root, "alpha", "a1"); + const after = await inode(root, "alpha"); + + // Different inode == the reader either had the old file open (still whole) + // or opens the new one (whole). There is no window where the path resolves + // to a zero-length file. + expect(after).not.toBe(before); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); + }); + + it("replaces the file on start, so a reset is never observable as a partial file", async () => { + const root = await makeRoot("start-atomic"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + const before = await inode(root, "alpha"); + + // A restart truncates to an empty flow — the write most likely to be caught + // mid-flight, since it is what a concurrent flow-execute would read as a + // green run over zero steps. + await start(root, "alpha"); + expect(await inode(root, "alpha")).not.toBe(before); + expect(await readMarkers(root, "alpha")).toEqual([]); + }); + + it("leaves no scratch file behind in the flows directory", async () => { + // The swap writes a sibling temp file first. Nothing enumerates this + // directory today, but a leftover must not accumulate per append either. + const root = await makeRoot("no-scratch"); + await start(root, "alpha"); + expect(await strayFiles(root, "alpha")).toEqual([]); + + await addStep(root, "alpha", "a1"); + await addEcho(root, "alpha", "note"); + await addStep(root, "alpha", "a2"); + + expect(await strayFiles(root, "alpha")).toEqual([]); + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1", "echo:note", "tool:a2"]); + }); + + it("never exposes an empty or unparseable file while appends are in flight", async () => { + // The property the two inode assertions above encode, observed the way a + // reader actually experiences it: poll the path as fast as the event loop + // allows across a run of appends, and require every single observation to + // be a complete, parseable flow. Against an in-place `fs.writeFile` this + // catches zero-length reads. + const root = await makeRoot("reader-race"); + await start(root, "alpha"); + + let polling = true; + const observed: number[] = []; + let torn: string | undefined; + const reader = (async () => { + while (polling) { + try { + const raw = await fs.readFile(flowPath(root, "alpha"), "utf8"); + observed.push(parseFlow(raw).steps.length); + } catch (err) { + // ENOENT is equally a failure here: the path must resolve to a + // complete file at every instant, never to a gap. + torn = `${(err as Error).message} — content boundary observed`; + break; + } + } + })(); + + for (let i = 0; i < 40; i++) await addStep(root, "alpha", `a${i}`); + polling = false; + await reader; + + expect(torn).toBeUndefined(); + // The reader has to have actually looked, or it proves nothing. + expect(observed.length).toBeGreaterThan(0); + // Step counts only ever grow: no observation caught a reset-to-empty file. + expect(observed).toEqual([...observed].sort((a, b) => a - b)); + expect((await readSteps(root, "alpha")).length).toBe(40); + }); +}); + // ── The append path's source of truth ──────────────────────────────── describe("appending to a recording whose file was hand-edited", () => { diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index c8d3f8ee6..b7e8c3844 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -137,6 +137,61 @@ describe("stop-simulator-server", () => { expect(registry.disposeService).toHaveBeenCalledOnce(); expect(registry.disposeService).toHaveBeenCalledWith("ChromiumCdp:chromium-cdp-9222"); }); + + // Both stop tools now resolve "which services does this device own" through + // one shared matcher. Before that, this tool looked its URNs up with an exact, + // case-sensitive `services.get()` — so the two disagreed about the same id. + + it("matches a UDID case-insensitively, like the scoped stop-all does", async () => { + // Agents pass through whatever spelling they were handed. A case mismatch + // silently no-op'd here while stop-all reaped the same device. + const services = new Map([ + ["SimulatorServer:AAAA-BBBB", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopSimulatorServerTool(registry); + + const result = await tool.execute!({}, { udid: "aaaa-bbbb" }); + + expect(result).toEqual({ stopped: true, udid: "aaaa-bbbb" }); + expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:AAAA-BBBB"); + }); + + it("does not let a bare IP claim every wireless-adb device at that address", async () => { + // An adb serial over wifi is itself `ip:port`, so the shared matcher must + // compare the whole tail rather than splitting on ":". + const services = new Map([ + ["SimulatorServer:192.168.1.5:5555", { state: ServiceState.RUNNING, dependents: [] }], + ["SimulatorServer:192.168.1.5:5557", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopSimulatorServerTool(registry); + + const result = await tool.execute!({}, { udid: "192.168.1.5" }); + + expect(result).toEqual({ stopped: false, udid: "192.168.1.5" }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + + it("leaves this device's devtools and AX services alone", async () => { + // Deliberately narrower than stop-all: this tool is also the documented + // recovery for a wedged transport, and dropping native-devtools on a retry + // would degrade another agent's in-progress recording to coordinate taps. + const udid = "AAAA-BBBB"; + const services = new Map([ + [`SimulatorServer:${udid}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${udid}`, { state: ServiceState.RUNNING, dependents: [] }], + [`AXService:${udid}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopSimulatorServerTool(registry); + + const result = await tool.execute!({}, { udid }); + + expect(result).toEqual({ stopped: true, udid }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${udid}`); + }); }); describe("stop-all-simulator-servers", () => { @@ -533,6 +588,55 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(registry.disposeService).toHaveBeenCalledTimes(2); }); + it("stops AXService and does not call a describe-only iOS session a typo", async () => { + // An iOS session that only ran boot/launch/describe owns `AXService:` + // and nothing else — nothing cascades to it from SimulatorServer. While that + // namespace was outside the tool's set, the mandated session-end call both + // left the in-sim ax daemon (spawned --timeout 3600) running AND reported + // the perfectly correct UDID as unmatched, i.e. as a mistyped id. + const services = new Map([ + [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`AXService:${MINE}`] }); + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).toHaveBeenCalledWith(`AXService:${MINE}`); + }); + + it("scopes the tcp-transport AXService URN to its own device", async () => { + // ios-remote gives AXService the same `:tcp` suffix NativeDevtools uses. + const services = new Map([ + [`AXService:${MINE}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], + [`AXService:${THEIRS}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`AXService:${MINE}:tcp`] }); + expect(registry.disposeService).not.toHaveBeenCalledWith(`AXService:${THEIRS}:tcp`); + }); + + it("reaps AXService on an unscoped machine-wide sweep too", async () => { + const services = new Map([ + [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, {}); + + expect(result).toEqual({ + stopped: [`AXService:${MINE}`, `SimulatorServer:${THEIRS}`], + }); + }); + it("names a repeated missing id only once", async () => { const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], From e8552f3ebb45a0a2686211413b224da32d724aca Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 28 Jul 2026 12:34:35 +0200 Subject: [PATCH 12/98] fix(flow): close the remaining teardown gaps, and pin what the tests only implied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A swarm review pass over the previous commit. Every finding below was reproduced before the fix and re-run after. stop-all-simulator-servers still misreported real device ids as typos, for the same reason AXService did: three more namespaces are keyed by a device and cascade from nothing, so a session that used them owned services the tool could not see. ScreenRecordingSession holds an ffmpeg child and the touch-visualizer overlay it enabled on the device; NativeProfilerSession holds an xctrace child, or an on-device perfetto process and its trace file; JsRuntimeDebugger holds a bound loopback server, the CDP socket to Metro and a log handle. The debugger URNs also interpose the Metro port (`::`), so matching them as `:` attributed them to no device at all — the matcher now knows both shapes, consuming only the first colon so a wireless adb serial after the port still compares whole. The atomic write introduced two regressions of its own. Its scratch name was derived from the flow file's basename, which has no length cap, so a long flow name that appended fine before now failed ENAMETOOLONG — the name is fixed-length now. And the cleanup guard started after the temp file was created, leaking a scratch file into the user's committed .argent/flows/ on any write that failed after opening. requireRecordingSession ended with "Call flow-start-recording first." That is reached for a key that was finished, superseded, or dropped by the concurrency cap as well as one never started, and in those cases the flow file on disk is fully populated while flow-start-recording truncates unconditionally and reports no `restarted`. The same doctrine assertSessionStillLive already follows now applies here. Tests: several passed against implementations they claimed to reject. The stop-simulator-server narrowness guard used a udid that classifies as android, where the iOS-only services it guards can never appear, so widening the iOS branch — the exact regression it exists to catch — kept it green. The client-mode finish asserted only the shape of `savedTo`, never its content, though in client mode that directive is the only thing that lands the file; an empty body satisfied every other assertion. Adds coverage for the swap-failure branch, the not-found message's root normalization, the superseded-echo wording, and concurrent recordings in client mode, which had none. Descriptions: teardown of a mid-recording devtools session is not silent (every degraded capture warns in the returned message), a TERMINATING node is not disposed (the registry early-returns), `devices: []` stops nothing, and stop-simulator-server also reaps TV-control daemons while deliberately leaving devtools alone. --- .../src/tools/flows/flow-finish-recording.ts | 2 +- .../src/tools/flows/flow-start-recording.ts | 12 +- .../tool-server/src/tools/flows/flow-utils.ts | 77 +++++++++--- .../src/tools/simulator/device-services.ts | 116 ++++++++++++++---- .../simulator/stop-all-simulator-servers.ts | 6 +- .../tools/simulator/stop-simulator-server.ts | 10 +- .../flows/flow-concurrent-recording.test.ts | 89 +++++++++++--- .../test/flows/flow-remote-recording.test.ts | 112 ++++++++++++++++- .../tool-server/test/flows/flow-utils.test.ts | 42 ++++++- packages/tool-server/test/stop-tools.test.ts | 107 +++++++++++++++- 10 files changed, 498 insertions(+), 75 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 97f7fd376..057b93f56 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -84,7 +84,7 @@ export const flowFinishRecordingTool: ToolDefinition< failedMsg: ({ params, failureSignal }) => `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving any other recordings in progress untouched. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. + description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving recordings under any OTHER key untouched. On this key it finishes whatever take is live, which is not necessarily the one you started: the (project_root, name) key has no ownership check, so if another agent restarted this name your steps were already discarded and you get ITS take, reported as yours. flow-add-step detects that case and fails loudly; this tool cannot, so pick a name unique to your task. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. You can still edit the .yaml file directly afterwards to remove or reorder steps.`, zodSchema, services: () => ({}), diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 3bb55d105..1b4626273 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -134,11 +134,21 @@ to remove or reorder steps.`, // there is no longer a switched-away-from flow to report. if (replaced) { const discardedSteps = replaced.flow.steps.length; + // Only claim the file was reset when this process actually reset it. In + // client mode the truncation happens only once the client applies the + // directive, and a rejected path or a failed write there surfaces as + // `savedTo: null` — so asserting the reset here would tell the agent its + // file is empty while it still holds the previous take. + const reset = + persist === "host" + ? `${filePath} reset to an empty flow.` + : `${filePath} is reset to an empty flow once your client applies \`savedTo\` ` + + `(a null \`savedTo\` means it did not).`; return { message: `Restarted recording "${params.name}" — the previous take ` + `(${discardedSteps} step${discardedSteps === 1 ? "" : "s"}) was discarded and ` + - `${filePath} reset to an empty flow.`, + reset, restarted: true, discardedSteps, flowFile, diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 6b7bccc41..f2f4badcd 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -101,12 +101,26 @@ export function assertSafeFlowName(name: string): void { * * Two different projects can never collide on one key. The converse is only * true up to `path.join`, which folds a trailing slash, `//` and `.` segments - * but NOT symlinks or case: a caller that spells one root two ways (`/tmp/p` - * vs `/private/tmp/p` on macOS, or a case-variant on APFS) gets two sessions - * writing one file, and their appends can lose each other. Callers pass a - * cwd-derived root, so this needs two callers disagreeing about the spelling of - * the same directory; resolving symlinks here is not an option because in - * "client" mode the root does not exist on this host at all. + * but NOT symlinks or case. Two spellings that the filesystem considers one + * path therefore mint two sessions — and two independent locks — over one file, + * which bypasses every guarantee here: neither session is ever "superseded", so + * a restart silently truncates the other's live take and both agents are told + * they finished successfully with a mixture of each other's steps. + * + * Two ways in, and the second is the likelier one: + * - the ROOT spelled two ways (`/tmp/p` vs `/private/tmp/p` on macOS). Needs + * two callers disagreeing about one directory; roots are cwd-derived, so this + * is rare. + * - the NAME cased two ways (`Login` vs `login`) on a case-insensitive volume, + * which APFS is by default. The name is agent-chosen free text, so this needs + * only two agents naming the same flow differently — hence the "pick a name + * unique to your task" warning on `flow-start-recording`. + * + * Neither is normalized away, because the correct normalization is the + * filesystem's and we cannot ask it: case-folding the key would wrongly merge + * two genuinely distinct flows on a case-SENSITIVE volume (ext4), and resolving + * symlinks is impossible in "client" mode, where the root does not exist on this + * host at all. */ export function getFlowPath(projectRoot: string, name: string): string { const flowsDir = getFlowsDir(projectRoot); @@ -273,7 +287,7 @@ export function withFlowFileLock( * {@link assertSessionStillLive} makes the next append fail loudly rather than * write into a recording the server has forgotten. */ -const MAX_RECORDINGS = 32; +export const MAX_RECORDINGS = 32; /** * Stamp a session as most-recently-used. A counter rather than `Date.now()`: @@ -368,9 +382,19 @@ export function requireRecordingSession(projectRoot: string, name: string): Reco const activeList = here.length ? `${here.map((r) => `"${r.name}"`).join(", ")}${others}` : `none in this project${others}`; + // Do NOT tell the agent to just call flow-start-recording. This message is + // reached when the key was never started, but equally when a take was + // finished, superseded, or dropped by the MAX_RECORDINGS backstop — and in + // those cases the flow file on disk is fully populated while no session + // owns it. flow-start-recording truncates unconditionally, so the advice + // that recovers the first case destroys the others. Same doctrine as + // {@link assertSessionStillLive}, which faces the identical ambiguity. throw new FailureError( `No active recording for flow "${name}" in ${projectRoot}. ` + - `Call flow-start-recording first. Active recordings: ${activeList}.`, + `If you have not started it yet, call flow-start-recording — but note it ` + + `truncates, so if ${getFlowPath(projectRoot, name)} already holds a take you ` + + `want (finished, or interrupted by a restart), copy it aside or record under ` + + `a fresh name instead. Active recordings: ${activeList}.`, { error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, failure_stage: "flow_require_recording", @@ -2345,21 +2369,37 @@ let flowWriteSeq = 0; * Writing to a sibling temp file and renaming makes the swap atomic: a reader * sees either the whole previous file or the whole new one. The temp name is * dotted and `.tmp`-suffixed so a half-written scratch file can never be - * mistaken for a flow (`getFlowPath` only ever produces `.yaml`, and - * nothing enumerates the flows directory). + * mistaken for a flow: `getFlowPath` only ever produces `.yaml`, and the + * one thing that enumerates this directory — `argent flow list` — filters on + * that extension. Keep both halves of that agreement if either side changes. + * + * It deliberately does NOT embed the flow name. A flow name has no length cap + * (`FLOW_NAME_PATTERN` constrains the character set only), so `.yaml` can + * legitimately run to NAME_MAX — and prefixing that with a discriminator would + * push the scratch name past the limit, turning an append that used to work + * into ENAMETOOLONG. pid + counter is unique on its own: the counter separates + * writers inside this process, the pid separates this process from any CLI + * sharing the directory. + * + * The swap costs two things a write-through would have kept, both accepted for + * the atomicity: it needs write permission on the DIRECTORY rather than on the + * file, and it replaces the inode, so a chmod on the flow file or a hardlink to + * it does not survive an append. */ async function writeFlowFile(filePath: string, content: string): Promise { const tmpPath = path.join( path.dirname(filePath), - `.${path.basename(filePath)}.${process.pid}.${++flowWriteSeq}.tmp` + `.argent-flow-${process.pid}-${++flowWriteSeq}.tmp` ); - await fs.writeFile(tmpPath, content, "utf8"); try { + await fs.writeFile(tmpPath, content, "utf8"); // Atomic within a filesystem, and the temp file is a sibling of the target, // so it is always the same one. await fs.rename(tmpPath, filePath); } catch (err) { - // Leave no scratch file behind on a failed swap (e.g. a read-only dir). + // Leave no scratch file behind, whichever half failed. The write itself can + // fail with the file already created (ENOSPC, EIO), so this has to cover it + // too — nothing else ever sweeps this directory. await fs.rm(tmpPath, { force: true }).catch(() => {}); throw err; } @@ -2406,8 +2446,15 @@ export type FlowSavedTo = string | ClientFileDirective; * been finished, restarted or evicted, leaving it with a session object that is * no longer the one registered for its key. Writing anyway is the worst outcome: * the step lands in a file that now belongs to a *different* take and the caller - * is told it succeeded. Re-check identity at write time — inside the flow-file - * lock, so the check sees the state the write will see — and fail loudly. + * is told it succeeded. Re-check identity at write time, inside the flow-file + * lock, and fail loudly. + * + * The lock makes this exact against the other flow tools, which all mutate + * `recordings` for a key while holding that key's lock. It is NOT exact against + * {@link evictIfOverCapacity}, which runs under some OTHER key's lock and can + * therefore drop this session between the check and the write. That race is + * benign — the step still lands in the file it was recorded for, and only the + * NEXT call on the key reports the recording gone. */ function assertSessionStillLive(session: RecordingSession, step: FlowStep): void { const current = recordings.get(getFlowPath(session.projectRoot, session.name)); diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index aee2011b3..70df40b1a 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -5,41 +5,83 @@ import { CHROMIUM_CDP_NAMESPACE } from "../../blueprints/chromium-cdp"; import { TV_CONTROL_NAMESPACE } from "../../blueprints/tv-control"; import { ANDROID_TV_CONTROL_NAMESPACE } from "../../blueprints/android-tv-control"; import { AX_SERVICE_NAMESPACE } from "../../blueprints/ax-service"; +import { SCREEN_RECORDING_SESSION_NAMESPACE } from "../../blueprints/screen-recording-session"; +import { NATIVE_PROFILER_SESSION_NAMESPACE } from "../../blueprints/native-profiler-session"; +import { JS_RUNTIME_DEBUGGER_NAMESPACE } from "../../blueprints/js-runtime-debugger"; +import { NETWORK_INSPECTOR_NAMESPACE } from "../../blueprints/network-inspector"; +import { REACT_PROFILER_SESSION_NAMESPACE } from "../../blueprints/react-profiler-session"; /** * Which services one device id owns — the single definition of that mapping, * shared by `stop-simulator-server` (one device, transport scope) and - * `stop-all-simulator-servers` (every device-owned service). Two independent - * matchers drifted apart once before: one was case-sensitive and blind to the - * `:tcp` suffix, so the same udid reaped different services depending on which - * tool the agent reached for. + * `stop-all-simulator-servers` (every device-owned service). The two tools had + * two separate matchers that drifted apart: one was case-sensitive and blind to + * the `:tcp` suffix, so the same udid reaped different services depending on + * which tool the agent reached for. + * + * Note this unifies how a URN is matched, not how a raw id is classified: + * `stop-simulator-server` still picks its namespace set from + * `resolveDevice().platform`, whose prefix tests are case-SENSITIVE, so an id + * spelled in the wrong case can still land on the wrong namespace set there. */ /** - * Every discriminator a device-scoped URN appends after the device id - * (`NativeDevtools::tcp` and `AXService::tcp`; every other URN in - * {@link DEVICE_OWNED_NAMESPACES} is a bare `:`). + * Every discriminator a device-scoped URN appends AFTER the device id + * (`NativeDevtools::tcp` and `AXService::tcp` are the only two; + * every other URN in {@link DEVICE_OWNED_NAMESPACES} ends at the device id). * Enumerated rather than matched as "anything after a colon", because a device * id can itself end in `:`: an adb serial over wifi is * `192.168.1.5:5555`, so a suffix wildcard would let the bare `192.168.1.5` * claim every device at that address and tear down another agent's — while * reporting nothing unmatched. */ -export const URN_SUFFIXES = ["", ":tcp"] as const; +const URN_SUFFIXES = ["", ":tcp"] as const; + +/** + * Namespaces whose URN interposes the Metro port between the namespace and the + * device id: `::`. Split off from the plain shape + * because the tail is not the device id — matching these as if it were would + * report every debugger session as belonging to no device. + * + * Only the FIRST colon is consumed. The remainder is compared whole, so a + * wireless adb serial (`JsRuntimeDebugger:8081:192.168.1.5:5555`) still + * resolves to `192.168.1.5:5555` and not to `192.168.1.5`. + */ +const PORT_KEYED_NAMESPACES: readonly string[] = [ + JS_RUNTIME_DEBUGGER_NAMESPACE, + // Both of these declare `getDependencies -> JsRuntimeDebugger:`, so + // disposing the debugger already cascades to them. Listed anyway so ownership + // is recognized even if only one of them is live. + NETWORK_INSPECTOR_NAMESPACE, + REACT_PROFILER_SESSION_NAMESPACE, +]; /** - * Every namespace whose service belongs to exactly one device, and whose + * Every namespace whose service belongs to exactly one device and whose * `dispose()` frees something worth freeing. A device owning none of these is * not a bad id — Vega is driven entirely by CLI/adb shell-outs and registers no * service at all. * - * `AXService` is here because its `dispose()` is the only thing that reaps the - * in-sim ax daemon (spawned `--timeout 3600`) and unlinks its socket; nothing - * cascades from `SimulatorServer`, so an iOS session that only ran - * boot/launch/describe owns this and nothing else. `TvControl` likewise owns two - * spawned `--timeout 3600` daemons. (`AndroidTvControl` is stateless adb - * shell-outs with a no-op dispose, but is included for symmetry so the snapshot - * is fully drained.) + * Membership is decided by "does dispose() reap a resource that outlives the + * call", because nothing here cascades: of all the blueprints, only + * NetworkInspector, ReactProfilerSession and ChromiumJsRuntimeDebugger declare + * `getDependencies`, so a namespace left out of this list is simply never torn + * down by a session-end stop. + * + * - `AXService` owns the in-sim ax daemon (spawned `--timeout 3600`) and its + * socket. An iOS session that only ran boot/launch/describe owns this and + * nothing else. + * - `TvControl` owns two spawned `--timeout 3600` daemons. + * - `ScreenRecordingSession` owns an ffmpeg child, an MJPEG frame stream, and + * the touch-visualizer overlay it enabled on the device. + * - `NativeProfilerSession` owns an xctrace child on iOS, and on Android an + * on-device perfetto process plus its trace file. + * - `JsRuntimeDebugger` owns a bound loopback HTTP/WebSocket server, the CDP + * socket to Metro, and a log file handle. + * + * (`AndroidTvControl` is stateless adb shell-outs with a no-op dispose, but is + * included for symmetry so the snapshot is fully drained. ChromiumJsRuntimeDebugger + * is omitted deliberately: it cascades from `ChromiumCdp`, which is listed.) */ export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ SIMULATOR_SERVER_NAMESPACE, @@ -49,6 +91,9 @@ export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ TV_CONTROL_NAMESPACE, ANDROID_TV_CONTROL_NAMESPACE, AX_SERVICE_NAMESPACE, + SCREEN_RECORDING_SESSION_NAMESPACE, + NATIVE_PROFILER_SESSION_NAMESPACE, + ...PORT_KEYED_NAMESPACES, ]; /** @@ -59,7 +104,7 @@ export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ * the documented recovery for a wedged transport ("stop it and retry"), and * widening it to devtools/AX would make a routine retry silently drop the * native-devtools connection another agent's in-progress recording depends on — - * degrading that flow to coordinate taps, which is the exact hazard + * degrading that flow to coordinate taps, which is the hazard * `stop-all-simulator-servers`' `devices` scope exists to prevent. Agents * finishing a session call `stop-all-simulator-servers` instead, which drains * everything. @@ -73,9 +118,20 @@ export function transportNamespacesForPlatform(platform: string): readonly strin } /** - * Which entry of `deviceIds` owns `urn` within `namespaces`, if any. The tail - * after the namespace is compared whole (never split on ":", see - * {@link URN_SUFFIXES}). + * The device-id portion of `urn` if it belongs to `namespace`, else undefined. + * Accounts for the two URN shapes (see {@link PORT_KEYED_NAMESPACES}). + */ +function deviceIdPortion(urn: string, namespace: string): string | undefined { + if (!urn.startsWith(`${namespace}:`)) return undefined; + const tail = urn.slice(namespace.length + 1); + if (!PORT_KEYED_NAMESPACES.includes(namespace)) return tail; + const afterPort = tail.indexOf(":"); + return afterPort < 0 ? undefined : tail.slice(afterPort + 1); +} + +/** + * Which entry of `deviceIds` owns `urn` within `namespaces`, if any. The device + * id is compared whole (never split on ":", see {@link URN_SUFFIXES}). * * Matching is case-insensitive: iOS UDIDs are conventionally upper-case but * agents pass through whatever they were given, and a case mismatch must not @@ -90,13 +146,19 @@ export function deviceIdOwningUrn( namespaces: readonly string[], deviceIds: readonly string[] ): string | undefined { - const namespace = namespaces.find((ns) => urn.startsWith(`${ns}:`)); - if (namespace === undefined) return undefined; - const tail = urn.slice(namespace.length + 1).toLowerCase(); - return deviceIds.find((id) => { - const lower = id.toLowerCase(); - return URN_SUFFIXES.some((suffix) => tail === `${lower}${suffix}`); - }); + for (const namespace of namespaces) { + const portion = deviceIdPortion(urn, namespace); + if (portion === undefined) continue; + const tail = portion.toLowerCase(); + const owner = deviceIds.find((id) => { + const lower = id.toLowerCase(); + return URN_SUFFIXES.some((suffix) => tail === `${lower}${suffix}`); + }); + // No namespace can contain ":", so at most one can prefix a given URN — + // a miss here is a miss outright, not a reason to keep scanning. + return owner; + } + return undefined; } /** Whether `urn` belongs to any of `namespaces`, regardless of which device. */ diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 805e031ef..1166c8661 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -42,9 +42,9 @@ export function createStopAllSimulatorServersTool( failedMsg: ({ failureSignal }) => `Failed to stop simulator servers: ${failureSignal.error_code}`, }, - description: `Stop running simulator-server processes (iOS + Android), native devtools and accessibility services, TV-control daemons, and Chromium CDP sessions, freeing their resources. Call this when your session ends or the user says they are done. -PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps, silently). Omit \`devices\` only when a machine-wide cleanup is what you actually want. -Returns { stopped } - the URNs of the services that were actually live and got shut down; ERROR/TERMINATING nodes are disposed too but never appear there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a device driven only through CLI/adb shell-outs (Vega) registers no service and always lands here, and so does a real device this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, + description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions and JS-runtime debugger sessions - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. +PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a device driven only through CLI/adb shell-outs (Vega) registers no service and always lands here, and so does a real device this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts index 95ea6911f..b1ba16a1a 100644 --- a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts +++ b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts @@ -23,7 +23,7 @@ export function createStopSimulatorServerTool( failedMsg: ({ params, failureSignal }) => `Failed to stop simulator server for ${params.udid}: ${failureSignal.error_code}`, }, - description: `Stop the transport session for a specific device (iOS / Android: simulator-server process; Chromium: CDP WebSocket) and free its resources. Use when you are done interacting with one device but want to keep others running. Returns { stopped, udid }. Fails silently if no session is open for the given id.`, + description: `Stop the transport session for a specific device (iOS / Android: simulator-server process; Chromium: CDP WebSocket) and free its resources; on a TV target it also reaps that device's TV-control daemons. Use when you are done interacting with one device but want to keep others running, or to restart a wedged transport. Deliberately leaves this device's native-devtools, accessibility, profiler and debugger services running - to drain those as well, use stop-all-simulator-servers with \`devices\`. Returns { stopped, udid }. Fails silently if no session is open for the given id.`, zodSchema, services: () => ({}), async execute(_services, params) { @@ -39,9 +39,11 @@ export function createStopSimulatorServerTool( const snapshot = registry.getSnapshot(); let stopped = false; // Scanned rather than looked up by exact URN, so this agrees with - // `stop-all-simulator-servers` on which services a device id owns: the - // match is case-insensitive and covers the `:tcp` transport suffix, both - // of which an exact `services.get()` silently missed. + // `stop-all-simulator-servers` on which services a device id owns. The + // live difference is case: an exact `services.get()` silently no-op'd on + // a lower-cased UDID that the scoped stop-all reaped. (The shared matcher + // also understands the `:tcp` suffix, which no namespace in this tool's + // set currently emits — it costs nothing and keeps one grammar.) const urns = [...snapshot.services.keys()].filter( (urn) => deviceIdOwningUrn(urn, namespaces, [udid]) !== undefined ); diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 3e8c2f10b..d106cc19e 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { readFileSync } from "node:fs"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; @@ -16,6 +15,7 @@ import { __resetRecordingsForTesting, getRecordingSession, listActiveRecordings, + MAX_RECORDINGS, parseFlow, serializeFlow, withFlowFileLock, @@ -45,23 +45,6 @@ import { const IOS_DEVICE = "00000000-0000-0000-0000-0000000000ab"; -/** - * The concurrent-recording cap is internal to flow-utils, and a copy of it here - * would silently stop testing the real backstop the day it changes. Read it out - * of the source instead. - */ -function readMaxRecordings(): number { - const source = readFileSync( - path.resolve(__dirname, "../../src/tools/flows/flow-utils.ts"), - "utf8" - ); - const match = /^const MAX_RECORDINGS = (\d+);$/m.exec(source); - if (!match) throw new Error("could not read MAX_RECORDINGS out of flow-utils.ts"); - return Number(match[1]); -} - -const MAX_RECORDINGS = readMaxRecordings(); - // ── Harness ────────────────────────────────────────────────────────── let roots: string[] = []; @@ -558,6 +541,46 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(await readMarkers(root, "alpha")).toEqual(["tool:a1", "echo:note", "tool:a2"]); }); + it("still appends under a flow name long enough to fill the filesystem's limit", async () => { + // A flow name has no length cap — FLOW_NAME_PATTERN constrains the + // character set only — so `.yaml` can legitimately reach NAME_MAX + // (255 on APFS/ext4). A scratch name derived from the flow file's basename + // would overflow that and fail an append that used to work, so the temp + // name must be a fixed-length one. + const root = await makeRoot("long-name"); + const name = "a".repeat(250); + expect(`${name}.yaml`.length).toBe(255); + + await start(root, name); + await addStep(root, name, "a1"); + await addEcho(root, name, "note"); + + expect(await readMarkers(root, name)).toEqual(["tool:a1", "echo:note"]); + expect(await strayFiles(root, name)).toEqual([]); + }); + + it("propagates a failed swap and leaves no scratch file behind", async () => { + // The only coverage the cleanup branch has otherwise is the success path, + // where the rename itself consumes the temp file — so deleting the whole + // try/catch passes. Force the rename to fail by planting a NON-EMPTY + // DIRECTORY where the flow file goes: `mkdir -p` on the parent still + // succeeds and the temp write still succeeds, so this reaches `fs.rename` + // and nothing else. (A read-only dir fails earlier, at the temp write.) + const root = await makeRoot("swap-fails"); + const target = flowPath(root, "alpha"); + await fs.mkdir(path.join(target, "occupied"), { recursive: true }); + + await expect(start(root, "alpha")).rejects.toThrow(); + + // The failure must not leave a scratch file in the user's committed + // .argent/flows/ — nothing else ever sweeps it. + const entries = await fs.readdir(path.dirname(target)); + expect(entries.filter((e) => e.endsWith(".tmp"))).toEqual([]); + // And it must not register a session for a file that was never written: + // the next append would otherwise die on an unrelated ENOENT. + expect(listActiveRecordings()).toEqual([]); + }); + it("never exposes an empty or unparseable file while appends are in flight", async () => { // The property the two inode assertions above encode, observed the way a // reader actually experiences it: poll the path as fast as the event loop @@ -778,6 +801,36 @@ describe("a restart that lands while a step is still running", () => { expect(finished.steps).toBe(1); }); + it("does not warn a superseded ECHO that it already ran on the device", async () => { + // The "repeating it repeats that action" caveat is true of a tool step, + // which executed live before the append was rejected. An echo is a label — + // it touched no device, so telling its author to weigh a repeat is the same + // class of false advice the fresh-name wording replaced. Only the tool + // branch of that ternary is asserted above, so pin the echo branch here. + const root = await makeRoot("supersede-echo"); + await start(root, "alpha"); + + // An echo has no live device step to park in, so the only window in which + // it can be superseded is the flow-file lock. Hold the lock, then queue the + // restart AHEAD of the echo: the echo still resolves its session now (the + // restart's body has not run, so the old session is still registered), but + // by the time it reaches the front of the queue the restart has replaced it. + const gate = openGate(); + const held = withFlowFileLock(root, "alpha", () => gate.promise); + const restarting = start(root, "alpha"); + const echoing = addEcho(root, "alpha", "a label"); + + gate.open(); + await held; + expect((await restarting).restarted).toBe(true); + + const err = await captureFailure(echoing); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); + expect((err as Error).message).toContain("Nothing was added to the flow file"); + expect((err as Error).message).toContain("fresh name"); + expect((err as Error).message).not.toContain("already ran on the device"); + }); + it("truncates and re-registers only once the flow's lock is free", async () => { const root = await makeRoot("restart-lock"); await start(root, "alpha"); diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index c5bfc4a8c..3d751c7b5 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -190,7 +190,21 @@ describe("flow recording with a remote client (probe miss)", () => { expect(result.steps).toBe(1); expect(result.summary).toEqual(["1. echo: only step"]); expect(result.path).toBe(CLIENT_FLOW_PATH); - expect(result.savedTo).toMatchObject({ [CLIENT_FILE_MARKER]: true }); + + // Assert the directive's CONTENT, not just its shape. `steps`, `summary` + // and `path` all derive from the in-memory flow, so they agree with each + // other no matter what `savedTo` carries — and in client mode `savedTo` is + // the only thing that lands the artifact (`path` names a file that does not + // exist on this host). A directive built with an empty body would satisfy + // every other assertion here while the client wrote a flow with no steps, + // which replays as a top-level PASS over nothing. + const savedTo = result.savedTo as { [CLIENT_FILE_MARKER]: true; path: string; content: string }; + expect(savedTo[CLIENT_FILE_MARKER]).toBe(true); + expect(savedTo.path).toBe(CLIENT_FLOW_PATH); + expect(parseFlow(savedTo.content).steps).toEqual([{ kind: "echo", message: "only step" }]); + // The finished YAML the caller is shown and the one the client writes must + // be the same bytes. + expect(savedTo.content).toBe(result.flowFile); await expect( flowFinishRecordingTool.execute({}, { name: "remote-flow", project_root: CLIENT_ROOT }) @@ -772,3 +786,99 @@ describe("flow_file containment", () => { ).rejects.toThrow("Invalid flow_file"); }); }); + +/** + * The concurrency contract, exercised in CLIENT mode. The two mechanisms cross + * here: the session key is a path that does not exist on this host, and the + * authoritative flow content is the in-memory copy rather than the file. The + * host-mode suite (flow-concurrent-recording.test.ts) cannot reach either. + */ +describe("concurrent recordings against a remote client", () => { + it("keeps genuinely overlapping remote appends complete and ordered", async () => { + // The overlap has to come from flow-add-step's LIVE sub-tool call, which is + // the only await in the client-mode path: once past it, push → validate → + // serialize runs synchronously, so echoes alone can never interleave and + // would prove nothing. Each sub-tool call parks until all of them have + // arrived, so every append is in flight simultaneously before any completes. + const arrived: (() => void)[] = []; + const allArrived = new Promise((resolve) => { + arrived.push(resolve); + }); + let seen = 0; + const registry = { + invokeTool: vi.fn(async () => { + if (++seen === 6) arrived[0](); + await allArrived; + return { tapped: true }; + }), + getTool: vi.fn(() => undefined), + } as unknown as Registry; + const addStep = createFlowAddStepTool(registry); + + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT }, + remoteCtx() + ); + + const results = await Promise.all( + Array.from({ length: 6 }, (_, i) => + addStep.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, command: "tap", args: `{"x":0.${i}}` } + ) + ) + ); + + // The last directive to be produced carries the full flow. Every one of the + // six steps must be in it exactly once — in client mode the in-memory copy + // is the ONLY copy, so a lost update is unrecoverable. + const contents = results.map((r) => { + const directive = r.savedTo as { [CLIENT_FILE_MARKER]: true; content: string }; + expect(directive[CLIENT_FILE_MARKER]).toBe(true); + return parseFlow(directive.content).steps; + }); + const fullest = contents.reduce((a, b) => (b.length > a.length ? b : a)); + expect(fullest).toHaveLength(6); + const xs = fullest.map((s) => (s.kind === "tool" ? String(s.args.x) : "?")); + expect(new Set(xs).size).toBe(6); + // Each append saw a strictly larger flow than the one before it. + expect(contents.map((c) => c.length).sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5, 6]); + }); + + it("fails an append whose remote recording was restarted, and writes nothing to this host", async () => { + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT }, + remoteCtx() + ); + await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "first take" } + ); + + // A second agent takes the same key on the same client project. + const restarted = await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT }, + remoteCtx() + ); + expect(restarted).toMatchObject({ restarted: true, discardedSteps: 1 }); + // The reset is the client's to perform, so the message must not assert it + // as done here — nothing on this host was touched. + expect((restarted as { message: string }).message).toContain("once your client applies"); + + // The new take is empty and usable; the discarded take's content is gone. + const after = await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "second take" } + ); + const directive = after.savedTo as { content: string }; + const flow = parseFlow(directive.content); + expect(flow.steps).toHaveLength(1); + expect(flow.steps[0]).toMatchObject({ kind: "echo", message: "second take" }); + + // Still nothing on this host: the client's root was never created here. + await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow(); + }); +}); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index f0389d796..a6f8de6f9 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -12,6 +12,7 @@ import { clearRecordingSession, listActiveRecordings, __resetRecordingsForTesting, + MAX_RECORDINGS, getFlowPath, appIdForPlatform, chromiumLaunchSpec, @@ -1079,6 +1080,45 @@ describe("recording sessions", () => { expect(message).not.toContain("/tmp/proj-c"); }); + it("treats a differently-spelled but identical root as THIS project", () => { + // The partition compares path.join-normalized flows dirs, not raw strings. + // A caller that spells its own root with a trailing slash must still be + // shown its own live recordings — a strict === would answer "none in this + // project (plus 1 in other projects)", degrading the message in exactly the + // wrong-project_root case it exists to diagnose. Every other test here + // spells both sides identically, so only this one separates the two. + start("/tmp/proj-a", "checkout"); + const message = (() => { + try { + requireRecordingSession("/tmp/proj-a/", "chekout"); + } catch (err) { + return (err as Error).message; + } + throw new Error("expected a throw"); + })(); + expect(message).toMatch(/Active recordings: "checkout"\./); + expect(message).not.toContain("other projects"); + }); + + it("does not tell the agent to just call flow-start-recording", () => { + // This message is reached for a key that was never started, but equally for + // one that was finished, superseded, or dropped by the concurrency cap — + // and in those cases the flow file on disk is fully populated. Naming + // flow-start-recording as the fix destroys it, because it truncates + // unconditionally and reports no `restarted` when no session was replaced. + const message = (() => { + try { + requireRecordingSession("/tmp/proj-a", "finished-earlier"); + } catch (err) { + return (err as Error).message; + } + throw new Error("expected a throw"); + })(); + expect(message).toContain("truncates"); + expect(message).toMatch(/record under a fresh name|copy it aside/); + expect(message).not.toMatch(/Call flow-start-recording first/); + }); + it('reports "none in this project" when nothing is being recorded', () => { expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( /Active recordings: none in this project\./ @@ -1157,7 +1197,7 @@ describe("recording sessions", () => { // ties when the whole fill and the touch land inside one millisecond, which // holds when this file runs alone but not under full-suite load. The // counter's tie-freedom is argued at `touch()` rather than pinned here. - const cap = 32; + const cap = MAX_RECORDINGS; for (let i = 0; i < cap; i++) start("/tmp/proj-a", `flow-${i}`); expect(listActiveRecordings()).toHaveLength(cap); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index b7e8c3844..e7643fe87 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -127,6 +127,10 @@ describe("stop-simulator-server", () => { it("does not target TvControl for a chromium id", async () => { const services = new Map([ ["ChromiumCdp:chromium-cdp-9222", { state: ServiceState.RUNNING, dependents: [] }], + // The negative control has to BE in the map. Without it, "disposed once" + // is satisfied by the single entry present and the chromium branch could + // return TvControl too without failing anything. + ["TvControl:chromium-cdp-9222", { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopSimulatorServerTool(registry); @@ -177,7 +181,14 @@ describe("stop-simulator-server", () => { // Deliberately narrower than stop-all: this tool is also the documented // recovery for a wedged transport, and dropping native-devtools on a retry // would degrade another agent's in-progress recording to coordinate taps. - const udid = "AAAA-BBBB"; + // + // The udid must be a REAL iOS UUID. `classifyDevice` only recognizes the + // 8-4-4-4-12 hex shape, so a short id like "AAAA-BBBB" classifies as + // android — and NativeDevtools/AXService, which are iOS-only, would never + // be candidates for it under any implementation. This test would then pass + // even if the iOS branch were widened to include them, which is the exact + // regression it exists to catch. + const udid = "00000000-0000-0000-0000-0000000000ab"; const services = new Map([ [`SimulatorServer:${udid}`, { state: ServiceState.RUNNING, dependents: [] }], [`NativeDevtools:${udid}`, { state: ServiceState.RUNNING, dependents: [] }], @@ -199,7 +210,12 @@ describe("stop-all-simulator-servers", () => { const services = new Map([ ["SimulatorServer:AAA", { state: ServiceState.RUNNING, dependents: [] }], ["SimulatorServer:BBB", { state: ServiceState.RUNNING, dependents: [] }], - ["JsRuntimeDebugger:CCC", { state: ServiceState.RUNNING, dependents: [] }], + // Deliberately excluded from the namespace set: it declares + // `getDependencies -> ChromiumCdp:`, so the registry cascades to it + // when that transport is disposed. Listing it too would be redundant, and + // disposing it directly here would claim a `stopped` entry for a service + // no device in this snapshot owns a transport for. + ["ChromiumJsRuntimeDebugger:CCC", { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -348,7 +364,7 @@ describe("stop-all-simulator-servers device scoping", () => { }); it("scopes the non-simulator namespaces too (ChromiumCdp / TvControl / AndroidTvControl)", async () => { - // Every namespace in PREFIXES must honour `devices`, not just + // Every namespace in DEVICE_OWNED_NAMESPACES must honour `devices`, not just // SimulatorServer/NativeDevtools: a TvControl daemon left running holds two // spawned --timeout 3600 processes, and reaping another agent's is exactly // the cross-session damage scoping exists to prevent. @@ -431,7 +447,10 @@ describe("stop-all-simulator-servers device scoping", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - const result = await tool.execute!({}, { devices: [MINE.toLowerCase()] }); + // Upper-cased id against a lower-cased URN AND vice versa: passing the + // lower-cased spelling here would leave the upper/upper and lower/lower + // pairs matching, so only a contrived asymmetric mutation would be caught. + const result = await tool.execute!({}, { devices: [MINE] }); expect(result).toEqual({ stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE.toLowerCase()}:tcp`], @@ -622,6 +641,86 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(registry.disposeService).not.toHaveBeenCalledWith(`AXService:${THEIRS}:tcp`); }); + it("owns and stops a device whose only service is a screen recording", async () => { + // ScreenRecordingSession holds an ffmpeg child, an MJPEG frame stream and + // the touch-visualizer overlay it enabled on the device, and nothing + // cascades to it. While it was outside the namespace set, a session that + // ran screen-recording-start and then the mandated teardown left ffmpeg + // running and was told its correct serial was a mistyped id. + const services = new Map([ + [`ScreenRecordingSession:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`ScreenRecordingSession:${MINE}`] }); + expect(result).not.toHaveProperty("unmatched"); + }); + + it("owns and stops a device whose only service is a native profiler session", async () => { + // Same shape: an xctrace child on iOS, an on-device perfetto process plus + // its trace file on Android. + const services = new Map([ + [`NativeProfilerSession:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`NativeProfilerSession:${MINE}`] }); + expect(result).not.toHaveProperty("unmatched"); + }); + + it("scopes the port-keyed debugger URNs to the right device", async () => { + // JsRuntimeDebugger's URN interposes the Metro port: `::`. + // Matched as `:` it belongs to nobody, so a debugger-only session + // was reported unmatched while its bound port and Metro CDP socket stayed + // open. Both devices sit behind the SAME port, so this also pins that the + // port is not what the scoping keys on. + const services = new Map([ + [`JsRuntimeDebugger:8081:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`JsRuntimeDebugger:8081:${MINE}`] }); + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).not.toHaveBeenCalledWith(`JsRuntimeDebugger:8081:${THEIRS}`); + }); + + it("does not let a port-keyed URN's port be mistaken for a wireless-adb device id", async () => { + // The device id after the port can itself be `ip:port`. Only the FIRST + // colon is the Metro port, so the remainder must be compared whole. + const serial = "192.168.1.5:5555"; + const services = new Map([ + [`JsRuntimeDebugger:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + expect(await tool.execute!({}, { devices: [serial] })).toEqual({ + stopped: [`JsRuntimeDebugger:8081:${serial}`], + }); + + // A bare IP must not claim it, and neither must the port. + const registry2 = createMockRegistry( + new Map([ + [`JsRuntimeDebugger:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + ]) + ); + const tool2 = createStopAllSimulatorServersTool(registry2); + expect(await tool2.execute!({}, { devices: ["192.168.1.5", "8081"] })).toEqual({ + stopped: [], + unmatched: ["192.168.1.5", "8081"], + }); + }); + it("reaps AXService on an unscoped machine-wide sweep too", async () => { const services = new Map([ [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], From cdcf43f8c6178c9ba9d61fb256377fec19cf1168 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 29 Jul 2026 11:36:05 +0200 Subject: [PATCH 13/98] fix(flow): make the scope key strict, and correct what the docs promise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stop-all-simulator-servers` declared `devices` on a plain z.object, so zod stripped any unrecognised key before execute ran. `udids` is the natural slip — every sibling tool in this directory spells the device parameter `udid` — and that typo left `params.devices` undefined, which is the machine-wide sweep: the caller tore down every other agent's devices believing it had scoped, with `unmatched` unreachable on that path so nothing in the response said otherwise. `.strict()` turns it into a validation error and puts `additionalProperties: false` in the schema advertised by GET /tools, so MCP, `argent run` and raw HTTP callers all get the rejection. `assertSessionStillLive` branched its `why` clause on whether the key is held by another session or by nothing, but always asserted "This key now belongs to another take" in the recovery clause. On the empty-key branch that is false by construction, so an agent whose recording was ended by its own finish, or by the concurrent-recording cap, was sent looking for a competing agent that does not exist — and in the eviction case it buried the actionable cause named one clause earlier. The recovery now branches too, and both branches are pinned. Also pins the touch inside `appendStepToFlow`, which nothing separated from the touch on resolve: a step that takes minutes on a device would otherwise leave its own session least-recently-used for that whole time, so the next flow-start-recording anywhere on the host evicted the recording that had just appended. The rest is documentation that had drifted from the code it describes: - flow-finish-recording claimed flow-add-step "detects that case and fails loudly". It only detects a takeover landing while a step is mid-flight; between calls it appends into the other agent's take and reports success — so a run of successful results read as evidence the key was still yours. - stop-simulator-server claimed it leaves the debugger running. True on iOS/Android/TV; on Chromium the JS-runtime debugger declares the CDP session as a dependency, so the transport stop cascades to it. - The argent-create-flow skill quoted a "Call flow-start-recording first" message the server no longer emits — and which the PR removed precisely because that advice truncates a populated file. - The recordings-map comment claimed concurrent agents "never see each other's state", contradicting the deliberate, bounded disclosure in the not-found path a few lines below. - The eviction backstop pointed at assertSessionStillLive for an append issued after an eviction; that one fails in requireRecordingSession. - The case-fold rationale enumerated three id spaces where the matcher accepts seven, omitting the only one that is an assumption rather than a guarantee. - Two CLI fixture comments and one in flow-start-recording described what the code used to be, unresolvable without the diff. --- .../test/run-flow-add-step-payload.test.ts | 6 +-- packages/argent-cli/test/run-help.test.ts | 6 +-- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-finish-recording.ts | 2 +- .../src/tools/flows/flow-start-recording.ts | 4 +- .../tool-server/src/tools/flows/flow-utils.ts | 31 +++++++++-- .../src/tools/simulator/device-services.ts | 21 +++++++- .../simulator/stop-all-simulator-servers.ts | 29 +++++++--- .../tools/simulator/stop-simulator-server.ts | 2 +- .../flows/flow-concurrent-recording.test.ts | 54 +++++++++++++++++++ packages/tool-server/test/stop-tools.test.ts | 23 +++++++- 11 files changed, 153 insertions(+), 27 deletions(-) diff --git a/packages/argent-cli/test/run-flow-add-step-payload.test.ts b/packages/argent-cli/test/run-flow-add-step-payload.test.ts index a5a552354..153a28ed3 100644 --- a/packages/argent-cli/test/run-flow-add-step-payload.test.ts +++ b/packages/argent-cli/test/run-flow-add-step-payload.test.ts @@ -41,9 +41,9 @@ function startServer(cap: Captured): Promise<{ url: string; close: () => Promise // zodObjectToJsonSchema over the zod schema in // packages/tool-server/src/tools/flows/flow-add-step.ts. `name` // and `project_root` identify which open recording the step - // belongs to and are required alongside `command`; a fixture that - // still advertised the old single-required shape would let a - // regression in how those flags are parsed slip through. + // belongs to and are required alongside `command`. All three have + // to be marked required here for the parser regression this test + // guards to be reachable at all. inputSchema: { type: "object", properties: { diff --git a/packages/argent-cli/test/run-help.test.ts b/packages/argent-cli/test/run-help.test.ts index 05d31a2e2..262b2f674 100644 --- a/packages/argent-cli/test/run-help.test.ts +++ b/packages/argent-cli/test/run-help.test.ts @@ -29,9 +29,9 @@ vi.mock("@argent/telemetry", () => telemetryMock); // zodObjectToJsonSchema over the zod schema in // packages/tool-server/src/tools/flows/flow-add-step.ts. Recordings are keyed // by `name` + `project_root`, so both are required alongside `command` and only -// `args` / `delayMs` are optional; a fixture still describing a single "active" -// recording with one required field would render help for a tool that no longer -// exists. +// `args` / `delayMs` are optional. Keep the fixture in step with that schema: a +// fixture marking fewer fields required renders help for a tool the server does +// not expose, and the mismatch passes silently. const flowAddStepMeta = { name: "flow-add-step", // Leading sentence of the real tool description, verbatim. diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 6edb48a46..8e556f1ea 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -142,7 +142,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. - **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. - **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. -- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . Call flow-start-recording first. Active recordings: ...`. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. +- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished, superseded by another agent, or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. ### flow-add-step arguments diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 057b93f56..8458e7f62 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -84,7 +84,7 @@ export const flowFinishRecordingTool: ToolDefinition< failedMsg: ({ params, failureSignal }) => `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving recordings under any OTHER key untouched. On this key it finishes whatever take is live, which is not necessarily the one you started: the (project_root, name) key has no ownership check, so if another agent restarted this name your steps were already discarded and you get ITS take, reported as yours. flow-add-step detects that case and fails loudly; this tool cannot, so pick a name unique to your task. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. + description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving recordings under any OTHER key untouched. On this key it finishes whatever take is live, which is not necessarily the one you started: the (project_root, name) key has no ownership check, so if another agent restarted this name your steps were already discarded and you get ITS take, reported as yours. Usually NOTHING tells you: flow-add-step detects a takeover only if one lands while a step of yours is mid-flight, so between calls - the common case - it re-resolves the key and appends into the other agent's take, reporting success. A run of successful \`Step added\` results is therefore not evidence that the key is still yours. Pick a name unique to your task. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. You can still edit the .yaml file directly afterwards to remove or reorder steps.`, zodSchema, services: () => ({}), diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 1b4626273..77104e084 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -130,8 +130,8 @@ to remove or reorder steps.`, ); // Only a same-key restart replaces anything — the documented "re-record it - // to fix it" workflow. Starting a *different* flow abandons nothing, so - // there is no longer a switched-away-from flow to report. + // to fix it" workflow. Recordings are keyed per flow file, so starting a + // *different* flow abandons nothing and there is nothing to report about it. if (replaced) { const discardedSteps = replaced.flow.steps.length; // Only claim the file was reset when this process actually reset it. In diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index f2f4badcd..761fd83e3 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -225,7 +225,13 @@ export interface RecordingSession { * being built. Two sessions on one key mean two writers on one output file (a * genuine collision); two different keys are independent, so concurrent agents * recording different flows — in one project or across projects, against one - * device or several — never see each other's state. + * device or several — never write into each other's take. + * + * Isolation of the recorded artifact, not of the fact that a recording exists: + * the not-found path of {@link requireRecordingSession} deliberately names the + * other live flows in the caller's own project (and counts the rest), so that + * disclosure is bounded rather than absent. See the comment there for what it + * discloses and why. * * The tool-server is a host-wide singleton shared by every MCP client, subagent * and CLI call on the machine, so this map is the only thing standing between @@ -284,8 +290,12 @@ export function withFlowFileLock( * after 30 min, but a long-lived server could accumulate recordings an agent * started and never finished. Well past any realistic concurrent-agent count, * so evicting should never be something an agent observes — and if it ever is, - * {@link assertSessionStillLive} makes the next append fail loudly rather than - * write into a recording the server has forgotten. + * the next append fails loudly rather than writing into a recording the server + * has forgotten. Which function reports it depends on the ordering: an append + * issued after the eviction fails in {@link requireRecordingSession}, since the + * key is already gone by the time it resolves; only one whose session was + * resolved BEFORE the eviction and landed after reaches + * {@link assertSessionStillLive}. */ export const MAX_RECORDINGS = 32; @@ -2470,13 +2480,24 @@ function assertSessionStillLive(session: RecordingSession, step: FlowStep): void // live take that just claimed this key (which restarting would both wipe and // steal), or the finished flow sitting on disk. Recording under a fresh name // is the only recovery that destroys nothing. + // Branch the same way `why` does. Asserting "this key now belongs to another + // take" on the `!current` branch is false by construction — that branch is + // selected precisely because the key is empty, and `startRecordingSession` + // registers under this key's lock, so a take that had claimed it would have + // selected the other branch. Naming a competing agent that does not exist + // sends an agent whose own finish (or the eviction named one clause earlier) + // ended the recording looking for the wrong cause. + const whatIsAtStake = current + ? `This key now belongs to another take and flow-start-recording truncates, so re-record ` + + `under a fresh name rather than restarting this one.` + : `The key is now free, but the finished take is on disk and flow-start-recording truncates ` + + `it unconditionally, so re-record under a fresh name rather than restarting this one.`; const recovery = `Nothing was added to the flow file` + (step.kind === "echo" ? ". " : ", but the step itself already ran on the device — repeating it repeats that action. ") + - `This key now belongs to another take and flow-start-recording truncates, so re-record ` + - `under a fresh name rather than restarting this one.`; + whatIsAtStake; throw new FailureError( `Recording of "${session.name}" in ${session.projectRoot} is no longer active — ${why}. ` + recovery, diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index 70df40b1a..8f24845f5 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -108,6 +108,13 @@ export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ * `stop-all-simulator-servers`' `devices` scope exists to prevent. Agents * finishing a session call `stop-all-simulator-servers` instead, which drains * everything. + * + * That narrowness is only as strong as the dependency graph, and on CHROMIUM it + * does not hold: `ChromiumJsRuntimeDebugger` declares `ChromiumCdp` as a + * dependency, so disposing the transport tears the debugger down as a dependent + * along with its captured console history. Nothing here can prevent that + * without leaving the wedged transport in place, which is the tool's whole + * purpose; `stop-simulator-server`'s description says so outright instead. */ export function transportNamespacesForPlatform(platform: string): readonly string[] { if (platform === "chromium") return [CHROMIUM_CDP_NAMESPACE]; @@ -135,8 +142,18 @@ function deviceIdPortion(urn: string, namespace: string): string | undefined { * * Matching is case-insensitive: iOS UDIDs are conventionally upper-case but * agents pass through whatever they were given, and a case mismatch must not - * silently turn a scoped stop into a no-op. No two distinct devices can differ - * only by case in any id space we support (UUID, emulator-N, chromium-cdp-N). + * silently turn a scoped stop into a no-op. + * + * That is safe only if no two distinct devices can differ by case alone. Of the + * id spaces we support, six are structurally case-safe: iOS UDIDs (hex UUID), + * `emulator-N`, `chromium-cdp-N`, adb-over-wifi `ip:port`, `remote:` for + * ios-remote, and Vega's `amazon-`. The seventh is an assumption rather + * than a guarantee: a physical Android serial is `ro.serialno`, which + * `device-info.ts` notes is vendor-defined and unconstrained, so a vendor could + * in principle ship two devices differing only in case. Accepted — colliding + * serials on ONE host would already be indistinguishable to `adb -s`, and the + * alternative (case-sensitive matching) reintroduces the silent no-op this + * exists to fix on the id space agents actually mistype, iOS UDIDs. * * Returns the caller's spelling of the id, so a tool can report which of the * ids it was given matched nothing. diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 1166c8661..d06da97d4 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -3,14 +3,27 @@ import { ServiceState, isLiveServiceState } from "@argent/registry"; import type { Registry, ToolDefinition } from "@argent/registry"; import { DEVICE_OWNED_NAMESPACES, deviceIdOwningUrn, isDeviceServiceUrn } from "./device-services"; -const zodSchema = z.object({ - devices: z - .array(z.string()) - .optional() - .describe( - "Device ids (iOS UDID / Android serial / Chromium id) to scope the teardown to — pass the devices THIS session actually used. Omit only for a deliberate machine-wide cleanup: the tool-server is shared by every agent on the host, so an unscoped stop also kills devices another agent is mid-session on." - ), -}); +const zodSchema = z + .object({ + devices: z + .array(z.string()) + .optional() + .describe( + "Device ids (iOS UDID / Android serial / Chromium id) to scope the teardown to — pass the devices THIS session actually used. Omit only for a deliberate machine-wide cleanup: the tool-server is shared by every agent on the host, so an unscoped stop also kills devices another agent is mid-session on." + ), + }) + // `.strict()` because omitting `devices` is the machine-wide sweep, so a + // misspelled key must not be silently stripped down to it. `udids` is the + // natural slip — every sibling tool in this directory spells the device + // parameter `udid`, and this is the only one that spells it `devices` — and + // under a stripping schema that typo tears down every other agent's devices + // while the caller believes it scoped, with `unmatched` unreachable on that + // path so nothing in the response says otherwise. Strict makes it a + // validation error instead, matching `stop-simulator-server`, where the same + // typo already fails loudly because `udid` is required. This also puts + // `additionalProperties: false` in the schema advertised by `GET /tools`, so + // MCP, `argent run` and raw HTTP callers all get the rejection. + .strict(); export function createStopAllSimulatorServersTool( registry: Registry diff --git a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts index b1ba16a1a..de9fc6caf 100644 --- a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts +++ b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts @@ -23,7 +23,7 @@ export function createStopSimulatorServerTool( failedMsg: ({ params, failureSignal }) => `Failed to stop simulator server for ${params.udid}: ${failureSignal.error_code}`, }, - description: `Stop the transport session for a specific device (iOS / Android: simulator-server process; Chromium: CDP WebSocket) and free its resources; on a TV target it also reaps that device's TV-control daemons. Use when you are done interacting with one device but want to keep others running, or to restart a wedged transport. Deliberately leaves this device's native-devtools, accessibility, profiler and debugger services running - to drain those as well, use stop-all-simulator-servers with \`devices\`. Returns { stopped, udid }. Fails silently if no session is open for the given id.`, + description: `Stop the transport session for a specific device (iOS / Android: simulator-server process; Chromium: CDP WebSocket) and free its resources; on a TV target it also reaps that device's TV-control daemons. Use when you are done interacting with one device but want to keep others running, or to restart a wedged transport. On iOS / Android / TV it deliberately leaves this device's native-devtools, accessibility, profiler and debugger services running - to drain those as well, use stop-all-simulator-servers with \`devices\`. On CHROMIUM that does not hold: the JS-runtime debugger declares the CDP session as a dependency, so stopping the transport cascades to it and its captured console history goes with it - reconnect with debugger-connect afterwards. Returns { stopped, udid }. Fails silently if no session is open for the given id.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index d106cc19e..2317c1296 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -786,6 +786,10 @@ describe("a restart that lands while a step is still running", () => { // the key (and take it back again). A fresh name is the only safe recovery. expect((err as Error).message).toContain("fresh name"); expect((err as Error).message).not.toMatch(/Call flow-start-recording/); + // This is the branch where a foreign take really does hold the key, so the + // message says so — the empty-key branches must not (see the finish and + // eviction cases). + expect((err as Error).message).toContain("belongs to another take"); // The step already ran live before the append was rejected, so an agent that // simply retries it would repeat the device action. expect((err as Error).message).toContain("already ran on the device"); @@ -1014,6 +1018,12 @@ describe("a finish that lands while a step is still running", () => { expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); expect((err as Error).message).toContain("no longer active"); + // The key is EMPTY here — this session's own finish cleared it, and a new + // take could only have claimed it under the same lock. Blaming a competing + // agent sends the reader hunting for one that does not exist; the hazard to + // name is the finished take now sitting on disk. + expect((err as Error).message).not.toMatch(/belongs to another take/); + expect((err as Error).message).toMatch(/finished take is on disk/); // The finished file is exactly what the finish reported. expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); @@ -1149,6 +1159,46 @@ describe("the concurrent-recording cap", () => { expect(await readMarkers(root, names[0])).toEqual(["echo:touch", "echo:still-live"]); }); + it("re-stamps a recording when its step LANDS, not just when it was resolved", async () => { + // A step that takes minutes on a device would otherwise leave its own + // session as the least-recently-used one for that whole time: the resolve + // stamped it before the step ran, and every quick call elsewhere on the + // host stamps later. The next `flow-start-recording` anywhere would then + // evict the recording that had just successfully appended, and the agent's + // next `flow-add-step` would fail on a take it was actively recording. + // + // Both callers touch on resolve via `requireRecordingSession`, so only the + // stamp inside `appendStepToFlow` separates the two — and every other + // eviction test drives recency through resolve, so none of them can. + const root = await makeRoot("touch-on-land"); + const names = await fillRecordings(root); + + // rec-0's step resolves (stamping it) and then parks on the device. + const gate = gateNextSubTool(); + const appending = addStep(root, "rec-0", "slow"); + await gate.reached; + + // Every other recording is used while that step is still running, so by + // resolve-time recency rec-0 is now the oldest entry on the table. + for (const name of names.slice(1)) await addEcho(root, name, "touch"); + + // The step lands, which must re-stamp rec-0 as most recently used. + gate.release(); + await appending; + expect(await readMarkers(root, "rec-0")).toEqual(["tool:slow"]); + + await start(root, "overflow"); + + // The recording that just appended survives; the victim is the one whose + // last use really is the oldest. Without the stamp on land, rec-0 is the + // one dropped here. + expect(getRecordingSession(root, "rec-0")).toBeDefined(); + expect(getRecordingSession(root, names[1])).toBeUndefined(); + // And it is still usable, not merely present. + await addEcho(root, "rec-0", "after"); + expect(await readMarkers(root, "rec-0")).toEqual(["tool:slow", "echo:after"]); + }); + it("rejects an append whose recording was evicted while the step ran", async () => { const root = await makeRoot("evict-inflight"); const names = await fillRecordings(root); @@ -1170,6 +1220,10 @@ describe("the concurrent-recording cap", () => { expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); expect(getFailureSignal(err)?.failure_stage).toBe("flow_session_superseded"); expect((err as Error).message).toContain("concurrent-recording cap"); + // Same empty-key branch as a self-finish: no other take holds this key, so + // the message must not send the agent looking for one — which would also + // bury the actionable cause named one clause earlier. + expect((err as Error).message).not.toMatch(/belongs to another take/); expect(await readMarkers(root, "rec-0")).toEqual([]); // A fresh call on the evicted key fails the ordinary not-live way. diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index e7643fe87..5839fcc04 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi } from "vitest"; -import { Registry, ServiceState } from "@argent/registry"; +import type { z } from "zod"; +import { Registry, ServiceState, zodObjectToJsonSchema } from "@argent/registry"; import { createStopSimulatorServerTool } from "../src/tools/simulator/stop-simulator-server"; import { createStopAllSimulatorServersTool } from "../src/tools/simulator/stop-all-simulator-servers"; import { stopMetroTool } from "../src/tools/simulator/stop-metro"; @@ -473,6 +474,26 @@ describe("stop-all-simulator-servers device scoping", () => { expect(registry.disposeService).not.toHaveBeenCalled(); }); + it("rejects a misspelled scope key instead of stripping it into a machine-wide sweep", async () => { + // `udids` is the natural slip: every sibling tool in this directory spells + // the device parameter `udid`. Under a stripping schema it left + // `params.devices` undefined, so the call fell through to the unscoped + // branch and tore down the other agent's devices while the caller believed + // it had scoped — and `unmatched` is unreachable on that path, so nothing + // in the response said otherwise. + const registry = createMockRegistry(twoAgentServices()); + const tool = createStopAllSimulatorServersTool(registry); + + const parsed = tool.zodSchema!.safeParse({ udids: [MINE] }); + + expect(parsed.success).toBe(false); + // And the same rejection reaches MCP / `argent run` / raw HTTP callers, + // which validate against the advertised JSON schema rather than the zod one. + expect(zodObjectToJsonSchema(tool.zodSchema as z.ZodObject)).toMatchObject({ + additionalProperties: false, + }); + }); + it("does not match a device id that is a prefix of another device's id", async () => { const services = new Map([ ["SimulatorServer:AAAA", { state: ServiceState.RUNNING, dependents: [] }], From 666b3cbc625364c971aa0012eae3973b42766aac Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 29 Jul 2026 11:51:12 +0200 Subject: [PATCH 14/98] test(flow): pin the lock map's self-cleanup, the one half nothing observed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation pass over the new tests found every one of them load-bearing except for a gap on the production side: deleting the whole `void held.then(...)` self-cleanup block left the entire test/flows suite green. The *condition* was already pinned — "still excludes a third acquirer" dies on an unconditional delete — but not the delete itself, which has no other observable effect: a retained lock entry behaves identically to a released one for every caller. So a host-wide singleton server could accumulate one permanent entry per flow anyone ever recorded with nothing to catch it. Needs a test-only accessor, since the map is module-private and deliberately invisible. The test asserts both directions, so it cannot pass by the map simply never being used: the count returns to its baseline after three record-append-finish cycles, and rises by exactly one while a lock is held. Verified by deleting the block again — this test alone goes red. --- .../tool-server/src/tools/flows/flow-utils.ts | 9 ++++++ .../flows/flow-concurrent-recording.test.ts | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 761fd83e3..00e67b841 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -426,6 +426,15 @@ export function __resetRecordingsForTesting(): void { flowFileLocks.clear(); } +/** + * How many flow files currently have a lock entry. Test-only: the map's + * self-cleanup is a leak backstop with no other observable effect, so nothing + * else can tell a released lock from a retained one. + */ +export function __flowFileLockCountForTesting(): number { + return flowFileLocks.size; +} + // ── Types ──────────────────────────────────────────────────────────── /** diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 2317c1296..cc51f3a61 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -19,6 +19,7 @@ import { parseFlow, serializeFlow, withFlowFileLock, + __flowFileLockCountForTesting, type FlowFile, type FlowStep, } from "../../src/tools/flows/flow-utils"; @@ -472,6 +473,35 @@ describe("the flow-file lock", () => { await heldC; expect(order).toEqual(["a-enter", "a-exit", "b-enter", "b-exit", "c-enter"]); }); + + it("drops the lock entry once released, so the map does not grow per flow ever recorded", async () => { + // The other half of the self-cleanup. The test above pins the CONDITION + // (only the tail may delete); this pins that the delete happens at all. + // Nothing else can observe it — a retained entry is functionally identical + // to a released one for every caller — so without this the whole + // `void held.then(...)` block can be deleted with the suite still green, + // and a long-lived host-wide server accumulates one permanent entry per + // flow anyone ever recorded. + const root = await makeRoot("lock-cleanup"); + const before = __flowFileLockCountForTesting(); + + for (const name of ["alpha", "beta", "gamma"]) { + await start(root, name); + await addEcho(root, name, "one"); + await finish(root, name); + } + expect(__flowFileLockCountForTesting()).toBe(before); + + // And while a lock is genuinely held, the entry IS there — so the + // assertion above is about release, not about the map never being used. + const gate = openGate(); + const held = withFlowFileLock(root, "alpha", () => gate.promise); + expect(__flowFileLockCountForTesting()).toBe(before + 1); + gate.open(); + await held; + await settle(); + expect(__flowFileLockCountForTesting()).toBe(before); + }); }); // ── What a READER of the flow file can observe ─────────────────────── From 35624da5dbb3cd9f5cd87a0dddf5944a59561858 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 29 Jul 2026 20:44:34 +0200 Subject: [PATCH 15/98] fix(flow): count the discarded take from disk, and correct what the docs claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A restart reported `discardedSteps` from the superseded session's in-memory flow, but in host mode it truncates the FILE. Hand-editing the .yaml mid-recording is a documented workflow, so the two diverge: four steps on disk were wiped and the agent was told it lost one. The count now comes from `countStepsOnDisk`, and when that file cannot be read or parsed no number is claimed at all — 0 would be the answer a genuinely empty take gives. `ChromiumJsRuntimeDebugger` joins DEVICE_OWNED_NAMESPACES. It cascades from `ChromiumCdp`, so it was already being torn down — silently, while `NetworkInspector` and `ReactProfilerSession` cascade the same way and were named. `stopped` is documented as the services that were live and got shut down, so the reporting is now symmetric. The rest is prose the code contradicts: - Vega does not "register no service at all": DEBUGGER_TOOL_CAPABILITY declares `vega: { vvd: true }`, so a Vega device owns `JsRuntimeDebugger` and `NetworkInspector` once debugger-connect or a network-log tool has run. - "Nothing here cascades" was denied 13 lines above it and again 18 below. Three blueprints declare getDependencies and all three are reachable by a cascade; listing them governs what `stopped` names, not whether they die. - An iOS session that only ran boot/launch/describe owns `NativeDevtools` too — both bootIos and launch-app's iOS handler resolve it unconditionally — so AXService closes a daemon leak, not a total gap. - `:tcp` is a shape `axServiceRef`/`nativeDevtoolsRef` can mint via `transport`, but no call site passes it and the remote host's forced-TCP decision happens inside the factory, after the URN is fixed. The coverage is defensive. - The tool-server is a singleton per install bundle, not per machine (`stateFileForBundle` hashes the bundle path, autospawn takes a free port). Two installs hold two recording maps and cannot see each other; what survives there is the temp-file swap, not the lock. - `argent flow list` does enumerate .argent/flows and filters on `.yaml` — that agreement is why a stray `.tmp` is invisible, not that nothing reads the dir. - The stop-tools comments narrated a "before" that only ever existed on this branch: at the merge base stop-all took no device id at all. - `renderToolArgs` referred to a "previous inline spelling" that exists only in the diff; the reason the body interpolates is now stated outright. - The isolation this map provides is of the artifact, not of the fact that a recording exists — the header claimed the latter while its own cases pin the disclosure. Tests: the client-mode case named for a superseded in-flight append restarted between calls, so the next append re-resolved the key and succeeded — the guard was never reached, and neutering it left the whole file green. It now parks an append in its live sub-tool call across the restart, and is the only case in that file that goes red when the guard is removed. --- packages/argent-mcp/src/mcp-server.ts | 2 +- packages/skills/rules/argent.md | 4 +- .../src/tools/flows/flow-finish-recording.ts | 11 ++- .../src/tools/flows/flow-start-recording.ts | 36 +++++-- .../tool-server/src/tools/flows/flow-utils.ts | 37 ++++++- .../src/tools/simulator/device-services.ts | 52 +++++++--- .../simulator/stop-all-simulator-servers.ts | 4 +- .../flows/flow-concurrent-recording.test.ts | 67 +++++++++++-- .../test/flows/flow-remote-recording.test.ts | 78 ++++++++++++++- .../tool-server/test/flows/flow-utils.test.ts | 5 +- packages/tool-server/test/stop-tools.test.ts | 98 ++++++++++++++----- 11 files changed, 327 insertions(+), 67 deletions(-) diff --git a/packages/argent-mcp/src/mcp-server.ts b/packages/argent-mcp/src/mcp-server.ts index dace59f45..ac9076c8f 100644 --- a/packages/argent-mcp/src/mcp-server.ts +++ b/packages/argent-mcp/src/mcp-server.ts @@ -241,7 +241,7 @@ export async function startMcpServer(options: StartMcpServerOptions): Promise/.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted, discardedSteps } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. Either way, re-record from the top rather than expecting to resume. +Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. \`discardedSteps\` counts the flow file as it stood at the reset, so a hand-edit made mid-recording is included — but it is omitted, \`restarted\` alone, when that file could not be read or parsed. Either way, re-record from the top rather than expecting to resume. Fails before anything is written on a \`project_root\` that is not absolute or contains a ".." segment, or a \`name\` outside letters/digits/underscore/hyphen. It can also fail on the .argent/flows/ directory not being creatable or the file not being writable - but only when the project root is on the tool-server host; against a remote client the YAML travels back in \`savedTo\` for the client to write and no host filesystem access happens. Recording state is independent: several flows can be recorded at once (different @@ -107,10 +109,23 @@ to remove or reorder steps.`, // lock, so a step from the take being discarded can neither slip into the // file between the reset and the swap, nor be written after both: it finds // its session superseded and fails instead. - const { savedTo, replaced } = await withFlowFileLock( + const { savedTo, replaced, discardedSteps } = await withFlowFileLock( params.project_root, params.name, async () => { + // Count the take BEFORE the truncate destroys it, and count it where it + // actually lives: on disk in host mode, since a hand-edit made + // mid-recording is part of the take and the session's in-memory copy + // only catches up on the next append (see {@link countStepsOnDisk}). In + // client mode this host has no file and the in-memory copy IS the take. + const previous = getRecordingSession(params.project_root, params.name); + const discardedSteps = + previous === undefined + ? undefined + : previous.persist === "host" + ? await countStepsOnDisk(previous.filePath) + : previous.flow.steps.length; + let savedTo: FlowSavedTo; if (persist === "host") { await writeNewFlowFile(filePath, flowFile); @@ -125,7 +140,7 @@ to remove or reorder steps.`, filePath, flow, }); - return { savedTo, replaced }; + return { savedTo, replaced, discardedSteps }; } ); @@ -133,7 +148,6 @@ to remove or reorder steps.`, // to fix it" workflow. Recordings are keyed per flow file, so starting a // *different* flow abandons nothing and there is nothing to report about it. if (replaced) { - const discardedSteps = replaced.flow.steps.length; // Only claim the file was reset when this process actually reset it. In // client mode the truncation happens only once the client applies the // directive, and a rejected path or a failed write there surfaces as @@ -144,13 +158,17 @@ to remove or reorder steps.`, ? `${filePath} reset to an empty flow.` : `${filePath} is reset to an empty flow once your client applies \`savedTo\` ` + `(a null \`savedTo\` means it did not).`; + // An unreadable or unparseable file leaves the loss genuinely uncounted, + // so say the take was discarded without putting a number on it rather + // than reporting one the file disagrees with. + const lost = + discardedSteps === undefined + ? "the previous take" + : `the previous take (${discardedSteps} step${discardedSteps === 1 ? "" : "s"})`; return { - message: - `Restarted recording "${params.name}" — the previous take ` + - `(${discardedSteps} step${discardedSteps === 1 ? "" : "s"}) was discarded and ` + - reset, + message: `Restarted recording "${params.name}" — ${lost} was discarded and ` + reset, restarted: true, - discardedSteps, + ...(discardedSteps === undefined ? {} : { discardedSteps }), flowFile, savedTo, }; diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 00e67b841..e71a8142e 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -233,9 +233,16 @@ export interface RecordingSession { * disclosure is bounded rather than absent. See the comment there for what it * discloses and why. * - * The tool-server is a host-wide singleton shared by every MCP client, subagent - * and CLI call on the machine, so this map is the only thing standing between - * two agents and a clobbered flow file. + * One tool-server serves every MCP client, subagent and CLI call using one + * argent install — `stateFileForBundle` gives each install its own record and + * autospawn takes a free port, so the singleton is per install bundle, not per + * machine. Within that scope this map is the only thing standing between two + * agents and a clobbered flow file. Across it there is nothing: two installs + * recording the same (project_root, name) hold two of these maps and cannot see + * each other, so each believes its own session is live while the other + * truncates and appends. What still holds there is {@link writeFlowFile}'s + * temp-file swap, which is a filesystem guarantee rather than an in-process + * one — each write stays whole, but a lost update is not prevented. */ const recordings = new Map(); @@ -2433,6 +2440,30 @@ export async function writeNewFlowFile(filePath: string, content: string): Promi await writeFlowFile(filePath, content); } +/** + * How many steps the flow file currently holds, or undefined if it cannot be + * read or parsed. + * + * For counting what a truncate is about to destroy. The file — not the + * session's in-memory `flow` — is the take in "host" mode: {@link appendStep} + * re-reads it before every append and `flow-finish-recording` reads it back for + * its summary, so a hand-edit made mid-recording (a documented workflow) is + * part of the take even though the in-memory copy only catches up on the next + * append. + * + * Undefined rather than 0 on a failure, because the two are not the same + * answer: a hand-edit can leave YAML that `parseFlow` rejects, and reporting + * "0 steps discarded" there would understate the loss in exactly the case that + * caused it. The caller reports no count instead. + */ +export async function countStepsOnDisk(filePath: string): Promise { + try { + return parseFlow(await fs.readFile(filePath, "utf8")).steps.length; + } catch { + return undefined; + } +} + /** Read and parse the flow file, append a step, write it back. */ export async function appendStep(filePath: string, step: FlowStep): Promise { const content = await fs.readFile(filePath, "utf8"); diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index 8f24845f5..7172a2c57 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -2,6 +2,7 @@ import { SIMULATOR_SERVER_NAMESPACE } from "../../blueprints/simulator-server"; import { NATIVE_DEVTOOLS_NAMESPACE } from "../../blueprints/native-devtools"; import { ANDROID_DEVTOOLS_NAMESPACE } from "../../blueprints/android-devtools"; import { CHROMIUM_CDP_NAMESPACE } from "../../blueprints/chromium-cdp"; +import { CHROMIUM_JS_RUNTIME_DEBUGGER_NAMESPACE } from "../../blueprints/chromium-js-runtime-debugger"; import { TV_CONTROL_NAMESPACE } from "../../blueprints/tv-control"; import { ANDROID_TV_CONTROL_NAMESPACE } from "../../blueprints/android-tv-control"; import { AX_SERVICE_NAMESPACE } from "../../blueprints/ax-service"; @@ -26,9 +27,15 @@ import { REACT_PROFILER_SESSION_NAMESPACE } from "../../blueprints/react-profile */ /** - * Every discriminator a device-scoped URN appends AFTER the device id - * (`NativeDevtools::tcp` and `AXService::tcp` are the only two; - * every other URN in {@link DEVICE_OWNED_NAMESPACES} ends at the device id). + * Every discriminator a device-scoped URN appends AFTER the device id. Only + * `:tcp` exists, and only two namespaces can ever emit it: `axServiceRef` and + * `nativeDevtoolsRef` append it for `transport: "tcp"`. No call site passes + * that option today — including the ios-remote branches, and the remote host's + * forced-TCP decision is made inside the factory, after the ref has already + * fixed the URN — so `:tcp` is a shape the refs can mint rather than one + * production currently produces. Matched anyway so the two stop tools cannot + * drift apart again the moment a caller does pass it. + * * Enumerated rather than matched as "anything after a colon", because a device * id can itself end in `:`: an adb serial over wifi is * `192.168.1.5:5555`, so a suffix wildcard would let the bare `192.168.1.5` @@ -49,9 +56,12 @@ const URN_SUFFIXES = ["", ":tcp"] as const; */ const PORT_KEYED_NAMESPACES: readonly string[] = [ JS_RUNTIME_DEBUGGER_NAMESPACE, - // Both of these declare `getDependencies -> JsRuntimeDebugger:`, so - // disposing the debugger already cascades to them. Listed anyway so ownership - // is recognized even if only one of them is live. + // Both declare `getDependencies -> JsRuntimeDebugger:`, so neither + // can be in a snapshot without it and neither adds any ownership the debugger + // entry does not already establish. They are listed for what `stopped` + // reports: a session that had a network inspector or a React profiler open is + // told those went away by name, rather than inferring it from the debugger + // line. `ChromiumJsRuntimeDebugger` is listed for the same reason. NETWORK_INSPECTOR_NAMESPACE, REACT_PROFILER_SESSION_NAMESPACE, ]; @@ -59,18 +69,28 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ /** * Every namespace whose service belongs to exactly one device and whose * `dispose()` frees something worth freeing. A device owning none of these is - * not a bad id — Vega is driven entirely by CLI/adb shell-outs and registers no - * service at all. + * not a bad id: Vega is driven by `vega` CLI shell-outs for boot, launch, + * describe, screenshot and the remote, so a Vega device owns a service only + * once `debugger-connect` or a network-log tool has run — `DEBUGGER_TOOL_CAPABILITY` + * declares `vega: { vvd: true }`, and those two namespaces (`JsRuntimeDebugger`, + * `NetworkInspector`) are the only ones on this list a Vega serial can ever + * match. * * Membership is decided by "does dispose() reap a resource that outlives the - * call", because nothing here cascades: of all the blueprints, only - * NetworkInspector, ReactProfilerSession and ChromiumJsRuntimeDebugger declare - * `getDependencies`, so a namespace left out of this list is simply never torn - * down by a session-end stop. + * call", and every namespace that meets that test is listed even when a cascade + * would already have reached it. Three blueprints declare `getDependencies` — + * NetworkInspector and ReactProfilerSession on `JsRuntimeDebugger`, + * ChromiumJsRuntimeDebugger on `ChromiumCdp` — and teardown runs + * dependency → dependents, so all three can arrive via a cascade. Listing them + * is about what `stopped` names, not about whether they die: an unlisted + * dependent is torn down silently, which contradicts what the tool documents + * `stopped` to be. * * - `AXService` owns the in-sim ax daemon (spawned `--timeout 3600`) and its - * socket. An iOS session that only ran boot/launch/describe owns this and - * nothing else. + * socket, and is the only entry that reaps it. An iOS session that only ran + * boot/launch/describe also owns `NativeDevtools` — `bootIos` and + * `launch-app`'s iOS handler both resolve it unconditionally — so leaving + * `AXService` out would not orphan the device, just that daemon. * - `TvControl` owns two spawned `--timeout 3600` daemons. * - `ScreenRecordingSession` owns an ffmpeg child, an MJPEG frame stream, and * the touch-visualizer overlay it enabled on the device. @@ -80,14 +100,14 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ * socket to Metro, and a log file handle. * * (`AndroidTvControl` is stateless adb shell-outs with a no-op dispose, but is - * included for symmetry so the snapshot is fully drained. ChromiumJsRuntimeDebugger - * is omitted deliberately: it cascades from `ChromiumCdp`, which is listed.) + * included for symmetry so the snapshot is fully drained.) */ export const DEVICE_OWNED_NAMESPACES: readonly string[] = [ SIMULATOR_SERVER_NAMESPACE, NATIVE_DEVTOOLS_NAMESPACE, ANDROID_DEVTOOLS_NAMESPACE, CHROMIUM_CDP_NAMESPACE, + CHROMIUM_JS_RUNTIME_DEBUGGER_NAMESPACE, TV_CONTROL_NAMESPACE, ANDROID_TV_CONTROL_NAMESPACE, AX_SERVICE_NAMESPACE, diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index d06da97d4..3070b732c 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -56,8 +56,8 @@ export function createStopAllSimulatorServersTool( `Failed to stop simulator servers: ${failureSignal.error_code}`, }, description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions and JS-runtime debugger sessions - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. -PASS \`devices\` with the device ids this session used — the tool-server is a host-wide singleton shared with every other agent and CLI call on the machine, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. -Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a device driven only through CLI/adb shell-outs (Vega) registers no service and always lands here, and so does a real device this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, +PASS \`devices\` with the device ids this session used — one tool-server serves every agent, subagent and CLI call using this argent install, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI shell-outs and registers nothing until \`debugger-connect\` or a network-log tool has run, so one you only booted and drove with the remote always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index cc51f3a61..09ffd470f 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -25,10 +25,12 @@ import { } from "../../src/tools/flows/flow-utils"; /** - * Concurrency contract of the recording tools. The tool-server is a host-wide - * singleton shared by every MCP client, subagent and CLI call on the machine, - * so several agents can legitimately be recording at the same moment — in one - * project or across projects. A recording is identified by its + * Concurrency contract of the recording tools. One tool-server serves every MCP + * client, subagent and CLI call using one argent install, so several agents can + * legitimately be recording at the same moment — in one project or across + * projects. (Two INSTALLS run two servers and two recording maps; nothing here + * covers that, and nothing can — see the note on `recordings` in flow-utils.) + * A recording is identified by its * (project_root, name) key, and these tests assert the ISOLATION that follows: * one recording's steps never land in another's file, addressing a key that * isn't live fails loudly (naming the ones that are), replaying a flow @@ -557,8 +559,10 @@ describe("flow-file writes as seen by a concurrent reader", () => { }); it("leaves no scratch file behind in the flows directory", async () => { - // The swap writes a sibling temp file first. Nothing enumerates this - // directory today, but a leftover must not accumulate per append either. + // The swap writes a sibling temp file first. `argent flow list` enumerates + // this directory and filters on `.yaml`, so a stray `.tmp` never surfaces + // as a flow — but that agreement only hides a leftover, it does not stop + // one accumulating per append. const root = await makeRoot("no-scratch"); await start(root, "alpha"); expect(await strayFiles(root, "alpha")).toEqual([]); @@ -785,6 +789,57 @@ describe("restarting a recording on one key", () => { expect(await readMarkers(rootA, "alpha")).toEqual(["tool:a1"]); expect(getRecordingSession(rootA, "alpha")?.flow.steps).toHaveLength(1); }); + + it("counts the steps the FILE held, not the ones this session appended", async () => { + // Hand-editing the .yaml mid-recording is a documented workflow, and in + // host mode the file is the take: every other host-mode operation re-reads + // it, and the in-memory copy only catches up on the next append. The + // restart is the one destructive operation, so counting from memory would + // report a fraction of what it just wiped. + const root = await makeRoot("restart-handedit"); + + await start(root, "alpha"); + await addEcho(root, "alpha", "a1"); + + await fs.writeFile( + flowPath(root, "alpha"), + serializeFlow({ + executionPrerequisite: "", + steps: [ + { kind: "echo", message: "a1" }, + { kind: "echo", message: "by-hand-2" }, + { kind: "echo", message: "by-hand-3" }, + { kind: "echo", message: "by-hand-4" }, + ], + }), + "utf8" + ); + + const restarted = await start(root, "alpha"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(4); + expect(restarted.message).toContain("(4 steps)"); + expect(await readMarkers(root, "alpha")).toEqual([]); + }); + + it("reports no count at all when the file it discarded could not be parsed", async () => { + // A hand-edit can also leave YAML `parseFlow` rejects. There is no honest + // number then — and 0 is the least honest of all, since it is the answer a + // genuinely empty take gives. `restarted` alone says the take is gone. + const root = await makeRoot("restart-unparseable"); + + await start(root, "alpha"); + await addEcho(root, "alpha", "a1"); + await fs.writeFile(flowPath(root, "alpha"), "steps: [ this: is: not: a: flow\n", "utf8"); + + const restarted = await start(root, "alpha"); + expect(restarted.restarted).toBe(true); + expect(restarted).not.toHaveProperty("discardedSteps"); + expect(restarted.message).not.toMatch(/\d+ steps?\)/); + expect(restarted.message).toContain("the previous take was discarded"); + // The reset still happened — the unreadable take is gone either way. + expect(await readMarkers(root, "alpha")).toEqual([]); + }); }); // ── A restart landing on top of an in-flight append ────────────────── diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index 3d751c7b5..43d5c0a74 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -3,7 +3,12 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import type { Registry, ToolContext } from "@argent/registry"; -import { ArtifactStore, CLIENT_FILE_MARKER } from "@argent/registry"; +import { + ArtifactStore, + CLIENT_FILE_MARKER, + FAILURE_CODES, + getFailureSignal, +} from "@argent/registry"; import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording"; import { flowInsertEchoTool } from "../../src/tools/flows/flow-insert-echo"; @@ -846,7 +851,7 @@ describe("concurrent recordings against a remote client", () => { expect(contents.map((c) => c.length).sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5, 6]); }); - it("fails an append whose remote recording was restarted, and writes nothing to this host", async () => { + it("starts the remote take over on a restart, discarding the previous one and writing nothing to this host", async () => { await flowStartRecordingTool.execute( {}, { name: "remote-flow", project_root: CLIENT_ROOT }, @@ -881,4 +886,73 @@ describe("concurrent recordings against a remote client", () => { // Still nothing on this host: the client's root was never created here. await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow(); }); + + it("rejects a remote append that was already in flight when the restart landed", async () => { + // The case above restarts BETWEEN calls, so the next append re-resolves the + // key and legitimately gets the new session — the supersede guard is never + // reached. Reaching it needs an append that resolved its session before the + // restart and lands after, and in client mode the live sub-tool call is the + // only await that can hold one open across it: past that point the client + // path (push → validate → serialize) runs to completion synchronously. + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let arrive!: () => void; + const reached = new Promise((resolve) => { + arrive = resolve; + }); + const registry = { + invokeTool: vi.fn(async () => { + arrive(); + await held; + return { tapped: true }; + }), + getTool: vi.fn(() => undefined), + } as unknown as Registry; + const addStep = createFlowAddStepTool(registry); + + await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT }, + remoteCtx() + ); + + const inFlight = addStep.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, command: "tap", args: '{"x":0.5}' } + ); + await reached; // parked in the live step, session already resolved + + const restarted = await flowStartRecordingTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT }, + remoteCtx() + ); + expect(restarted).toMatchObject({ restarted: true }); + + release(); + let caught: unknown; + try { + await inFlight; + throw new Error("expected the superseded append to fail"); + } catch (err) { + caught = err; + } + expect((caught as Error).message).toMatch(/no longer active/); + expect(getFailureSignal(caught)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + + // The step ran on the device but never entered the new take, and the client + // is told so — in client mode the in-memory copy is the only copy, so a + // superseded step landing in it would be unrecoverable. + const after = await flowInsertEchoTool.execute( + {}, + { name: "remote-flow", project_root: CLIENT_ROOT, message: "second take" } + ); + const flow = parseFlow((after.savedTo as { content: string }).content); + expect(flow.steps).toHaveLength(1); + expect(flow.steps[0]).toMatchObject({ kind: "echo", message: "second take" }); + + await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow(); + }); }); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index a6f8de6f9..38d24787b 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1012,7 +1012,10 @@ describe("native launch shorthand", () => { // Recordings live in a map keyed by the resolved flow file path, so a session // has no identity beyond (project_root, name) — two agents recording at once -// must never observe each other's state. +// must never write into each other's take. What is isolated is the artifact, +// not the fact that a recording exists: the not-found message deliberately +// names the other live flows in the caller's own project and counts the rest, +// and the two cases below pin that disclosure as bounded rather than absent. describe("recording sessions", () => { beforeEach(() => { __resetRecordingsForTesting(); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 5839fcc04..8b95b7d04 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -19,7 +19,16 @@ function createMockRegistry(services: Map { const node = services.get(urn); - if (node) node.state = ServiceState.IDLE; + if (!node || node.state === ServiceState.IDLE) return; + node.state = ServiceState.IDLE; + // …and it cascades dependency → dependents first (Registry._teardown), so + // a service whose dependency is disposed goes down with it. Mirror that + // too, or a test cannot tell a namespace this tool reaps by name from one + // that merely dies as somebody else's dependent. + for (const dependent of node.dependents) { + const child = services.get(dependent); + if (child) child.state = ServiceState.IDLE; + } }), } as unknown as Registry; } @@ -143,13 +152,16 @@ describe("stop-simulator-server", () => { expect(registry.disposeService).toHaveBeenCalledWith("ChromiumCdp:chromium-cdp-9222"); }); - // Both stop tools now resolve "which services does this device own" through - // one shared matcher. Before that, this tool looked its URNs up with an exact, - // case-sensitive `services.get()` — so the two disagreed about the same id. + // Both stop tools resolve "which services does this device own" through one + // shared matcher. This tool used to look its URNs up with an exact, + // case-sensitive `services.get()`, which no-op'd on a mis-cased udid; stop-all + // took no device id at all and swept every matching namespace on the host, so + // there was no second opinion to compare against. Now that stop-all is scoped, + // the same spelling has to reach the same services through both. it("matches a UDID case-insensitively, like the scoped stop-all does", async () => { - // Agents pass through whatever spelling they were handed. A case mismatch - // silently no-op'd here while stop-all reaped the same device. + // Agents pass through whatever spelling they were handed, and a case + // mismatch must not silently turn a scoped stop into a no-op. const services = new Map([ ["SimulatorServer:AAAA-BBBB", { state: ServiceState.RUNNING, dependents: [] }], ]); @@ -211,11 +223,9 @@ describe("stop-all-simulator-servers", () => { const services = new Map([ ["SimulatorServer:AAA", { state: ServiceState.RUNNING, dependents: [] }], ["SimulatorServer:BBB", { state: ServiceState.RUNNING, dependents: [] }], - // Deliberately excluded from the namespace set: it declares - // `getDependencies -> ChromiumCdp:`, so the registry cascades to it - // when that transport is disposed. Listing it too would be redundant, and - // disposing it directly here would claim a `stopped` entry for a service - // no device in this snapshot owns a transport for. + // A device-owned namespace like any other: a session that ran + // debugger-connect against a Chromium app owns it, and the sweep drains + // it whether or not its transport happens to be in the same snapshot. ["ChromiumJsRuntimeDebugger:CCC", { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); @@ -224,13 +234,49 @@ describe("stop-all-simulator-servers", () => { const result = await tool.execute!({}, {}); expect(result).toEqual({ - stopped: ["SimulatorServer:AAA", "SimulatorServer:BBB"], + stopped: ["SimulatorServer:AAA", "SimulatorServer:BBB", "ChromiumJsRuntimeDebugger:CCC"], }); - expect(registry.disposeService).toHaveBeenCalledTimes(2); + expect(registry.disposeService).toHaveBeenCalledTimes(3); expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:AAA"); expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:BBB"); }); + it("names a cascading debugger in `stopped` instead of letting it die anonymously", async () => { + // `stopped` is documented as "the services that were actually live and got + // shut down". ChromiumJsRuntimeDebugger declares `getDependencies -> + // ChromiumCdp`, so disposing the transport takes it down regardless — while + // it was outside the namespace set, that shutdown was invisible, and an + // agent reading `stopped` was not told its console history was gone. The + // registry inserts a dependent before its dependency (`_resolve` creates + // the node, then `_initialize` resolves what it needs), so the map order + // here is the real one. + const services = new Map([ + [ + "ChromiumJsRuntimeDebugger:chromium-cdp-9222", + { state: ServiceState.RUNNING, dependents: [] }, + ], + [ + "ChromiumCdp:chromium-cdp-9222", + { + state: ServiceState.RUNNING, + dependents: ["ChromiumJsRuntimeDebugger:chromium-cdp-9222"], + }, + ], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: ["chromium-cdp-9222"] }); + + expect(result).toEqual({ + stopped: ["ChromiumJsRuntimeDebugger:chromium-cdp-9222", "ChromiumCdp:chromium-cdp-9222"], + }); + expect(services.get("ChromiumJsRuntimeDebugger:chromium-cdp-9222")?.state).toBe( + ServiceState.IDLE + ); + expect(services.get("ChromiumCdp:chromium-cdp-9222")?.state).toBe(ServiceState.IDLE); + }); + it("returns empty list when no simulators are running", async () => { const services = new Map(); const registry = createMockRegistry(services); @@ -295,9 +341,9 @@ describe("stop-all-simulator-servers", () => { }); }); -// The tool-server is a host-wide singleton, so an unscoped teardown reaps -// whatever device another agent is mid-session on. `devices` narrows the sweep -// to the ids the calling session actually used. +// One tool-server serves every agent using one argent install, so an unscoped +// teardown reaps whatever device another agent is mid-session on. `devices` +// narrows the sweep to the ids the calling session actually used. const MINE = "AAAA-1111"; const THEIRS = "BBBB-2222"; @@ -524,11 +570,12 @@ describe("stop-all-simulator-servers device scoping", () => { }); describe("stop-all-simulator-servers unmatched ids", () => { - // A scoped stop whose ids owned nothing used to return a bare `{ stopped: [] }` - // — byte-identical to the answer on a genuinely clean machine. So a mistyped - // id, a device *name* passed where an id was expected, or an empty string all - // read as success while the services they were meant to reap (on tvOS, two - // spawned --timeout 3600 daemons) stayed running. `unmatched` names them. + // Without `unmatched`, a scoped stop whose ids owned nothing answers with a + // bare `{ stopped: [] }` — byte-identical to the answer on a genuinely clean + // machine. A mistyped id, a device *name* passed where an id was expected, or + // an empty string would all read as success while the services they were + // meant to reap (on tvOS, two spawned --timeout 3600 daemons) stayed running. + // `unmatched` names them, so scoping cannot fail silently. it("names an unknown id in unmatched while still stopping the live device", async () => { const services = new Map([ @@ -547,7 +594,7 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(registry.disposeService).toHaveBeenCalledTimes(2); }); - it("reports a typo, a device name, and an empty-string id — the shapes that used to look clean", async () => { + it("reports a typo, a device name, and an empty-string id — the shapes that would otherwise look clean", async () => { const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], ]); @@ -648,7 +695,12 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); it("scopes the tcp-transport AXService URN to its own device", async () => { - // ios-remote gives AXService the same `:tcp` suffix NativeDevtools uses. + // `axServiceRef(device, { transport: "tcp" })` appends `:tcp`, exactly as + // `nativeDevtoolsRef` does. No call site passes that option today — the + // remote host's forced-TCP decision happens inside the factory, after the + // ref has fixed the URN — so this is a shape the ref can mint rather than + // one production currently produces, and the coverage is defensive: the + // matcher must not start splitting a device id on ":" if one ever does. const services = new Map([ [`AXService:${MINE}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], [`AXService:${THEIRS}:tcp`, { state: ServiceState.RUNNING, dependents: [] }], From 0da027540f32ad8d161fa26a92220c2cdca5ed12 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 29 Jul 2026 21:07:52 +0200 Subject: [PATCH 16/98] fix(flow): qualify what the disk count promises, and stop the mock from hiding a cascade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A five-lens sweep over the previous commit found the fixes had defects of their own. `flow-start-recording`'s new description asserted the disk semantics with no mode qualifier, but the count only comes from disk in HOST mode — against a remote client this host never sees the file, so it still counts the steps recorded through the server and a hand-edit on the client is invisible. That is verbatim the failure the disk fix was written to remove, still live on the client path and now mis-promised. Qualified in the tool description, the skill doc and `countStepsOnDisk` itself. `stop-tools.test.ts`'s mock handed the tool the LIVE service map, while the real `getSnapshot` copies each node. With the cascade added last commit, a disposal retroactively rewrote the state the sweep was still iterating, so the answer depended on map insertion order — an artifact production does not have, and one that would give a future test a false red. The mock now copies and recurses the way `_teardown` does, and the cascade case runs under both orders, asserting membership rather than order. Two Vega claims were wrong in the other direction: only boot and launch go through the `vega` CLI (describe, screenshot and tv-remote are adb-only, and say so in their own source), and "the only namespaces a Vega serial can match" is falsified by ERROR nodes — `registry.invokeTool` does not enforce a tool's capability, only the HTTP layer does, so a call that reaches the registry another way leaves a node behind under whatever namespace it resolved. Reproduced: `SimulatorServer:` in ERROR, matched and counted. Also: the `devices` param's own `.describe()` still said "shared by every agent on the host" 22 lines above the corrected sentence, and it ships in the schema `GET /tools` advertises; the prose list of what gets stopped never mentioned the network inspector or React profiler sessions, which `stopped` names; the `ChromiumJsRuntimeDebugger` rationale landed inside the `PORT_KEYED_NAMESPACES` literal, where acting on it would have moved the namespace into the wrong array and matched no device at all. `countStepsOnDisk` was public API covered only through two tool-level tests; it now has direct cases for the distinction it exists to make — 0 for an empty take, undefined for one that cannot be read. --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-start-recording.ts | 2 +- .../tool-server/src/tools/flows/flow-utils.ts | 10 ++- .../src/tools/simulator/device-services.ts | 29 +++++-- .../simulator/stop-all-simulator-servers.ts | 6 +- .../flows/flow-concurrent-recording.test.ts | 4 +- .../tool-server/test/flows/flow-utils.test.ts | 66 +++++++++++++- packages/tool-server/test/stop-tools.test.ts | 86 +++++++++++-------- 8 files changed, 150 insertions(+), 55 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 8e556f1ea..7a972cb28 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -140,7 +140,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Every step runs live.** You see the real tool result (including screenshots) — verify the step worked before continuing. **Only successful steps are recorded**: a failed call writes nothing to the flow file; fix the issue and try again. - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. -- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` + `discardedSteps` report only a discarded _in-memory_ take; their **absence does not mean nothing was overwritten**. Starting a _different_ flow abandons nothing. +- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` counts the `.yaml` as it stood at the reset — a hand-edit made mid-recording is included — and is omitted entirely when that file could not be read or parsed, so `restarted: true` can arrive without it. That is for a project root on the tool-server host; against a remote client the host never sees your `.yaml`, so there the number counts only the steps recorded through the server. Starting a _different_ flow abandons nothing. - **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. - **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished, superseded by another agent, or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 0c7e5ba91..3ad1b2b11 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -65,7 +65,7 @@ export const flowStartRecordingTool: ToolDefinition< description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory. Use when you want to capture a reusable sequence of device interactions for later replay. Returns { message, flowFile, savedTo }. -Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. \`discardedSteps\` counts the flow file as it stood at the reset, so a hand-edit made mid-recording is included — but it is omitted, \`restarted\` alone, when that file could not be read or parsed. Either way, re-record from the top rather than expecting to resume. +Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. \`discardedSteps\` counts the flow file as it stood at the reset, so a hand-edit made mid-recording is included — but it is omitted, \`restarted\` alone, when that file could not be read or parsed. That holds when the project root is on the tool-server host; against a REMOTE client the host never sees the file, so there the count is of the steps recorded through this server and a hand-edit on your machine is not in it. Either way, re-record from the top rather than expecting to resume. Fails before anything is written on a \`project_root\` that is not absolute or contains a ".." segment, or a \`name\` outside letters/digits/underscore/hyphen. It can also fail on the .argent/flows/ directory not being creatable or the file not being writable - but only when the project root is on the tool-server host; against a remote client the YAML travels back in \`savedTo\` for the client to write and no host filesystem access happens. Recording state is independent: several flows can be recorded at once (different diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index e71a8142e..9c4b102fd 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2444,8 +2444,14 @@ export async function writeNewFlowFile(filePath: string, content: string): Promi * How many steps the flow file currently holds, or undefined if it cannot be * read or parsed. * - * For counting what a truncate is about to destroy. The file — not the - * session's in-memory `flow` — is the take in "host" mode: {@link appendStep} + * For counting what a truncate is about to destroy, and therefore only ever + * called in "host" mode. In "client" mode the file lives on the client's + * machine and this host cannot read it at all, so the in-memory copy is both + * the take and the only thing countable — the guarantee below does not carry + * across that boundary, and `flow-start-recording`'s description says so. + * + * The file — not the session's in-memory `flow` — is the take in "host" mode: + * {@link appendStep} * re-reads it before every append and `flow-finish-recording` reads it back for * its summary, so a hand-edit made mid-recording (a documented workflow) is * part of the take even though the in-memory copy only catches up on the next diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index 7172a2c57..a79da9e28 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -61,7 +61,7 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ // entry does not already establish. They are listed for what `stopped` // reports: a session that had a network inspector or a React profiler open is // told those went away by name, rather than inferring it from the debugger - // line. `ChromiumJsRuntimeDebugger` is listed for the same reason. + // line. NETWORK_INSPECTOR_NAMESPACE, REACT_PROFILER_SESSION_NAMESPACE, ]; @@ -69,12 +69,21 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ /** * Every namespace whose service belongs to exactly one device and whose * `dispose()` frees something worth freeing. A device owning none of these is - * not a bad id: Vega is driven by `vega` CLI shell-outs for boot, launch, - * describe, screenshot and the remote, so a Vega device owns a service only - * once `debugger-connect` or a network-log tool has run — `DEBUGGER_TOOL_CAPABILITY` - * declares `vega: { vvd: true }`, and those two namespaces (`JsRuntimeDebugger`, - * `NetworkInspector`) are the only ones on this list a Vega serial can ever - * match. + * not a bad id: Vega is driven by shell-outs — the `vega` CLI for boot and + * launch, adb for describe, screenshot and the remote — so a Vega device owns a + * RUNNING service only once `debugger-connect` or a network-log tool has run. + * `DEBUGGER_TOOL_CAPABILITY` declares `vega: { vvd: true }`, and those two + * (`JsRuntimeDebugger`, `NetworkInspector`) are the only entries here a Vega + * serial can hold live. + * + * It can still MATCH others, because ownership is counted regardless of state + * and a failed resolve leaves its node behind in ERROR (`Registry._resolve` + * inserts before the factory runs, and nothing is ever removed). A tool's + * capability is enforced by the HTTP layer, not by `registry.invokeTool`, so a + * call that reaches the registry another way — `flow-add-step` takes `command` + * as a bare string — can mint e.g. `SimulatorServer:` in ERROR. A + * Vega serial appearing in `stopped` is therefore impossible, but one absent + * from `unmatched` is not. * * Membership is decided by "does dispose() reap a resource that outlives the * call", and every namespace that meets that test is listed even when a cascade @@ -98,6 +107,12 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ * on-device perfetto process plus its trace file. * - `JsRuntimeDebugger` owns a bound loopback HTTP/WebSocket server, the CDP * socket to Metro, and a log file handle. + * - `ChromiumJsRuntimeDebugger` owns the same, plus its captured console + * history. It is the one member whose dependency (`ChromiumCdp`) is also + * here, so a scoped stop reaches it twice over — it is listed for the naming, + * which is the same reason the two `JsRuntimeDebugger` dependents are. + * Note its URN is `:`, NOT port-keyed, so it belongs in this + * list and not in {@link PORT_KEYED_NAMESPACES}. * * (`AndroidTvControl` is stateless adb shell-outs with a no-op dispose, but is * included for symmetry so the snapshot is fully drained.) diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 3070b732c..44a0c9df4 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -9,7 +9,7 @@ const zodSchema = z .array(z.string()) .optional() .describe( - "Device ids (iOS UDID / Android serial / Chromium id) to scope the teardown to — pass the devices THIS session actually used. Omit only for a deliberate machine-wide cleanup: the tool-server is shared by every agent on the host, so an unscoped stop also kills devices another agent is mid-session on." + "Device ids (iOS UDID / Android serial / Chromium id) to scope the teardown to — pass the devices THIS session actually used. Omit only for a deliberate machine-wide cleanup: one tool-server serves every agent using this argent install, so an unscoped stop also kills devices another agent is mid-session on." ), }) // `.strict()` because omitting `devices` is the machine-wide sweep, so a @@ -55,9 +55,9 @@ export function createStopAllSimulatorServersTool( failedMsg: ({ failureSignal }) => `Failed to stop simulator servers: ${failureSignal.error_code}`, }, - description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions and JS-runtime debugger sessions - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. + description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions, and JS-runtime debugger sessions along with the network inspectors and React profiler sessions that ride on them - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — one tool-server serves every agent, subagent and CLI call using this argent install, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. -Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI shell-outs and registers nothing until \`debugger-connect\` or a network-log tool has run, so one you only booted and drove with the remote always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 09ffd470f..65d6cff34 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -482,8 +482,8 @@ describe("the flow-file lock", () => { // Nothing else can observe it — a retained entry is functionally identical // to a released one for every caller — so without this the whole // `void held.then(...)` block can be deleted with the suite still green, - // and a long-lived host-wide server accumulates one permanent entry per - // flow anyone ever recorded. + // and a long-lived server accumulates one permanent entry per flow anyone + // using that argent install ever recorded. const root = await makeRoot("lock-cleanup"); const before = __flowFileLockCountForTesting(); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 38d24787b..51888d2a0 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1,7 +1,10 @@ -import { describe, it, expect, beforeEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; import * as path from "node:path"; import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; import { + countStepsOnDisk, serializeFlow, parseFlow, describeSelector, @@ -1705,3 +1708,64 @@ describe("sibling selector scopes and the universal selector", () => { ).toThrow(/secret/); }); }); + +// ── countStepsOnDisk ───────────────────────────────────────────────── + +// The count `flow-start-recording` reports for a take it is about to truncate. +// Its contract is the distinction between "0 steps" and "no answer": an empty +// take really did hold nothing, while an unreadable one is a loss of unknown +// size, and reporting the first for the second understates it in exactly the +// case that produced it. +describe("countStepsOnDisk", () => { + let dir: string; + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "count-steps-")); + }); + + afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); + }); + + const write = async (content: string) => { + const file = path.join(dir, "flow.yaml"); + await fs.writeFile(file, content, "utf8"); + return file; + }; + + it("counts the steps a readable flow file holds", async () => { + const file = await write( + serializeFlow({ + executionPrerequisite: "", + steps: [ + { kind: "echo", message: "one" }, + { kind: "echo", message: "two" }, + { kind: "echo", message: "three" }, + ], + }) + ); + expect(await countStepsOnDisk(file)).toBe(3); + }); + + it("counts an empty take as 0, which is a real answer", async () => { + const file = await write(serializeFlow({ executionPrerequisite: "", steps: [] })); + expect(await countStepsOnDisk(file)).toBe(0); + }); + + it("returns undefined for a file that does not exist", async () => { + expect(await countStepsOnDisk(path.join(dir, "absent.yaml"))).toBeUndefined(); + }); + + it("returns undefined rather than 0 for YAML the parser rejects", async () => { + // A hand-edit can leave this behind, and `parseFlow("")` returning an empty + // flow with no error is the reason 0 cannot double as "unknown". + const file = await write("steps: [ this: is: not: a: flow\n"); + expect(await countStepsOnDisk(file)).toBeUndefined(); + }); + + it("returns undefined for a directory in the file's place", async () => { + const asDir = path.join(dir, "flow-dir.yaml"); + await fs.mkdir(asDir); + expect(await countStepsOnDisk(asDir)).toBeUndefined(); + }); +}); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 8b95b7d04..a58b25753 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -7,8 +7,16 @@ import { stopMetroTool } from "../src/tools/simulator/stop-metro"; function createMockRegistry(services: Map) { return { + // The real `getSnapshot` COPIES each node into a fresh map + // (Registry.getSnapshot), so a disposal during the sweep cannot rewrite the + // state the caller is still iterating. Handing over the live map instead + // would make a cascade retroactively hide its own victim, and the result + // would depend on the map's insertion order — an artifact of the mock that + // production does not have. getSnapshot: vi.fn(() => ({ - services, + services: new Map( + [...services].map(([urn, n]) => [urn, { ...n, dependents: [...n.dependents] }]) + ), namespaces: [], tools: [], })), @@ -17,18 +25,16 @@ function createMockRegistry(services: Map { + disposeService: vi.fn(async function dispose(urn: string) { const node = services.get(urn); if (!node || node.state === ServiceState.IDLE) return; + // …and it recurses into dependents BEFORE clearing the node + // (Registry._teardown), so a service whose dependency is disposed goes + // down with it. Mirror that too, or a test cannot tell a namespace this + // tool reaps by name from one that merely dies as somebody else's + // dependent. Mark first, so a dependency cycle cannot recurse forever. node.state = ServiceState.IDLE; - // …and it cascades dependency → dependents first (Registry._teardown), so - // a service whose dependency is disposed goes down with it. Mirror that - // too, or a test cannot tell a namespace this tool reaps by name from one - // that merely dies as somebody else's dependent. - for (const dependent of node.dependents) { - const child = services.get(dependent); - if (child) child.state = ServiceState.IDLE; - } + for (const dependent of node.dependents) await dispose(dependent); }), } as unknown as Registry; } @@ -241,40 +247,44 @@ describe("stop-all-simulator-servers", () => { expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:BBB"); }); - it("names a cascading debugger in `stopped` instead of letting it die anonymously", async () => { - // `stopped` is documented as "the services that were actually live and got - // shut down". ChromiumJsRuntimeDebugger declares `getDependencies -> - // ChromiumCdp`, so disposing the transport takes it down regardless — while - // it was outside the namespace set, that shutdown was invisible, and an - // agent reading `stopped` was not told its console history was gone. The - // registry inserts a dependent before its dependency (`_resolve` creates - // the node, then `_initialize` resolves what it needs), so the map order - // here is the real one. - const services = new Map([ - [ - "ChromiumJsRuntimeDebugger:chromium-cdp-9222", - { state: ServiceState.RUNNING, dependents: [] }, - ], - [ - "ChromiumCdp:chromium-cdp-9222", - { - state: ServiceState.RUNNING, - dependents: ["ChromiumJsRuntimeDebugger:chromium-cdp-9222"], - }, - ], - ]); + // `stopped` is documented as "the services that were actually live and got + // shut down". ChromiumJsRuntimeDebugger declares `getDependencies -> + // ChromiumCdp`, so disposing the transport takes it down regardless — while + // it was outside the namespace set, that shutdown was invisible, and an agent + // reading `stopped` was not told its console history was gone. + // + // Run under both map orders. The registry usually inserts a dependent before + // its dependency (`_resolve` creates the node, then `_initialize` resolves + // what it needs), but a session that booted and described before attaching + // the debugger inserts `ChromiumCdp` first — and since `getSnapshot` copies, + // the answer must not depend on which happened. + const CDP = "ChromiumCdp:chromium-cdp-9222"; + const CHROMIUM_DEBUGGER = "ChromiumJsRuntimeDebugger:chromium-cdp-9222"; + const live = () => ({ state: ServiceState.RUNNING, dependents: [] as string[] }); + const cdpWithDependent = () => ({ + state: ServiceState.RUNNING, + dependents: [CHROMIUM_DEBUGGER], + }); + + it.each([ + ["debugger-connect first (dependent inserted first)", [CHROMIUM_DEBUGGER, CDP]], + ["boot/describe first (dependency inserted first)", [CDP, CHROMIUM_DEBUGGER]], + ])("names a cascading debugger in `stopped` — %s", async (_label, order) => { + const services = new Map( + order.map((urn) => [urn, urn === CDP ? cdpWithDependent() : live()] as const) + ); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); const result = await tool.execute!({}, { devices: ["chromium-cdp-9222"] }); - expect(result).toEqual({ - stopped: ["ChromiumJsRuntimeDebugger:chromium-cdp-9222", "ChromiumCdp:chromium-cdp-9222"], - }); - expect(services.get("ChromiumJsRuntimeDebugger:chromium-cdp-9222")?.state).toBe( - ServiceState.IDLE + // Order follows the snapshot; membership must not. + expect((result as { stopped: string[] }).stopped.slice().sort()).toEqual( + [CDP, CHROMIUM_DEBUGGER].sort() ); - expect(services.get("ChromiumCdp:chromium-cdp-9222")?.state).toBe(ServiceState.IDLE); + expect(result).not.toHaveProperty("unmatched"); + expect(services.get(CHROMIUM_DEBUGGER)?.state).toBe(ServiceState.IDLE); + expect(services.get(CDP)?.state).toBe(ServiceState.IDLE); }); it("returns empty list when no simulators are running", async () => { From 881d305cb74c68ab91f8c8c4cd8f378078d74985 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 30 Jul 2026 10:13:24 +0200 Subject: [PATCH 17/98] docs(flow): cut the recording tool descriptions back to what the tool does The four recording tools had grown descriptions where the edge cases outweighed the behaviour. flow-finish-recording spent most of its text on the no-ownership-check takeover, and three tools explained `savedTo` at length - which the client resolves to the flow file's path before the agent ever sees it, so there was nothing there to explain. flow-start-recording is back to the original four-line header: the truncation paragraph is three words on the first line, and the return line uses the "optionally { ... } if ..." form again, which is also more honest than promising `discardedSteps` is always present. Input constraints moved to the inputs - `name` now documents its charset in its own describe, which is why the failure line no longer enumerates validation errors. The long-form concurrency warning stays in argent-create-flow/SKILL.md, where the workflow guidance lives. --- .../src/tools/flows/flow-add-step.ts | 2 +- .../src/tools/flows/flow-finish-recording.ts | 2 +- .../src/tools/flows/flow-insert-echo.ts | 2 +- .../src/tools/flows/flow-start-recording.ts | 44 ++++++++++++------- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index 01ebd2b78..036625767 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -450,7 +450,7 @@ export function createFlowAddStepTool( failedMsg: ({ params, failureSignal }) => `Failed to add ${params.command} step to flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Execute a tool call and record it as a step in the flow named by \`name\` + \`project_root\` (the recording must already be open — see flow-start-recording). Use when recording a flow and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile, savedTo } on success - \`savedTo\` is where the YAML landed: a host path, or, against a remote client, the directive that has the client write it (the only field naming the destination in that mode). If it fails an error is returned and nothing is recorded. + description: `Execute a tool call and record it as a step in the flow named by \`name\` + \`project_root\` (the recording must already be open — see flow-start-recording). Use when recording a flow and you want to run and capture each action. A coordinate \`gesture-tap\` is recorded as a portable \`tap: { selector }\` step when the tapped element has stable text/identifier (otherwise coordinates are kept with a warning); a \`restart-app\` is recorded as a \`launch\` step (record one FIRST to make the flow a self-contained e2e flow; restart-app has no chromium support, so a chromium flow records as a fragment — add the \`launch: { chromium: }\` line to the YAML afterward, deleting the executionPrerequisite line if one was recorded: a flow that starts with a launch must not declare it). Returns { message, toolResult, flowFile, savedTo } on success. If it fails an error is returned and nothing is recorded. If a step was recorded by mistake, edit the .yaml file directly to remove it.`, zodSchema, services: () => ({}), diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 78b52238e..73c0f63cb 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -84,7 +84,7 @@ export const flowFinishRecordingTool: ToolDefinition< failedMsg: ({ params, failureSignal }) => `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving recordings under any OTHER key untouched. On this key it finishes whatever take is live, which is not necessarily the one you started: the (project_root, name) key has no ownership check, so if another agent restarted this name your steps were already discarded and you get ITS take, reported as yours. Usually NOTHING tells you: flow-add-step detects a takeover only if one lands while a step of yours is mid-flight, so between calls - the common case - it re-resolves the key and appends into the other agent's take, reporting success. A run of successful \`Step added\` results is therefore not evidence that the key is still yours. Pick a name unique to your task. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. In client mode \`savedTo\` is the directive that lands the file in your project, while \`path\` names a file that does not exist on the tool-server host. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. + description: `Finish recording the flow named by \`name\` + \`project_root\`, leaving recordings under any other key untouched. Returns { message, path, executionPrerequisite, steps, summary, flowFile, savedTo } - a summary of all recorded steps plus the final YAML. Use when you have added all desired steps and want to finalize the flow file. Fails if that flow has no recording in progress. You can still edit the .yaml file directly afterwards to remove or reorder steps.`, zodSchema, services: () => ({}), diff --git a/packages/tool-server/src/tools/flows/flow-insert-echo.ts b/packages/tool-server/src/tools/flows/flow-insert-echo.ts index 4075ebb4e..90f1772a9 100644 --- a/packages/tool-server/src/tools/flows/flow-insert-echo.ts +++ b/packages/tool-server/src/tools/flows/flow-insert-echo.ts @@ -29,7 +29,7 @@ export const flowInsertEchoTool: ToolDefinition< }, description: `Record an echo step in the flow named by \`name\` + \`project_root\`. Echo steps print a message when the flow is replayed — useful as labels between tool calls. Use when you want to annotate a recorded flow with a human-readable label or checkpoint message. -Returns { message, flowFile, savedTo } - \`savedTo\` is where the YAML landed: a host path, or, against a remote client, the directive that has the client write it (the only field naming the destination in that mode). Fails if that flow has no recording in progress.`, +Returns { message, flowFile, savedTo }. Fails if that flow has no recording in progress.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 3ad1b2b11..edd769f78 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -15,7 +15,11 @@ import { } from "./flow-utils"; const zodSchema = z.object({ - name: z.string().describe('Name for this flow (e.g. "settings-explore")'), + name: z + .string() + .describe( + 'Name for this flow (e.g. "settings-explore") — letters, digits, underscore and hyphen only.' + ), project_root: z .string() .describe( @@ -58,23 +62,33 @@ export const flowStartRecordingTool: ToolDefinition< > = { id: "flow-start-recording", interaction: { - startedMsg: () => "Starting flow recording", - completedMsg: () => "Started flow recording", - failedMsg: ({ failureSignal }) => `Failed to start flow recording: ${failureSignal.error_code}`, + // Name the flow: recordings are concurrent, so several of these lines can + // interleave in one log and "flow recording" would not identify which. + startedMsg: ({ params }) => `Starting recording of flow ${params.name}`, + completedMsg: ({ params, result }) => { + if (!result.restarted) return `Started recording flow ${params.name}`; + // A restart destroyed a live take. Reporting that as a plain start would + // hide the discard, which is the one outcome here worth reading twice. + // `discardedSteps` is absent when the superseded file could not be read + // or parsed - 0 is the answer a genuinely empty take gives - so say it + // was discarded without claiming a count we do not have. + const discarded = result.discardedSteps; + return discarded === undefined + ? `Restarted recording flow ${params.name}, discarding the previous take` + : `Restarted recording flow ${params.name}, discarding ${discarded} ${discarded === 1 ? "step" : "steps"}`; + }, + failedMsg: ({ params, failureSignal }) => + `Failed to start recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory. + description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory, replacing any existing one. Use when you want to capture a reusable sequence of device interactions for later replay. -Returns { message, flowFile, savedTo }. -Starting ALWAYS truncates /.argent/flows/.yaml to an empty flow — including a name that exists only as a saved file with no recording in progress, so starting under the name of a committed flow overwrites it. { restarted } is added only when a LIVE recording of the same flow was discarded; its absence does NOT mean nothing was overwritten. \`discardedSteps\` counts the flow file as it stood at the reset, so a hand-edit made mid-recording is included — but it is omitted, \`restarted\` alone, when that file could not be read or parsed. That holds when the project root is on the tool-server host; against a REMOTE client the host never sees the file, so there the count is of the steps recorded through this server and a hand-edit on your machine is not in it. Either way, re-record from the top rather than expecting to resume. -Fails before anything is written on a \`project_root\` that is not absolute or contains a ".." segment, or a \`name\` outside letters/digits/underscore/hyphen. It can also fail on the .argent/flows/ directory not being creatable or the file not being writable - but only when the project root is on the tool-server host; against a remote client the YAML travels back in \`savedTo\` for the client to write and no host filesystem access happens. +Returns { message, flowFile, savedTo } and optionally { restarted, discardedSteps } if a live recording of the same flow was discarded. +Fails if the .argent/flows/ directory cannot be created or the flow file cannot be written. -Recording state is independent: several flows can be recorded at once (different -names, different projects) and one recording's steps never land in another's -file. Steps still execute LIVE on a device, so give each concurrent recording its -own device. Every subsequent recording tool takes the same \`name\` + -\`project_root\` to say which one it is addressing — and the (project_root, name) -key has no ownership check, so pick a name unique to your task or another agent -starting the same one takes the key and your next step lands in its recording. +Several flows can be recorded at once — each keyed by the \`name\` + \`project_root\` +that every subsequent recording tool repeats — and one recording's steps never +land in another's file. Steps still run LIVE, so give each concurrent recording +its own device and pick a name unique to your task. After starting, use flow-add-step to append tool calls — each step is executed LIVE so you can verify it works before it gets recorded. For a self-contained From 3f07d8869603778e821f099d50d87adbec3567ec Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 30 Jul 2026 10:34:40 +0200 Subject: [PATCH 18/98] test(flow): give the echo interaction-message case the params its schema requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sensitive-input case built `flow-add-echo` params as `{ message }` alone, which was the whole schema when it was written. Recordings are now keyed by `name` + `project_root`, both required, and the completed message names the flow — so against the current tool that fixture rendered "Added note to flow undefined" and the assertion passed on a string no real call can produce. Pass the key, and assert the rendered line: the flow name is shown (it is a constrained identifier, not caller free text) while the echoed message — the part that is caller-authored — still stays out. --- packages/tool-server/test/interaction-messages.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/test/interaction-messages.test.ts b/packages/tool-server/test/interaction-messages.test.ts index a409bce8a..0dfe5f28c 100644 --- a/packages/tool-server/test/interaction-messages.test.ts +++ b/packages/tool-server/test/interaction-messages.test.ts @@ -130,13 +130,17 @@ describe("tool interaction messages", () => { params: { udid: "chromium-1", action: "set", name: "session", value: secret }, result: { set: true }, }), + // Recordings are keyed by `name` + `project_root`, so both are required + // and the message names the flow. The echoed `message` is the sensitive + // part — it is caller-authored free text — and stays out. definitions.get("flow-add-echo")!.interaction!.completedMsg!({ - params: { message: secret }, + params: { name: "checkout", project_root: "/tmp/proj", message: secret }, result: { message: secret, flowFile: "/tmp/flow.yaml", savedTo: "project" }, }), ]; expect(messages.join("\n")).not.toContain(secret); expect(messages).toContain("Opening example.com"); + expect(messages).toContain("Added note to flow checkout"); }); }); From a9e7148905f354850911b27ccfe0f1152e8cc965 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Mon, 3 Aug 2026 13:28:24 +0200 Subject: [PATCH 19/98] fix(flow): rebind the recorded device scope, and correct what the comments claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stop-all-simulator-servers`' `devices` is a device-id-carrying parameter that `DEVICE_BIND_KEYS` did not cover, so `flow-add-step` kept the recording host's id verbatim and the runner never rebound it. Reproduced against two booted simulators: a teardown recorded on 809A848B replayed against 9E82D0E0 ran with `devices: ["809A848B…"]`, reported `pass`, and left every one of the replay device's services RUNNING. Two permanent comments promise the opposite for exactly this case. `devices` now strips at record time and rebinds to `[resolvedDevice]` at replay - including over a recorded unscoped sweep, which a replayed artifact must not perform on another agent's devices. Also pins four invariants the diff documents as load-bearing that nothing observed - each confirmed by mutating the source and watching the new test die: the two port-keyed `JsRuntimeDebugger` dependents' membership, both `stop-all-simulator-servers` interaction formatters, `flow-start-recording`'s restart branch and its no-count sub-branch, and the scratch file's sibling directory, whose relocation is invisible under a test root that is itself in /tmp and is an EXDEV on any project that is not. The rest is prose the code contradicts: `Never throws.` alongside a `.strict()` schema; a cross-reference to a paragraph a later commit deleted; a `renderToolArgs` justification naming a state the parser makes impossible; a `completedMsg` deriving from `result.path` what `params.name` already is; a uniqueness claim shared by three members, not one; two `dispose()` comments promising process shutdown after their namespaces became reachable from a tool call; and three test comments naming the wrong layer as the one that validates. --- .../test/run-flow-add-step-payload.test.ts | 12 +- packages/argent-cli/test/run-help.test.ts | 9 +- .../src/blueprints/native-profiler-session.ts | 9 +- .../blueprints/screen-recording-session.ts | 14 +- .../src/tools/flows/flow-device.ts | 27 +++- .../src/tools/flows/flow-finish-recording.ts | 32 ++--- .../tool-server/src/tools/flows/flow-utils.ts | 5 +- .../src/tools/simulator/device-services.ts | 11 +- .../simulator/stop-all-simulator-servers.ts | 2 +- .../test/flows/flow-composition.test.ts | 37 ++++- .../flows/flow-concurrent-recording.test.ts | 55 ++++++- .../tool-server/test/flows/flow-tools.test.ts | 30 ++++ .../test/interaction-messages.test.ts | 50 +++++++ packages/tool-server/test/stop-tools.test.ts | 135 +++++++++++++++++- 14 files changed, 381 insertions(+), 47 deletions(-) diff --git a/packages/argent-cli/test/run-flow-add-step-payload.test.ts b/packages/argent-cli/test/run-flow-add-step-payload.test.ts index 153a28ed3..dc965b26d 100644 --- a/packages/argent-cli/test/run-flow-add-step-payload.test.ts +++ b/packages/argent-cli/test/run-flow-add-step-payload.test.ts @@ -41,9 +41,15 @@ function startServer(cap: Captured): Promise<{ url: string; close: () => Promise // zodObjectToJsonSchema over the zod schema in // packages/tool-server/src/tools/flows/flow-add-step.ts. `name` // and `project_root` identify which open recording the step - // belongs to and are required alongside `command`. All three have - // to be marked required here for the parser regression this test - // guards to be reachable at all. + // belongs to and are required alongside `command`. + // + // Only `properties` is load-bearing here: `parseFlags` reads it + // to decide whether `args` belongs to the tool, and reads + // `required` nowhere (its one consumer is `formatSchemaUsage`, + // the help renderer, which this file never invokes — that is + // covered by run-help.test.ts). The array is kept faithful so the + // fixture stays readable as the real schema, not because dropping + // an entry would fail here. inputSchema: { type: "object", properties: { diff --git a/packages/argent-cli/test/run-help.test.ts b/packages/argent-cli/test/run-help.test.ts index 262b2f674..4cb9a9c0f 100644 --- a/packages/argent-cli/test/run-help.test.ts +++ b/packages/argent-cli/test/run-help.test.ts @@ -29,9 +29,12 @@ vi.mock("@argent/telemetry", () => telemetryMock); // zodObjectToJsonSchema over the zod schema in // packages/tool-server/src/tools/flows/flow-add-step.ts. Recordings are keyed // by `name` + `project_root`, so both are required alongside `command` and only -// `args` / `delayMs` are optional. Keep the fixture in step with that schema: a -// fixture marking fewer fields required renders help for a tool the server does -// not expose, and the mismatch passes silently. +// `args` / `delayMs` are optional. The assertions below pin `required` in both +// directions — each entry against its `(required)` marker, each non-entry +// against a negative lookahead — so dropping or adding one here fails loudly. +// The drift that does pass silently is the opposite one: if the real +// flow-add-step schema ever relaxes, nothing here notices this fixture went +// stale. const flowAddStepMeta = { name: "flow-add-step", // Leading sentence of the real tool description, verbatim. diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index d58d5b3ec..ce43d61ce 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -94,9 +94,12 @@ export interface NativeProfilerSessionApi { androidOnDeviceTracePath: string | null; } -// Dispose only fires on process shutdown, where an in-flight recording is being -// abandoned: skip the SIGINT finalise grace (that's the native-profiler-stop -// contract) and SIGKILL straight away so shutdown isn't held up. +// Dispose fires on process shutdown, and — since `NativeProfilerSession` joined +// `DEVICE_OWNED_NAMESPACES` — on `stop-all-simulator-servers`, the call every +// agent makes at session end. Either way an in-flight capture is being +// abandoned with nobody waiting on the trace, so skip the SIGINT finalise grace +// (that's the native-profiler-stop contract, and a caller that wants the trace +// calls that) and SIGKILL straight away rather than holding the caller up. const DISPOSE_REAP_MS = 1_000; const ANDROID_DISPOSE_ADB_TIMEOUT_MS = 5_000; diff --git a/packages/tool-server/src/blueprints/screen-recording-session.ts b/packages/tool-server/src/blueprints/screen-recording-session.ts index 2dd139f21..50788f547 100644 --- a/packages/tool-server/src/blueprints/screen-recording-session.ts +++ b/packages/tool-server/src/blueprints/screen-recording-session.ts @@ -42,7 +42,8 @@ export interface ScreenRecordingSessionApi { /** True while a stop is running; a concurrent start/stop must not interleave. */ stopPending: boolean; /** - * Set the moment dispose() begins (process shutdown). A start suspended at a + * Set the moment dispose() begins — process shutdown, or a scoped/unscoped + * `stop-all-simulator-servers` naming this device. A start suspended at a * pre-spawn await (resolving ffmpeg, connecting to the frame stream) checks * this immediately before spawning and aborts — otherwise it would spawn an * encoder AFTER dispose already ran, orphaning a process that `pendingChild` @@ -102,10 +103,13 @@ export interface ScreenRecordingSessionApi { lastExitInfo: { code: number | null; signal: string | null } | null; } -// Dispose only fires on process shutdown, where an in-flight recording is -// being abandoned. Closing ffmpeg's stdin is what finalizes the container, so -// give that one short grace before SIGKILL — shutdown must not be held up by a -// slow finalize, but a playable file is worth a moment. +// Dispose fires on process shutdown, and — since `ScreenRecordingSession` joined +// `DEVICE_OWNED_NAMESPACES` — on `stop-all-simulator-servers`, the call every +// agent makes at session end. Either way an in-flight recording is being +// abandoned, so the video is a best-effort salvage rather than something a +// caller is waiting on: closing ffmpeg's stdin is what finalizes the container, +// so give that one short grace before SIGKILL. A caller that wants the file +// calls `screen-recording-stop`, which has its own (longer) finalize contract. const DISPOSE_FINALIZE_GRACE_MS = 1_500; const DISPOSE_REAP_MS = 1_000; diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index 50a732fb9..d715e1d10 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -42,6 +42,28 @@ const DEVICE_BIND_KEYS = ["udid", "device_id", "device"] as const; */ const DEVICE_ARG_KEYS = [...DEVICE_BIND_KEYS, "device"] as const; +/** + * Args keys holding a LIST of device ids. Same treatment as + * {@link DEVICE_BIND_KEYS} — stripped at record time, re-injected at replay — + * but rebound to `[deviceId]`, since the runner resolves exactly one device per + * run and a flow that named several would be naming the recording host's. + * + * `stop-all-simulator-servers`' `devices` is the only such key. It is a scope + * rather than a target, but the failure is the same one: kept verbatim, a + * recorded teardown names the machine it was recorded on, so on any other host + * it reaps nothing and passes — a stale baked-in id overriding the run target, + * which is exactly what this binding exists to prevent. The scoped form is what + * the tool description, the MCP instructions and the skills all now tell agents + * to call, so it is the form that gets recorded. + * + * Binding is unconditional once the tool declares the key, so a recording of + * the UNSCOPED sweep replays as a stop of the run device. That direction is + * deliberate: the replayed artifact must not tear down devices another agent is + * mid-session on, which is the hazard the `devices` scope was added for, and a + * flow has exactly one resolved device to be talking about. + */ +const DEVICE_BIND_LIST_KEYS = ["devices"] as const; + interface RawDevice { platform: FlowPlatform; state?: string; @@ -129,6 +151,7 @@ export async function resolveFlowDevice( export function stripDeviceKeys(args: Record): Record { const out = { ...args }; for (const k of DEVICE_BIND_KEYS) delete out[k]; + for (const k of DEVICE_BIND_LIST_KEYS) delete out[k]; return out; } @@ -138,7 +161,8 @@ export function stripDeviceKeys(args: Record): Record `Finishing recording of flow ${params.name}`, - // Derived from the resolved path rather than `params.name` so the line - // reports the file that was actually written. Holds in client mode too: - // `path` is still the resolved spelling, it just names a file on the - // client's disk rather than this host's, and only its basename is read. - completedMsg: ({ result }) => { - const flowName = - result.path - .split(/[\\/]/) - .pop() - ?.replace(/\.ya?ml$/, "") ?? "flow"; - return `Saved recorded flow ${flowName}`; - }, + // `params.name` rather than the basename of `result.path`: the two are the + // same string on every branch — `assertSafeFlowName` admits no dots or + // separators, so `getFlowPath` produces `.yaml` and nothing else — + // and this spelling matches the two formatters either side of it. + completedMsg: ({ params }) => `Saved recorded flow ${params.name}`, failedMsg: ({ params, failureSignal }) => `Failed to finish recording of flow ${params.name}: ${failureSignal.error_code}`, }, @@ -152,12 +145,15 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps * unrenderable step. * * The body interpolates rather than returning `JSON.stringify(args)` directly, - * and that is load-bearing: `JSON.stringify(undefined)` is the VALUE - * `undefined`, not a string, so a `tool:` step with no `args` would return - * `undefined` through a `string`-typed signature (TypeScript does not catch it - * — `JSON.stringify`'s overload is declared to return `string`). The template - * literal renders it as the text "undefined" instead, which is what - * {@link summarizeSteps} has always shown for that step. + * because `JSON.stringify(undefined)` is the VALUE `undefined`, not a string, + * and would leave through a `string`-typed signature uncaught (TypeScript does + * not flag it — `JSON.stringify`'s overload is declared to return `string`). + * No reachable input is undefined today: every caller comes through + * {@link summarizeSteps}, which is only ever handed `parseFlow` output, and + * `fromYamlStep` normalises a missing/`null` `args:` to `{}` on the way + * through. It is the `default:` arm of that switch this guards — a step kind + * added without its own `case` lands there and is rendered as a `tool:` step, + * with no `args` field to read. */ function renderToolArgs(args: unknown): string { try { diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 9c4b102fd..03560fb16 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2448,7 +2448,10 @@ export async function writeNewFlowFile(filePath: string, content: string): Promi * called in "host" mode. In "client" mode the file lives on the client's * machine and this host cannot read it at all, so the in-memory copy is both * the take and the only thing countable — the guarantee below does not carry - * across that boundary, and `flow-start-recording`'s description says so. + * across that boundary. The tool descriptions do not spell that out — they are + * kept to what the tool does; the agent-facing statement of it lives in + * `packages/skills/skills/argent-create-flow/SKILL.md`, in the + * "Starting always truncates the `.yaml`" bullet. * * The file — not the session's in-memory `flow` — is the take in "host" mode: * {@link appendStep} diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index a79da9e28..e02c87f89 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -108,11 +108,12 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ * - `JsRuntimeDebugger` owns a bound loopback HTTP/WebSocket server, the CDP * socket to Metro, and a log file handle. * - `ChromiumJsRuntimeDebugger` owns the same, plus its captured console - * history. It is the one member whose dependency (`ChromiumCdp`) is also - * here, so a scoped stop reaches it twice over — it is listed for the naming, - * which is the same reason the two `JsRuntimeDebugger` dependents are. - * Note its URN is `:`, NOT port-keyed, so it belongs in this - * list and not in {@link PORT_KEYED_NAMESPACES}. + * history. Its dependency (`ChromiumCdp`) is listed here too, so a scoped + * stop reaches it twice over — as it does the two `JsRuntimeDebugger` + * dependents, whose own dependency is equally listed. All three are here for + * the naming, not for the reaping. What is particular to this one is its URN + * SHAPE: `:`, not port-keyed like the other two dependents, so + * it belongs in this list and not in {@link PORT_KEYED_NAMESPACES}. * * (`AndroidTvControl` is stateless adb shell-outs with a no-op dispose, but is * included for symmetry so the snapshot is fully drained.) diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 44a0c9df4..f522471b3 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -57,7 +57,7 @@ export function createStopAllSimulatorServersTool( }, description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions, and JS-runtime debugger sessions along with the network inspectors and React profiler sessions that ride on them - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — one tool-server serves every agent, subagent and CLI call using this argent install, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. -Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Never throws.`, +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Past the schema - which rejects an unknown key outright, so the \`udids\` slip is an error rather than a silent machine-wide sweep - the call always succeeds; reaping nothing is a result, not a failure.`, zodSchema, services: () => ({}), async execute(_services, params) { diff --git a/packages/tool-server/test/flows/flow-composition.test.ts b/packages/tool-server/test/flows/flow-composition.test.ts index 2ed6619ee..3f4d5f4ba 100644 --- a/packages/tool-server/test/flows/flow-composition.test.ts +++ b/packages/tool-server/test/flows/flow-composition.test.ts @@ -2375,8 +2375,10 @@ describe("device binding (portability)", () => { expect(out).toEqual({ foo: 1 }); }); - it("stripDeviceKeys removes udid / device_id / device", () => { - expect(stripDeviceKeys({ udid: "A", device_id: "B", device: "C", x: 1 })).toEqual({ x: 1 }); + it("stripDeviceKeys removes udid / device_id / device / devices, leaving other args untouched", () => { + expect( + stripDeviceKeys({ udid: "A", device_id: "B", device: "C", devices: ["D", "E"], x: 1 }) + ).toEqual({ x: 1 }); }); it("rebinds a nested flow-execute onto the run device (issue #607)", () => { @@ -2400,6 +2402,37 @@ describe("device binding (portability)", () => { // bound, because device resolution returns before it is ever read. expect(stripDeviceKeys({ platform: "android", x: 1 })).toEqual({ platform: "android", x: 1 }); }); + + it("injects devices: [resolvedId] for a tool that declares it in its schema", () => { + // stop-all-simulator-servers' `devices` is a scope, not a single-device + // target, but it names the recording host's device ids the same way `udid` + // does — so it gets the same schema-aware rebind, as a one-element list. + const out = bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "RESOLVED", {}); + expect(out).toEqual({ devices: ["RESOLVED"] }); + }); + + it("does not invent a devices key for a tool that doesn't declare it", () => { + const out = bindDeviceArgs(reg({ foo: {} }), "x", "RESOLVED", { foo: 1 }); + expect(out).toEqual({ foo: 1 }); + expect(out).not.toHaveProperty("devices"); + }); + + it("replaces a stale recorded devices list rather than merging or appending to it", () => { + // The runner is authoritative on device — a flow recorded on one host must + // not carry that host's ids forward when replayed on another. + const out = bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "RESOLVED", { + devices: ["OLD-HOST-ID", "OTHER"], + }); + expect(out).toEqual({ devices: ["RESOLVED"] }); + }); + + it("binds a scalar and a list device key together when a tool declares both", () => { + const out = bindDeviceArgs(reg({ udid: {}, devices: {} }), "hypothetical-tool", "RESOLVED", { + udid: "STALE", + devices: ["OLD"], + }); + expect(out).toEqual({ udid: "RESOLVED", devices: ["RESOLVED"] }); + }); }); describe("flow validation", () => { diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 65d6cff34..bb3f3ada3 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -24,6 +24,16 @@ import { type FlowStep, } from "../../src/tools/flows/flow-utils"; +// Wrap (not replace) `rename` so every call still does the real filesystem +// rename — every other test's atomicity assertions depend on that — while +// letting the atomic-swap test below inspect exactly which paths each write +// renamed between. Everything else in `node:fs/promises` passes through +// untouched. +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rename: vi.fn(actual.rename) }; +}); + /** * Concurrency contract of the recording tools. One tool-server serves every MCP * client, subagent and CLI call using one argent install, so several agents can @@ -96,9 +106,13 @@ function createMockRegistry(): Registry { return { invokeTool: vi.fn(async (id: string) => { if (id === "list-devices") return { devices: [] }; - // Yield a macrotask: flow-add-step runs the step LIVE before it appends, - // so this is what lets several calls issued without an await in between - // reach the append phase concurrently (see the lost-update test). + // Yield a macrotask, so calls issued without an await in between all + // finish their LIVE phase before any of them appends. This is NOT what + // creates the overlap the file tests: `appendStep`'s own + // `await fs.readFile` already suspends every caller inside the + // read-modify-write, so the append phases interleave with or without this + // line. It stands in for a real sub-tool's I/O, and lines the calls up at + // the same starting gun. await new Promise((resolve) => setTimeout(resolve, 0)); if (subToolGate) await subToolGate(); return { ok: true }; @@ -575,6 +589,41 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(await readMarkers(root, "alpha")).toEqual(["tool:a1", "echo:note", "tool:a2"]); }); + it("renames the scratch file from beside the target, never from a shared temp dir", async () => { + // "leaves no scratch file behind" (above) only proves the .tmp is gone by + // the time the call returns — a scratch file built under `os.tmpdir()` + // instead of the flow's own directory satisfies that identically, since it + // was never IN the flows directory to begin with. What actually keeps the + // swap atomic is `fs.rename` staying on ONE filesystem, which only holds + // because the scratch path is a sibling of the target — so pin THAT + // property directly, on the arguments the real rename call is made with, + // rather than on a side effect two different implementations both produce. + // + // This does not depend on the test root's filesystem: `flowsDir` here is + // always a subdirectory of whatever `os.tmpdir()` returns (`makeRoot` + // mkdtemps under it), never equal to it, so relocating the scratch file to + // `os.tmpdir()` itself is caught by the directory comparison below on any + // host, without ever needing two real filesystems to reproduce EXDEV. + const root = await makeRoot("scratch-sibling"); + const flowsDir = path.dirname(flowPath(root, "alpha")); + vi.mocked(fs.rename).mockClear(); + + await start(root, "alpha"); // writeNewFlowFile → writeFlowFile → 1 rename + await addStep(root, "alpha", "a1"); // appendStep → writeFlowFile → 1 rename + await addEcho(root, "alpha", "a2"); // appendStep → writeFlowFile → 1 rename + + const renameCalls = vi.mocked(fs.rename).mock.calls; + expect(renameCalls).toHaveLength(3); + for (const [from, to] of renameCalls) { + // The rename target is always the flow file itself… + expect(path.dirname(String(to))).toBe(flowsDir); + // …and the scratch source must sit right next to it. If it didn't, this + // same rename would cross filesystems in production (project root vs. + // OS temp dir) and fail with EXDEV instead of swapping atomically. + expect(path.dirname(String(from))).toBe(flowsDir); + } + }); + it("still appends under a flow name long enough to fill the filesystem's limit", async () => { // A flow name has no length cap — FLOW_NAME_PATTERN constrains the // character set only — so `.yaml` can legitimately reach NAME_MAX diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 96a3a485c..04fa11fa9 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -1287,6 +1287,36 @@ describe("flow-add-step", () => { expect(flow.steps).toEqual([]); }); + it("strips the devices list when recording a scoped teardown (device ids stay off disk)", async () => { + // stop-all-simulator-servers' `devices` names the recording host's device + // ids the same way a udid does; a recorded scoped teardown must not bake + // that host's ids into the flow, or replay on another host stops nothing. + const registry = createMockRegistry({ + "stop-all-simulator-servers": { result: { stopped: 1 } }, + }); + const tool = createFlowAddStepTool(registry); + + await flowStartRecordingTool.execute({}, { name: "teardown-test", project_root: tmpDir }); + const result = await tool.execute( + {}, + { + name: "teardown-test", + project_root: tmpDir, + command: "stop-all-simulator-servers", + args: JSON.stringify({ devices: ["00000000-HOST-DEVICE-ID"] }), + } + ); + + // Ran live with the real devices to stop… + expect(registry.invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { + devices: ["00000000-HOST-DEVICE-ID"], + }); + // …but the recorded step carries no device id, keeping the flow portable. + expect(parseFlow(result.flowFile).steps).toEqual([ + { kind: "tool", name: "stop-all-simulator-servers", args: {} }, + ]); + }); + it("propagates error when tool is not registered in the registry", async () => { const registry = createMockRegistry({}); // no tools registered const tool = createFlowAddStepTool(registry); diff --git a/packages/tool-server/test/interaction-messages.test.ts b/packages/tool-server/test/interaction-messages.test.ts index 0dfe5f28c..f2cd4d1fc 100644 --- a/packages/tool-server/test/interaction-messages.test.ts +++ b/packages/tool-server/test/interaction-messages.test.ts @@ -110,6 +110,56 @@ describe("tool interaction messages", () => { ); }); + it("distinguishes a fresh recording start from a destructive restart", () => { + // A restart truncates and replaces a live take; if its message ever + // collapsed to the same wording as a fresh start (or reported a step + // count that was never actually obtained), an agent re-recording a flow + // would have no way to notice it just destroyed prior work. + const definitions = definitionsById(createRegistry()); + const completedMsg = definitions.get("flow-start-recording")!.interaction!.completedMsg!; + const params = { name: "checkout", project_root: "/tmp/proj" }; + + expect( + completedMsg({ + params, + result: { message: "", flowFile: "", savedTo: "project" }, + }) + ).toBe("Started recording flow checkout"); + + expect( + completedMsg({ + params, + result: { message: "", flowFile: "", savedTo: "project", restarted: true }, + }) + ).toBe("Restarted recording flow checkout, discarding the previous take"); + + expect( + completedMsg({ + params, + result: { + message: "", + flowFile: "", + savedTo: "project", + restarted: true, + discardedSteps: 1, + }, + }) + ).toBe("Restarted recording flow checkout, discarding 1 step"); + + expect( + completedMsg({ + params, + result: { + message: "", + flowFile: "", + savedTo: "project", + restarted: true, + discardedSteps: 4, + }, + }) + ).toBe("Restarted recording flow checkout, discarding 4 steps"); + }); + it("does not expose sensitive inputs", () => { const definitions = definitionsById(createRegistry()); const secret = "INTERACTION_MESSAGE_SECRET"; diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index a58b25753..e1f13f5ae 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -543,8 +543,14 @@ describe("stop-all-simulator-servers device scoping", () => { const parsed = tool.zodSchema!.safeParse({ udids: [MINE] }); expect(parsed.success).toBe(false); - // And the same rejection reaches MCP / `argent run` / raw HTTP callers, - // which validate against the advertised JSON schema rather than the zod one. + // The zod parse above is the only gate: MCP, `argent run` and raw HTTP all + // forward the caller's args verbatim (`argent run` accepts unknown flags on + // purpose, see flag-parser.ts) and the tool-server parses them with this + // schema. What the assertion below pins is the ADVERTISED shape, derived + // from `.strict()` by `zodObjectToJsonSchema` — the schema an agent reads + // out of `GET /tools` to learn the key is `devices`. An advertised schema + // still admitting extra keys would document the `udids` typo as legal and + // leave the rejection looking like a server bug. expect(zodObjectToJsonSchema(tool.zodSchema as z.ZodObject)).toMatchObject({ additionalProperties: false, }); @@ -804,6 +810,63 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); }); + it("scopes the port-keyed NetworkInspector and ReactProfilerSession URNs to the right device", async () => { + // NetworkInspector and ReactProfilerSession share JsRuntimeDebugger's + // port-keyed URN shape (`::`) but are declared apart + // from it in PORT_KEYED_NAMESPACES. Without that membership neither + // namespace is in DEVICE_OWNED_NAMESPACES at all, so a standalone node + // (no JsRuntimeDebugger present to cascade through) would match nothing + // and never be named in `stopped`. Both devices sit behind the SAME port, + // so this also pins that the port is not what the scoping keys on. + const services = new Map([ + [`NetworkInspector:8081:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NetworkInspector:8081:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ReactProfilerSession:8081:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ReactProfilerSession:8081:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ + stopped: [`NetworkInspector:8081:${MINE}`, `ReactProfilerSession:8081:${MINE}`], + }); + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).not.toHaveBeenCalledWith(`NetworkInspector:8081:${THEIRS}`); + expect(registry.disposeService).not.toHaveBeenCalledWith(`ReactProfilerSession:8081:${THEIRS}`); + }); + + it("does not let a NetworkInspector/ReactProfilerSession port be mistaken for a wireless-adb device id", async () => { + // Mirrors the JsRuntimeDebugger case above: the device id after the port + // can itself be `ip:port`, so only the FIRST colon may be consumed as the + // Metro port. + const serial = "192.168.1.5:5555"; + const services = new Map([ + [`NetworkInspector:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ReactProfilerSession:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + expect(await tool.execute!({}, { devices: [serial] })).toEqual({ + stopped: [`NetworkInspector:8081:${serial}`, `ReactProfilerSession:8081:${serial}`], + }); + + // A bare IP must not claim it, and neither must the port. + const registry2 = createMockRegistry( + new Map([ + [`NetworkInspector:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ReactProfilerSession:8081:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + ]) + ); + const tool2 = createStopAllSimulatorServersTool(registry2); + expect(await tool2.execute!({}, { devices: ["192.168.1.5", "8081"] })).toEqual({ + stopped: [], + unmatched: ["192.168.1.5", "8081"], + }); + }); + it("reaps AXService on an unscoped machine-wide sweep too", async () => { const services = new Map([ [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], @@ -891,6 +954,74 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); }); +describe("stop-all-simulator-servers interaction messages", () => { + // Both formatters previously had no coverage at all — flattening either to + // an unconditional string left the whole suite green. Pin the exact wording + // for every branch a caller can hit. + function tool() { + return createStopAllSimulatorServersTool(createMockRegistry(new Map())); + } + + it("startedMsg reports a machine-wide sweep when devices is omitted", () => { + const startedMsg = tool().interaction!.startedMsg!; + expect(startedMsg({ params: {} })).toBe("Stopping all simulator servers"); + }); + + it("startedMsg is singular for exactly one device", () => { + const startedMsg = tool().interaction!.startedMsg!; + expect(startedMsg({ params: { devices: [MINE] } })).toBe( + "Stopping simulator servers for 1 device" + ); + }); + + it("startedMsg is plural for two or more devices", () => { + const startedMsg = tool().interaction!.startedMsg!; + expect(startedMsg({ params: { devices: [MINE, THEIRS] } })).toBe( + "Stopping simulator servers for 2 devices" + ); + }); + + it("completedMsg has no unmatched clause when nothing was unmatched, singular and zero counts", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect(completedMsg({ params: {}, result: { stopped: [`SimulatorServer:${MINE}`] } })).toBe( + "Stopped 1 simulator server" + ); + expect(completedMsg({ params: {}, result: { stopped: [] } })).toBe( + "Stopped 0 simulator servers" + ); + }); + + it("completedMsg pluralizes 'servers' for more than one stopped", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect( + completedMsg({ + params: {}, + result: { stopped: [`SimulatorServer:${MINE}`, `SimulatorServer:${THEIRS}`] }, + }) + ).toBe("Stopped 2 simulator servers"); + }); + + it("completedMsg appends the singular unmatched clause for exactly one bad id", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect( + completedMsg({ + params: { devices: [MINE, "GHOST-9999"] }, + result: { stopped: [`SimulatorServer:${MINE}`], unmatched: ["GHOST-9999"] }, + }) + ).toBe("Stopped 1 simulator server (1 supplied id matched no service)"); + }); + + it("completedMsg appends the plural unmatched clause for two or more bad ids", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect( + completedMsg({ + params: { devices: ["GHOST-1", "GHOST-2"] }, + result: { stopped: [], unmatched: ["GHOST-1", "GHOST-2"] }, + }) + ).toBe("Stopped 0 simulator servers (2 supplied ids matched no service)"); + }); +}); + describe("stop-metro", () => { it("defaults to port 8081", () => { expect(stopMetroTool.zodSchema).toBeDefined(); From 0068f0c2214fd35922c1db81173fa2116284c3bd Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 14:10:51 +0200 Subject: [PATCH 20/98] fix(flow): keep the restart discard-report correct under a racing eviction, and stop calling a teardown a shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read the take a restart discards once, inside the flow-file lock, and drive both restarted and discardedSteps off it — an evictIfOverCapacity landing between startRecordingSession's own read and the register no longer downgrades a destructive restart to a plain fresh start. assertNotDisposed no longer asserts the server is shutting down: dispose() also fires when stop-all-simulator-servers reaps a device-owned session (commonly another agent's teardown), where a retry succeeds. The message names both causes; the four sibling dispose comments say so too. --- .../src/blueprints/native-profiler-session.ts | 12 +++---- .../blueprints/screen-recording-session.ts | 13 ++++---- .../src/tools/flows/flow-start-recording.ts | 33 ++++++++++++------- .../src/tools/screen-recording/capture.ts | 8 +++-- .../screen-recording-start.ts | 3 +- .../tools/screen-recording/session-guards.ts | 22 ++++++++++--- 6 files changed, 59 insertions(+), 32 deletions(-) diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index ce43d61ce..e8ad1a503 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -94,12 +94,12 @@ export interface NativeProfilerSessionApi { androidOnDeviceTracePath: string | null; } -// Dispose fires on process shutdown, and — since `NativeProfilerSession` joined -// `DEVICE_OWNED_NAMESPACES` — on `stop-all-simulator-servers`, the call every -// agent makes at session end. Either way an in-flight capture is being -// abandoned with nobody waiting on the trace, so skip the SIGINT finalise grace -// (that's the native-profiler-stop contract, and a caller that wants the trace -// calls that) and SIGKILL straight away rather than holding the caller up. +// Dispose fires on process shutdown, and on `stop-all-simulator-servers` (which +// reaps every device-owned service, `NativeProfilerSession` among them) — the +// call every agent makes at session end. Either way an in-flight capture is +// being abandoned with nobody waiting on the trace, so skip the SIGINT finalise +// grace (that's the native-profiler-stop contract, and a caller that wants the +// trace calls that) and SIGKILL straight away rather than holding the caller up. const DISPOSE_REAP_MS = 1_000; const ANDROID_DISPOSE_ADB_TIMEOUT_MS = 5_000; diff --git a/packages/tool-server/src/blueprints/screen-recording-session.ts b/packages/tool-server/src/blueprints/screen-recording-session.ts index 50788f547..2c4ef8816 100644 --- a/packages/tool-server/src/blueprints/screen-recording-session.ts +++ b/packages/tool-server/src/blueprints/screen-recording-session.ts @@ -42,8 +42,9 @@ export interface ScreenRecordingSessionApi { /** True while a stop is running; a concurrent start/stop must not interleave. */ stopPending: boolean; /** - * Set the moment dispose() begins — process shutdown, or a scoped/unscoped - * `stop-all-simulator-servers` naming this device. A start suspended at a + * Set the moment dispose() begins — process shutdown, or a + * `stop-all-simulator-servers` that reaps this device (a scoped call + * including it, or an unscoped machine-wide sweep). A start suspended at a * pre-spawn await (resolving ffmpeg, connecting to the frame stream) checks * this immediately before spawning and aborts — otherwise it would spawn an * encoder AFTER dispose already ran, orphaning a process that `pendingChild` @@ -103,10 +104,10 @@ export interface ScreenRecordingSessionApi { lastExitInfo: { code: number | null; signal: string | null } | null; } -// Dispose fires on process shutdown, and — since `ScreenRecordingSession` joined -// `DEVICE_OWNED_NAMESPACES` — on `stop-all-simulator-servers`, the call every -// agent makes at session end. Either way an in-flight recording is being -// abandoned, so the video is a best-effort salvage rather than something a +// Dispose fires on process shutdown, and on `stop-all-simulator-servers` (which +// reaps every device-owned service, `ScreenRecordingSession` among them) — the +// call every agent makes at session end. Either way an in-flight recording is +// being abandoned, so the video is a best-effort salvage rather than something a // caller is waiting on: closing ffmpeg's stdin is what finalizes the container, // so give that one short grace before SIGKILL. A caller that wants the file // calls `screen-recording-stop`, which has its own (longer) finalize contract. diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index edd769f78..d5699e9dd 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -127,18 +127,29 @@ to remove or reorder steps.`, params.project_root, params.name, async () => { - // Count the take BEFORE the truncate destroys it, and count it where it - // actually lives: on disk in host mode, since a hand-edit made - // mid-recording is part of the take and the session's in-memory copy - // only catches up on the next append (see {@link countStepsOnDisk}). In - // client mode this host has no file and the in-memory copy IS the take. - const previous = getRecordingSession(params.project_root, params.name); + // Read the take being discarded ONCE, here, and drive both the + // `restarted` flag and its step count off that single read. Count it + // BEFORE the truncate destroys it, and where it actually lives: on disk + // in host mode, since a hand-edit made mid-recording is part of the take + // and the session's in-memory copy only catches up on the next append + // (see {@link countStepsOnDisk}). In client mode this host has no file + // and the in-memory copy IS the take. + // + // `replaced` is this read, NOT `startRecordingSession`'s return: in host + // mode two awaits (countStepsOnDisk, writeNewFlowFile) sit between them, + // and {@link evictIfOverCapacity} runs under some OTHER key's lock, so + // it can drop this key in that window. Reading `replaced` after the + // register would then see the key already gone and report a destructive + // restart — the file is already truncated — as a plain fresh start, + // discarding the count computed here. This read is inside our own key's + // lock, so it and the count agree. + const replaced = getRecordingSession(params.project_root, params.name) ?? null; const discardedSteps = - previous === undefined + replaced === null ? undefined - : previous.persist === "host" - ? await countStepsOnDisk(previous.filePath) - : previous.flow.steps.length; + : replaced.persist === "host" + ? await countStepsOnDisk(replaced.filePath) + : replaced.flow.steps.length; let savedTo: FlowSavedTo; if (persist === "host") { @@ -147,7 +158,7 @@ to remove or reorder steps.`, } else { savedTo = clientFileDirective(filePath, flowFile); } - const replaced = startRecordingSession({ + startRecordingSession({ name: params.name, projectRoot: params.project_root, persist, diff --git a/packages/tool-server/src/tools/screen-recording/capture.ts b/packages/tool-server/src/tools/screen-recording/capture.ts index aa5aad9bf..72c0f1772 100644 --- a/packages/tool-server/src/tools/screen-recording/capture.ts +++ b/packages/tool-server/src/tools/screen-recording/capture.ts @@ -330,8 +330,9 @@ async function startCaptureLocked( } // No await between here and `api.pendingChild = child`: if dispose() ran - // (shutdown) while this start was suspended above, abort now rather than - // spawn an encoder the teardown can no longer reap. + // (shutdown, or a stop-all-simulator-servers teardown of this device) while + // this start was suspended above, abort now rather than spawn an encoder the + // teardown can no longer reap. assertNotDisposed(api, "screen_recording_start"); child = spawn(ffmpeg, ffmpegArgs({ outputFile, logoFile, graph }), { stdio: ["pipe", "ignore", "pipe"], @@ -424,7 +425,8 @@ async function startCaptureLocked( if (params.pointer) { // Arm the touch visualizer before returning, so the very first interaction // is already drawn into the recording. Store the teardown first so a - // shutdown racing this await still restores the overlay. Best-effort: a + // shutdown (or a stop-all-simulator-servers teardown of this device) racing + // this await still restores the overlay. Best-effort: a // failure only costs the touch markers, surfaced as a warning at stop. api.pointerDisable = params.pointer.disable; api.pointerFailed = !(await params.pointer.enable()); diff --git a/packages/tool-server/src/tools/screen-recording/screen-recording-start.ts b/packages/tool-server/src/tools/screen-recording/screen-recording-start.ts index 608d0d8ff..bab91cfa3 100644 --- a/packages/tool-server/src/tools/screen-recording/screen-recording-start.ts +++ b/packages/tool-server/src/tools/screen-recording/screen-recording-start.ts @@ -160,7 +160,8 @@ Fails if a recording is already running on the device, the device is not booted, * * `disable` waits for any in-flight `enable` to settle before sending its own * `show:false`. Enabling is the one suspension point after a recording is - * stamped, so a dispose (shutdown) can call `disable` while `enable`'s + * stamped, so a dispose (shutdown, or a stop-all-simulator-servers teardown of + * this device) can call `disable` while `enable`'s * `show:true` request is still outstanding. Without this barrier the two * requests race and the earlier-issued `show:false` can be overtaken by the * later `show:true`, leaving simulator-server's overlay stuck on after the diff --git a/packages/tool-server/src/tools/screen-recording/session-guards.ts b/packages/tool-server/src/tools/screen-recording/session-guards.ts index 8d751fff3..8683c2768 100644 --- a/packages/tool-server/src/tools/screen-recording/session-guards.ts +++ b/packages/tool-server/src/tools/screen-recording/session-guards.ts @@ -121,15 +121,27 @@ export function assertStoppableSession(api: ScreenRecordingSessionApi, stage: st } /** - * Reject a start whose readiness resumed after the session was disposed - * (process shutdown). Call synchronously right before spawn, with no await - * between this check and the spawn/pendingChild stamp, so no capture is - * launched that dispose's teardown can no longer see and reap. + * Reject a start whose readiness resumed after the session was disposed. Call + * synchronously right before spawn, with no await between this check and the + * spawn/pendingChild stamp, so no capture is launched that dispose's teardown + * can no longer see and reap. + * + * `dispose()` runs on process shutdown, but ALSO whenever + * `stop-all-simulator-servers` reaps this device — `ScreenRecordingSession` is a + * device-owned namespace, so a session-end teardown (commonly another agent's) + * disposes it. The two are indistinguishable from `api.disposed` alone, so the + * message names both and does not tell the caller a retry is pointless: on the + * teardown branch the device is usually still up and starting again succeeds. + * (`SCREEN_RECORDING_SERVER_SHUTTING_DOWN` is the enum carried into telemetry; + * the shutdown wording there is historical, not a second claim of the cause.) */ export function assertNotDisposed(api: ScreenRecordingSessionApi, stage: string): void { if (api.disposed) { throw new FailureError( - `The tool-server is shutting down; screen recording was not started on device ${api.deviceId}.`, + `The screen-recording session for device ${api.deviceId} was torn down while this start ` + + `was still initializing, so nothing was recorded. That is either the tool-server shutting ` + + `down or a stop-all-simulator-servers reaping this device (e.g. another agent ending its ` + + `session). If the device is still up, start the recording again.`, { error_code: FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN, failure_stage: stage, From 2d53f6cdffe2329d0a2da2e019b349d0c9fba101 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 14:11:09 +0200 Subject: [PATCH 21/98] fix(flow): name the flow file when an atomic write fails, and correct the recording-map comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed temp-file swap surfaced the raw errno against the internal .argent-flow--.tmp scratch path — a file already deleted by the cleanup and named in no agent-facing doc. Rethrow FLOW_FILE_WRITE_FAILED naming the flow file and its directory (the real cause), keeping the errno as cause. Comment corrections: the not-found branch no longer lists 'superseded' as a populated-file state (a superseded key is held by the superseding session, so it resolves to success and that restart already truncated the file); the scratch pid separates this tool-server from a second install bundle, not a CLI (which writes its destination directly). --- packages/registry/src/failure-codes.ts | 1 + .../tool-server/src/tools/flows/flow-utils.ts | 52 +++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index 8f2139597..ce5d3d572 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -222,6 +222,7 @@ export const FAILURE_CODES = { FLOW_NAME_INVALID: "FLOW_NAME_INVALID", FLOW_NO_ACTIVE_RECORDING: "FLOW_NO_ACTIVE_RECORDING", FLOW_FILE_INVALID: "FLOW_FILE_INVALID", + FLOW_FILE_WRITE_FAILED: "FLOW_FILE_WRITE_FAILED", FLOW_ENTRY_UNRECOGNIZED: "FLOW_ENTRY_UNRECOGNIZED", FLOW_E2E_HAS_PREREQUISITE: "FLOW_E2E_HAS_PREREQUISITE", FLOW_DEVICE_RESOLUTION: "FLOW_DEVICE_RESOLUTION", diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 03560fb16..cab56ef62 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -401,11 +401,15 @@ export function requireRecordingSession(projectRoot: string, name: string): Reco : `none in this project${others}`; // Do NOT tell the agent to just call flow-start-recording. This message is // reached when the key was never started, but equally when a take was - // finished, superseded, or dropped by the MAX_RECORDINGS backstop — and in - // those cases the flow file on disk is fully populated while no session - // owns it. flow-start-recording truncates unconditionally, so the advice - // that recovers the first case destroys the others. Same doctrine as - // {@link assertSessionStillLive}, which faces the identical ambiguity. + // finished or dropped by the MAX_RECORDINGS backstop — and in those cases + // the flow file on disk is fully populated while no session owns it. (A key + // SUPERSEDED by a restart is NOT one of them: the superseding session holds + // the key, so this call resolves to it and returns success rather than + // reaching here — and that restart has already truncated the file, so + // "fully populated" would not hold there anyway.) flow-start-recording + // truncates unconditionally, so the advice that recovers the never-started + // case destroys the others. Same doctrine as {@link assertSessionStillLive}, + // which faces the identical ambiguity. throw new FailureError( `No active recording for flow "${name}" in ${projectRoot}. ` + `If you have not started it yet, call flow-start-recording — but note it ` + @@ -2374,8 +2378,14 @@ export function parseFlow(content: string): FlowFile { /** * Suffix counter for {@link writeFlowFile}'s scratch file. Paired with the pid, - * this keeps two concurrent writers — in this process or in a CLI sharing the - * directory — off each other's temp file. + * this keeps two concurrent writers off each other's temp file: the counter + * separates writers inside this process, and the pid separates this process + * from a SECOND tool-server — a different install bundle can record the same + * `(project_root, name)` and compute the same scratch path (see the + * cross-install note on {@link recordings}). The CLI is not one of the writers: + * it writes the destination flow file directly and mints no scratch file, and + * host and client persist modes are mutually exclusive per call, so no CLI is + * writing this directory while the tool-server is. */ let flowWriteSeq = 0; @@ -2404,8 +2414,9 @@ let flowWriteSeq = 0; * legitimately run to NAME_MAX — and prefixing that with a discriminator would * push the scratch name past the limit, turning an append that used to work * into ENAMETOOLONG. pid + counter is unique on its own: the counter separates - * writers inside this process, the pid separates this process from any CLI - * sharing the directory. + * writers inside this process, the pid separates this process from a second + * tool-server (a different install bundle) that could be writing the same + * directory. * * The swap costs two things a write-through would have kept, both accepted for * the atomicity: it needs write permission on the DIRECTORY rather than on the @@ -2427,7 +2438,28 @@ async function writeFlowFile(filePath: string, content: string): Promise { // fail with the file already created (ENOSPC, EIO), so this has to cover it // too — nothing else ever sweeps this directory. await fs.rm(tmpPath, { force: true }).catch(() => {}); - throw err; + // Rethrow against the flow file, never the scratch path. The temp name is + // an internal detail — pid+counter suffixed, and already removed above — so + // surfacing its raw errno (`EACCES: … open '.argent-flow--.tmp'`) + // would name a file that no longer exists and never mention the flow. The + // swap writes a sibling and renames, so the actual cause is write + // permission (or space) on the DIRECTORY: name the flow file and the + // directory, and keep the original errno as `cause`. + const code = + err instanceof Error && typeof (err as NodeJS.ErrnoException).code === "string" + ? (err as NodeJS.ErrnoException).code + : undefined; + throw new FailureError( + `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — an append replaces ` + + `the file via a sibling temp file and rename, so ${path.dirname(filePath)} must be writable.`, + { + error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, + failure_stage: "flow_file_write", + failure_area: "tool_server", + error_kind: "unknown", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); } } From 73f1e19b2b724c43bb3297310ebdb7116c84ff6c Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 14:11:09 +0200 Subject: [PATCH 22/98] docs: correct the device-services ownership, matcher, and device-arg comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ChromiumJsRuntimeDebugger owns a loopback server, a log handle and its console history, but NOT the CDP socket (its dispose leaves that to ChromiumCdp; no Metro on chromium) — which is why the transport-teardown cascade holds. - The two stop tools share one URN matcher but keep different namespace SETS by design; drop the 'drifted apart / same udid reaped different services' past-tense that a squash-merge unmoors and that is still true by design. - extractDeviceArg reads the scoped-stop 'devices' spelling; the capability-gate, platformFromArgs and recordChildInvocation enumerations name all three. - captureRunTarget can return flow AND warning together; the JSDoc says so. --- packages/registry/src/types.ts | 12 ++++---- packages/tool-server/src/http.ts | 28 +++++++++++++------ .../src/tools/flows/flow-add-step.ts | 4 --- .../src/tools/simulator/device-services.ts | 25 +++++++++++------ .../tools/simulator/stop-simulator-server.ts | 10 +++---- 5 files changed, 48 insertions(+), 31 deletions(-) diff --git a/packages/registry/src/types.ts b/packages/registry/src/types.ts index 4036dc6b6..3c3321410 100644 --- a/packages/registry/src/types.ts +++ b/packages/registry/src/types.ts @@ -111,11 +111,13 @@ export interface InvokeToolOptions { * * The outer request's AI client is inherited unchanged. The platform is * re-derived from each sub-tool's own `childArgs` (its `udid` / `device_id` / - * `avdName`), falling back to the outer request's platform when the sub-tool - * carries no device arg — an orchestrator like flow-execute has no platform of - * its own and a single flow can target several devices, so the child's device - * arg is the only correct platform source. Opaque to the registry — it neither - * reads nor validates the recorded metadata. + * `devices` / `avdName`), falling back to the outer request's platform when the + * sub-tool carries no device arg — an orchestrator like flow-execute has no + * platform of its own and a single flow can target several devices, so the + * child's device arg is the only correct platform source. (A replayed + * `stop-all-simulator-servers` step carries `devices`, injected by + * `bindDeviceArgs`, so it resolves rather than falling back.) Opaque to the + * registry — it neither reads nor validates the recorded metadata. */ recordChildInvocation?: (toolInvocationId: string, childArgs?: unknown) => () => void; /** diff --git a/packages/tool-server/src/http.ts b/packages/tool-server/src/http.ts index 07ca8a5ea..afeaa07bf 100644 --- a/packages/tool-server/src/http.ts +++ b/packages/tool-server/src/http.ts @@ -143,6 +143,14 @@ function extractDeviceArg(data: unknown): string | null { const record = data as Record; if (typeof record.udid === "string") return record.udid; if (typeof record.device_id === "string") return record.device_id; + // `devices: string[]` is a third spelling, used only by + // `stop-all-simulator-servers`' scoped teardown. A call can name several + // devices of different platforms; the first is enough for the coarse + // telemetry platform. It never reaches the capability gate — that tool + // declares no capability — so this is a telemetry-only refinement. + if (Array.isArray(record.devices) && typeof record.devices[0] === "string") { + return record.devices[0]; + } return null; } @@ -199,9 +207,10 @@ function extractInvocationMeta( /** * Telemetry platform from a tool call's device arg, or null when it carries none. - * A `udid` / `device_id` resolves through the runtime-kind cache and refines to - * `tvos` / `android-tv` once that cache is warm (coarse `ios` / `android` until - * then); only the `avdName`-only fallback is unconditionally coarse. + * A `udid` / `device_id` (or the first of a scoped stop-all's `devices`) resolves + * through the runtime-kind cache and refines to `tvos` / `android-tv` once that + * cache is warm (coarse `ios` / `android` until then); only the `avdName`-only + * fallback is unconditionally coarse. */ function platformFromArgs(data: unknown): TelemetryPlatform | null { if (!data || typeof data !== "object") return null; @@ -737,11 +746,14 @@ export function createHttpApp(registry: Registry, options?: HttpAppOptions): Htt // Cross-platform tools double-check inside their dispatch helper, so // non-HTTP callers (run-sequence, flow-run) are also covered. // - // Tools spell the device parameter two ways — `udid` (legacy iOS-only - // tools and gestures) and `device_id` (debugger / profiler / network - // tools). Honour both so an Android serial reaching an iOS-only - // device_id-tool is rejected at the gate instead of falling through - // to the deeper blueprint error (which surfaces as a generic 500). + // Tools spell the device parameter three ways — `udid` (legacy iOS-only + // tools and gestures), `device_id` (debugger / profiler / network tools), + // and `devices` (only `stop-all-simulator-servers`' scoped teardown). + // `extractDeviceArg` honours all three so an Android serial reaching an + // iOS-only device_id-tool is rejected at the gate instead of falling + // through to the deeper blueprint error (which surfaces as a generic 500). + // Only the first two ever reach this gate — the `devices` tool declares no + // capability — but the third is read the same way for telemetry platform. const deviceArg = extractDeviceArg(parsedData); if (def.capability && deviceArg) { try { diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index 036625767..f7eae0e3b 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -526,10 +526,6 @@ If a step was recorded by mistake, edit the .yaml file directly to remove it.`, step = { kind: "launch", app: strippedArgs.bundleId as string }; } else if (runTarget?.flow) { step = { kind: "run", flow: runTarget.flow }; - // A resolved target can still carry a warning (a same-named sibling in - // another project), so this branch surfaces it too — not only the - // kept-the-raw-step one below. - warning = runTarget.warning; } else { warning = runTarget?.warning; // The step ran live with the full args (incl. the device id), but the diff --git a/packages/tool-server/src/tools/simulator/device-services.ts b/packages/tool-server/src/tools/simulator/device-services.ts index e02c87f89..2fb0e3161 100644 --- a/packages/tool-server/src/tools/simulator/device-services.ts +++ b/packages/tool-server/src/tools/simulator/device-services.ts @@ -15,13 +15,16 @@ import { REACT_PROFILER_SESSION_NAMESPACE } from "../../blueprints/react-profile /** * Which services one device id owns — the single definition of that mapping, * shared by `stop-simulator-server` (one device, transport scope) and - * `stop-all-simulator-servers` (every device-owned service). The two tools had - * two separate matchers that drifted apart: one was case-sensitive and blind to - * the `:tcp` suffix, so the same udid reaped different services depending on - * which tool the agent reached for. + * `stop-all-simulator-servers` (every device-owned service). Both resolve a + * URN through one matcher here, so a given udid resolves to the same URNs for + * either tool — case-insensitively, and with the `:tcp` suffix understood. * - * Note this unifies how a URN is matched, not how a raw id is classified: - * `stop-simulator-server` still picks its namespace set from + * This unifies how a URN is MATCHED, not which namespaces each tool sweeps: + * `stop-simulator-server` deliberately scopes to the transport session (see + * {@link transportNamespacesForPlatform}) while `stop-all-simulator-servers` + * takes every {@link DEVICE_OWNED_NAMESPACES} entry, so the same udid still + * reaps a different SET through each tool — by design. Nor does it unify how a + * raw id is CLASSIFIED: `stop-simulator-server` picks its namespace set from * `resolveDevice().platform`, whose prefix tests are case-SENSITIVE, so an id * spelled in the wrong case can still land on the wrong namespace set there. */ @@ -107,9 +110,13 @@ const PORT_KEYED_NAMESPACES: readonly string[] = [ * on-device perfetto process plus its trace file. * - `JsRuntimeDebugger` owns a bound loopback HTTP/WebSocket server, the CDP * socket to Metro, and a log file handle. - * - `ChromiumJsRuntimeDebugger` owns the same, plus its captured console - * history. Its dependency (`ChromiumCdp`) is listed here too, so a scoped - * stop reaches it twice over — as it does the two `JsRuntimeDebugger` + * - `ChromiumJsRuntimeDebugger` owns a bound loopback server, a log handle and + * its captured console history — but NOT the CDP socket: its `dispose()` + * deliberately leaves that to `ChromiumCdp` (and there is no Metro on the + * chromium path). That is precisely why the narrowness note below holds — + * disposing `ChromiumCdp` cascades to this one BECAUSE this one does not own + * the transport. Its dependency (`ChromiumCdp`) is listed here too, so a + * scoped stop reaches it twice over — as it does the two `JsRuntimeDebugger` * dependents, whose own dependency is equally listed. All three are here for * the naming, not for the reaping. What is particular to this one is its URN * SHAPE: `:`, not port-keyed like the other two dependents, so diff --git a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts index de9fc6caf..0366691d2 100644 --- a/packages/tool-server/src/tools/simulator/stop-simulator-server.ts +++ b/packages/tool-server/src/tools/simulator/stop-simulator-server.ts @@ -39,11 +39,11 @@ export function createStopSimulatorServerTool( const snapshot = registry.getSnapshot(); let stopped = false; // Scanned rather than looked up by exact URN, so this agrees with - // `stop-all-simulator-servers` on which services a device id owns. The - // live difference is case: an exact `services.get()` silently no-op'd on - // a lower-cased UDID that the scoped stop-all reaped. (The shared matcher - // also understands the `:tcp` suffix, which no namespace in this tool's - // set currently emits — it costs nothing and keeps one grammar.) + // `stop-all-simulator-servers` on which services a device id owns — in + // particular the match is case-insensitive, where an exact + // `services.get()` would silently no-op on a lower-cased UDID. (The shared + // matcher also understands the `:tcp` suffix, which no namespace in this + // tool's set currently emits — it costs nothing and keeps one grammar.) const urns = [...snapshot.services.keys()].filter( (urn) => deviceIdOwningUrn(urn, namespaces, [udid]) !== undefined ); From efd15d6a5b0b4df65d3ac3739171287dfc1ebb8e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 14:11:21 +0200 Subject: [PATCH 23/98] test(flow): pin the coverage gaps, and tighten the create-flow skill New pins: the restart-vs-eviction discard report; the write-half of the atomic swap (ENOSPC after the temp file exists) plus the flow-named error; every recording-tool interaction formatter naming its flow; a negative control for the unscoped sweep's namespace filter; the android narrowness of stop-simulator-server. Reword the counterfactual 'was reported unmatched' / 'used to look up' test comments to durable properties. Trim the create-flow skill (redundant table clauses, discardedSteps/remote minutiae, an over-long chromium-boot and debugger aside) and fix the superseded-state drift so the skill and code agree. --- .../skills/skills/argent-create-flow/SKILL.md | 14 +-- .../flows/flow-concurrent-recording.test.ts | 95 +++++++++++++++++++ .../test/flows/flow-remote-recording.test.ts | 5 +- .../tool-server/test/flows/flow-tools.test.ts | 2 + .../test/interaction-messages.test.ts | 30 ++++++ packages/tool-server/test/stop-tools.test.ts | 92 ++++++++++++++---- 6 files changed, 210 insertions(+), 28 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 7a972cb28..fb542762f 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -129,9 +129,9 @@ The standalone command uses only the auto-started local tool server. It is unava | Tool | Purpose | | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | | `flow-start-recording` | Start recording — takes `name` + `project_root` and (fragments only) an optional `executionPrerequisite`; creates the file, truncating any existing one | -| `flow-add-step` | Execute a tool call live and, if it succeeds, record it into the flow named by `name` + `project_root` | -| `flow-add-echo` | Add a label/comment that prints during replay, into the flow named by `name` + `project_root` | -| `flow-finish-recording` | Stop recording the flow named by `name` + `project_root` and get a summary | +| `flow-add-step` | Execute a tool call live and record it if it succeeds | +| `flow-add-echo` | Add a label/comment that prints during replay | +| `flow-finish-recording` | Stop recording and get a summary | | `flow-read-prerequisite` | Read a flow's execution prerequisite without running it (same `name`/`flow_path` sources) | | `flow-execute` | Replay a flow — a saved one by `name`, or any flow YAML by absolute `flow_path` | @@ -140,9 +140,9 @@ Every tool during recording returns the current flow file contents, so you can t - **Every step runs live.** You see the real tool result (including screenshots) — verify the step worked before continuing. **Only successful steps are recorded**: a failed call writes nothing to the flow file; fix the issue and try again. - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. -- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` counts the `.yaml` as it stood at the reset — a hand-edit made mid-recording is included — and is omitted entirely when that file could not be read or parsed, so `restarted: true` can arrive without it. That is for a project root on the tool-server host; against a remote client the host never sees your `.yaml`, so there the number counts only the steps recorded through the server. Starting a _different_ flow abandons nothing. +- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` (in the return value) counts the discarded take, but can be absent even on a restart. Starting a _different_ flow abandons nothing. - **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. -- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished, superseded by another agent, or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. +- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. (A takeover by another agent is different: it resolves to _their_ recording and succeeds — see the previous bullet — rather than reaching this error.) The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. ### flow-add-step arguments @@ -205,7 +205,7 @@ Then polish the saved file: the two `await-ui-element` steps become `await:` dir ## Replaying -Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow can boot its own instance and tear it down after, but only when all of these hold: no `device`, the launch resolves to chromium (an explicit `platform: "chromium"`, or a single-key `launch: { chromium: … }` map), and that launch value is a real Electron app path on the tool-server host. With no chromium hint - a bare-string or multi-platform `launch:` and no `platform` - the run auto-detects a booted device instead. **Don't reach for `platform: "chromium"` to force the self-boot on a recorded flow:** it does not fall through, it selects the boot branch, and a bare-string `launch:` - what the recorder always writes - holds an installed-app _bundle id_, which that branch reads as an app path. The whole `flow-execute` call then fails with `Electron boot: path does not exist: …`. Hand-edit the launch to `{ chromium: }` first.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. +Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow boots and tears down its own instance, but only when the launch resolves to a real Electron app path — a `launch: { chromium: }` map, or `platform: "chromium"` with `device` unset. **Don't force it with `platform: "chromium"` on a recorded flow:** the recorder writes a bare-string `launch:` holding a bundle _id_, which the boot branch reads as an app path and fails with `Electron boot: path does not exist: …`. Hand-edit the launch to `{ chromium: }` first.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. **What each step reports.** Raw `tool:` steps include the underlying tool's full `result` (screenshots and other outputs render as usual). The directive steps are summarized: `tap`/`type`/`await`/`assert` report only `status` + `reason`, and `snapshot` adds `artifacts` only when there is something to look at — a failed comparison (baseline/current/diff paths), a missing-baseline failure (`current` only), or a baseline write; a clean pass reports just `status` + `reason`. So converting a `tool: gesture-tap` into a `tap:` directive during cleanup drops only that tap's (uninteresting) raw result — output-bearing tools like `screenshot` have no directive form and stay `tool:` steps, so their results keep flowing through. @@ -267,7 +267,7 @@ For silent misfires and partial divergence, echo annotations (see _Making flows `debugger-component-tree` is an **authoring aid only — never record a `debugger-*` step into a flow.** `device_id` is stripped at record time and re-injected at replay, but `port` is not a device-bind key, so a recorded debugger step carries whatever `port` it was given (or falls through to the 8081 default at replay) and runs against whatever Metro happens to be on that port. - When calling any `debugger-*` tool directly, mind the shared-Metro rules: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. And a legacy-inspector device (RN 0.72 / Vega) reports no `logicalDeviceId`, so it cannot be singled out of a Metro shared with other devices — give it its own Metro port. That last one does not rescue `debugger-component-tree` itself: it is capability-gated off Vega, and on a legacy-inspector RN 0.72 Hermes it fails fast with a coded error pointing you at `describe` (the binding it delivers the tree over is ACKed but never installed, and a probe at connect catches that) - so use `describe` there. + When calling any `debugger-*` tool directly, mind the shared-Metro rule: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. 4. Compare current state to what the failed step expected. Classify the root cause: diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index bb3f3ada3..ef11aee76 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -664,6 +664,44 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(listActiveRecordings()).toEqual([]); }); + it("cleans up the scratch file and names the flow when the WRITE half fails", async () => { + // The "failed swap" case above reaches only `fs.rename`; a read-only dir + // fails even earlier, at the temp open, before a file exists. Neither + // exercises the other live trigger the cleanup exists for: the write itself + // failing (ENOSPC / EIO) with the scratch file ALREADY created. Force + // exactly that — write the real temp file, then throw — and assert both that + // the scratch file is swept and that the surfaced error names the flow file + // rather than the internal `.argent-flow-*.tmp` path (which is gone by then). + const root = await makeRoot("write-fails"); + const target = flowPath(root, "alpha"); + await fs.mkdir(path.dirname(target), { recursive: true }); + + const realWriteFile = fs.writeFile; + const spy = vi.spyOn(fs, "writeFile").mockImplementationOnce(async (p, data, opts) => { + await realWriteFile(p as Parameters[0], data as string, opts as never); + const err: NodeJS.ErrnoException = new Error( + `ENOSPC: no space left on device, write ${String(p)}` + ); + err.code = "ENOSPC"; + throw err; + }); + + const err = await start(root, "alpha").catch((e: unknown) => e); + spy.mockRestore(); + + expect(err).toBeInstanceOf(Error); + const message = (err as Error).message; + // Names the flow file and the directory (the real cause), not the scratch path. + expect(message).toContain(target); + expect(message).not.toMatch(/\.argent-flow-\d+-\d+\.tmp/); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + + // The half-written scratch file must not survive in the committed flows dir. + const entries = await fs.readdir(path.dirname(target)); + expect(entries.filter((e) => e.endsWith(".tmp"))).toEqual([]); + expect(listActiveRecordings()).toEqual([]); + }); + it("never exposes an empty or unparseable file while appends are in flight", async () => { // The property the two inode assertions above encode, observed the way a // reader actually experiences it: poll the path as fast as the event loop @@ -1365,6 +1403,63 @@ describe("the concurrent-recording cap", () => { expect(getFailureSignal(late)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); expect(getFailureSignal(late)?.failure_stage).toBe("flow_require_recording"); }); + + it("reports a destructive restart even when eviction drops the key mid-restart", async () => { + // A restart reads the take it is discarding ONCE, at the top of its critical + // section, and drives BOTH `restarted` and `discardedSteps` off that read. + // It must not re-derive `restarted` from the map after the truncate: + // `evictIfOverCapacity` runs under another key's lock and can drop this key + // in the window between the read and the register, and a `restarted` read + // there would see the key already gone and report a restart that truncated a + // real take (its file already reset) as a plain fresh start. + const root = await makeRoot("restart-evict-race"); + const names = await fillRecordings(root); + + // rec-0 holds a real step and is the least-recently-used entry: record the + // step first, then touch every other recording, so rec-0's last use is + // oldest and the next overflow evicts exactly it. + await addStep(root, "rec-0", "real"); + for (const name of names.slice(1)) await addEcho(root, name, "touch"); + + // Park rec-0's restart on its own `countStepsOnDisk` read — after it has + // captured the live session, before it truncates or re-registers. + const target = flowPath(root, "rec-0"); + const arrived = openGate(); + const held = openGate(); + let gated = false; + const realReadFile = fs.readFile; + const spy = vi.spyOn(fs, "readFile").mockImplementation((async ( + p: unknown, + ...rest: unknown[] + ) => { + if (!gated && String(p) === target) { + gated = true; + arrived.open(); + await held.promise; + } + return (realReadFile as (...a: unknown[]) => Promise)(p, ...rest); + }) as unknown as typeof fs.readFile); + + const restarting = start(root, "rec-0"); + await arrived.promise; + + // A 33rd recording overflows the cap and evicts the LRU — rec-0's key — + // while the restart is parked with rec-0's live session already captured. + await start(root, "overflow"); + expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + + held.open(); + const res = await restarting; + spy.mockRestore(); + + // The take really was destroyed… + expect(await readMarkers(root, "rec-0")).toEqual([]); + // …and the result says so, rather than collapsing to a plain fresh start. + // Reading `restarted` from `startRecordingSession`'s post-eviction return + // instead leaves `restarted` undefined here, so this separates the two. + expect(res.restarted).toBe(true); + expect(res.discardedSteps).toBe(1); + }); }); // ── Recording a flow-execute step ──────────────────────────────────── diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index 43d5c0a74..6af42bb0c 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -139,9 +139,12 @@ describe("flow recording with a remote client (probe miss)", () => { const stepResult = await addStep.execute( {}, { + name: "remote-flow", + project_root: CLIENT_ROOT, command: "flow-execute", args: JSON.stringify({ name: "sub", project_root: CLIENT_ROOT, device: "RECORD-TIME-ID" }), - } + }, + remoteCtx() ); const directive = stepResult.savedTo as { content: string }; diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 04fa11fa9..bb254f570 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -617,6 +617,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-pinned", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ name: "elsewhere", project_root: tmpDir, device: "ABC" }), } diff --git a/packages/tool-server/test/interaction-messages.test.ts b/packages/tool-server/test/interaction-messages.test.ts index f2cd4d1fc..fa03071de 100644 --- a/packages/tool-server/test/interaction-messages.test.ts +++ b/packages/tool-server/test/interaction-messages.test.ts @@ -160,6 +160,36 @@ describe("tool interaction messages", () => { ).toBe("Restarted recording flow checkout, discarding 4 steps"); }); + it("names the flow in every recording-tool interaction line", () => { + // Recordings are concurrent, so several of these lines interleave in one log + // and an unqualified "flow recording" would not say which one died or + // finished. Only two of the twelve formatters on the four recording tools + // are pinned elsewhere (flow-start-recording.completedMsg above, + // flow-add-echo.completedMsg in the secrets test), so the other ten could + // silently revert to name-free wording. Hold every one to naming the flow — + // the property the concurrency support introduced — including the failure + // lines, which are the diagnostic when several recordings are live. + const definitions = definitionsById(createRegistry()); + const name = "checkout"; + const params = { name, project_root: "/tmp/proj", command: "gesture-tap", message: "note" }; + const result = { message: "", flowFile: "", savedTo: "project" as const }; + + for (const id of [ + "flow-start-recording", + "flow-add-step", + "flow-add-echo", + "flow-finish-recording", + ]) { + const i = definitions.get(id)!.interaction!; + expect(i.startedMsg!({ params }), `${id}.startedMsg`).toContain(name); + expect(i.completedMsg!({ params, result }), `${id}.completedMsg`).toContain(name); + expect( + i.failedMsg!({ params, error: new Error("raw error"), failureSignal }), + `${id}.failedMsg` + ).toContain(name); + } + }); + it("does not expose sensitive inputs", () => { const definitions = definitionsById(createRegistry()); const secret = "INTERACTION_MESSAGE_SECRET"; diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index e1f13f5ae..bd15718bd 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -158,12 +158,12 @@ describe("stop-simulator-server", () => { expect(registry.disposeService).toHaveBeenCalledWith("ChromiumCdp:chromium-cdp-9222"); }); - // Both stop tools resolve "which services does this device own" through one - // shared matcher. This tool used to look its URNs up with an exact, - // case-sensitive `services.get()`, which no-op'd on a mis-cased udid; stop-all - // took no device id at all and swept every matching namespace on the host, so - // there was no second opinion to compare against. Now that stop-all is scoped, - // the same spelling has to reach the same services through both. + // Both stop tools resolve "which services does this device own" through the + // one shared matcher in device-services.ts, so a given udid — whatever its + // case — reaches the same services through either. Case-insensitivity is the + // property that matters here: an exact `services.get()` would no-op on a + // mis-cased udid, leaving a device the caller believes it stopped still + // running while the scoped stop-all (which folds case) reaps it. it("matches a UDID case-insensitively, like the scoped stop-all does", async () => { // Agents pass through whatever spelling they were handed, and a case @@ -222,6 +222,30 @@ describe("stop-simulator-server", () => { expect(registry.disposeService).toHaveBeenCalledOnce(); expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${udid}`); }); + + it("leaves an android device's devtools service alone", async () => { + // The android twin of the iOS narrowness case above, and the branch the + // rationale in device-services.ts covers but no prior test did. + // stop-simulator-server is the wedged-transport recovery, and + // AndroidDevtools is the tree source an Android recording's selector capture + // runs on — dropping it on a retry degrades another agent's flow to + // coordinate taps, exactly what the narrow set exists to prevent. An + // `emulator-N` serial classifies as android, so widening the android branch + // to include AndroidDevtools would dispose it here and fail this case. + const serial = "emulator-5554"; + const services = new Map([ + [`SimulatorServer:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + [`AndroidDevtools:${serial}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopSimulatorServerTool(registry); + + const result = await tool.execute!({}, { udid: serial }); + + expect(result).toEqual({ stopped: true, udid: serial }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${serial}`); + }); }); describe("stop-all-simulator-servers", () => { @@ -247,6 +271,29 @@ describe("stop-all-simulator-servers", () => { expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:BBB"); }); + it("leaves a service whose namespace is not device-owned untouched", async () => { + // The negative control for the unscoped sweep's namespace filter. Every + // blueprint registered today is device-owned, so nothing real is left out — + // but `isDeviceServiceUrn` is the only guard between this machine-wide stop + // (the session-end call every agent makes) and any future non-device + // service, or a namespace added to the list by mistake. A synthetic + // out-of-set URN pins that the sweep is namespace-scoped, not "dispose + // everything": degrade `isDeviceServiceUrn` to `return true` and this fails. + const services = new Map([ + ["SimulatorServer:AAA", { state: ServiceState.RUNNING, dependents: [] }], + ["NotADeviceService:global-singleton", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, {}); + + expect(result).toEqual({ stopped: ["SimulatorServer:AAA"] }); + expect(registry.disposeService).toHaveBeenCalledOnce(); + expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:AAA"); + expect(registry.disposeService).not.toHaveBeenCalledWith("NotADeviceService:global-singleton"); + }); + // `stopped` is documented as "the services that were actually live and got // shut down". ChromiumJsRuntimeDebugger declares `getDependencies -> // ChromiumCdp`, so disposing the transport takes it down regardless — while @@ -504,9 +551,12 @@ describe("stop-all-simulator-servers device scoping", () => { const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); - // Upper-cased id against a lower-cased URN AND vice versa: passing the - // lower-cased spelling here would leave the upper/upper and lower/lower - // pairs matching, so only a contrived asymmetric mutation would be caught. + // MINE is upper-cased, and the snapshot pairs it against an upper-cased URN + // (`SimulatorServer:${MINE}`) and a lower-cased one + // (`NativeDevtools:${MINE.toLowerCase()}:tcp`) — so this exercises + // upper-id/upper-URN and upper-id/lower-URN. The reverse direction (a + // lower-cased id against an upper-cased URN) is covered by a separate case + // below; both must match for a case mismatch never to silently no-op. const result = await tool.execute!({}, { devices: [MINE] }); expect(result).toEqual({ @@ -693,10 +743,11 @@ describe("stop-all-simulator-servers unmatched ids", () => { it("stops AXService and does not call a describe-only iOS session a typo", async () => { // An iOS session that only ran boot/launch/describe owns `AXService:` - // and nothing else — nothing cascades to it from SimulatorServer. While that - // namespace was outside the tool's set, the mandated session-end call both - // left the in-sim ax daemon (spawned --timeout 3600) running AND reported - // the perfectly correct UDID as unmatched, i.e. as a mistyped id. + // — and also `NativeDevtools:`, which bootIos and launch-app resolve + // unconditionally (omitted from this snapshot to isolate the AXService + // case). `AXService` is a device-owned namespace holding the in-sim ax + // daemon (spawned --timeout 3600), so a scoped stop reaps it AND does not + // report the correct UDID as unmatched: it owns a real service, not a typo. const services = new Map([ [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], ]); @@ -733,9 +784,9 @@ describe("stop-all-simulator-servers unmatched ids", () => { it("owns and stops a device whose only service is a screen recording", async () => { // ScreenRecordingSession holds an ffmpeg child, an MJPEG frame stream and // the touch-visualizer overlay it enabled on the device, and nothing - // cascades to it. While it was outside the namespace set, a session that - // ran screen-recording-start and then the mandated teardown left ffmpeg - // running and was told its correct serial was a mistyped id. + // cascades to it. It is a device-owned namespace, so a session that ran + // screen-recording-start is correctly reaped by a scoped stop and its + // serial is not reported as a mistyped id. const services = new Map([ [`ScreenRecordingSession:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], ]); @@ -765,10 +816,11 @@ describe("stop-all-simulator-servers unmatched ids", () => { it("scopes the port-keyed debugger URNs to the right device", async () => { // JsRuntimeDebugger's URN interposes the Metro port: `::`. - // Matched as `:` it belongs to nobody, so a debugger-only session - // was reported unmatched while its bound port and Metro CDP socket stayed - // open. Both devices sit behind the SAME port, so this also pins that the - // port is not what the scoping keys on. + // Matched as `:` it would belong to nobody, so a debugger-only + // session's serial would read as unmatched while its bound port and Metro + // CDP socket stayed open — the port-keyed match is what prevents that. Both + // devices sit behind the SAME port, so this also pins that the port is not + // what the scoping keys on. const services = new Map([ [`JsRuntimeDebugger:8081:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], [`JsRuntimeDebugger:8081:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], From 3d9a6f9304b8088ebe738c5f53b40dc3c3e7058a Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 08:16:51 +0200 Subject: [PATCH 24/98] fix(flow): count a device-list arg as acting on a device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stop-all-simulator-servers` gained a `devices` scope in this branch, and `DEVICE_BIND_LIST_KEYS` was added so the runner strips it at record time and rebinds it at replay. `DEVICE_ARG_KEYS` — what `toolRequiresDevice` consults — was not extended, and that tool declares no other property, so a recorded teardown step counted as needing no device: the run resolved `device: null` and the binding rebound the scope to `[""]`. The result was worse than no injection. The replayed teardown named an id that owns nothing, reaped nothing, and reported pass — the exact failure the list binding was added to prevent. Derive `DEVICE_ARG_KEYS` from both bind sets so a key added to either is covered by construction, and correct the runner comment that claimed a device-less step can only be one whose tool declares no device argument. --- .../src/tools/flows/flow-device.ts | 25 +++++--- .../tool-server/src/tools/flows/flow-run.ts | 8 ++- .../test/flows/flow-deviceless.test.ts | 59 +++++++++++++++++-- 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index d715e1d10..c34b3351a 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -34,14 +34,6 @@ export type FlowPlatform = WhenPlatform; */ const DEVICE_BIND_KEYS = ["udid", "device_id", "device"] as const; -/** - * Keys that mean a tool acts on a device. A superset of the keys the runner - * injects: `device` names one without receiving the run's own (a nested flow - * takes it that way), and a step that drives a device must count as needing one - * even when the runner does not hand it over. - */ -const DEVICE_ARG_KEYS = [...DEVICE_BIND_KEYS, "device"] as const; - /** * Args keys holding a LIST of device ids. Same treatment as * {@link DEVICE_BIND_KEYS} — stripped at record time, re-injected at replay — @@ -64,6 +56,23 @@ const DEVICE_ARG_KEYS = [...DEVICE_BIND_KEYS, "device"] as const; */ const DEVICE_BIND_LIST_KEYS = ["devices"] as const; +/** + * Keys that mean a tool acts on a device — every key either bind set covers. + * + * `toolRequiresDevice` consults this, and `resolveRunDevice` skips resolving a + * device for a flow no step here matches. So a key that is BOUND but not listed + * here is worse than an unbound one: the run resolves `device: null`, and + * `bindDeviceArgs(…, device?.id ?? "", …)` then rebinds the recorded value to + * the empty string rather than leaving it alone. For `devices` that is + * `{ devices: [""] }` — a teardown scoped to an id that owns nothing, which + * reaps nothing and still reports pass, exactly the failure + * {@link DEVICE_BIND_LIST_KEYS} exists to prevent. + * + * Both sets, therefore, and not a hand-maintained superset: a key added to + * either one is covered here by construction. + */ +const DEVICE_ARG_KEYS = [...DEVICE_BIND_KEYS, ...DEVICE_BIND_LIST_KEYS] as const; + interface RawDevice { platform: FlowPlatform; state?: string; diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index f025fbaa3..4e4c97e2e 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -2114,9 +2114,11 @@ async function execLeafStep( } case "tool": { - // With no device, the step reached here only because its tool declares no - // device argument, so there is nothing to inject — binding still strips - // any device key the recorded args carried. + // A device-less run reaches here only for a tool declaring none of + // `DEVICE_ARG_KEYS`, so binding injects nothing and merely strips any + // device key the recorded args carried. The `?? ""` is unreachable in + // that pairing and must stay unreachable: injecting the empty string + // would not fail the step, it would silently retarget it at no device. const args = bindDeviceArgs(registry, step.name, device?.id ?? "", step.args); const outputHint = registry.getTool(step.name)?.outputHint; if (step.delayMs && !(await sleepOrAbort(step.delayMs, signal))) { diff --git a/packages/tool-server/test/flows/flow-deviceless.test.ts b/packages/tool-server/test/flows/flow-deviceless.test.ts index 8b3eec9a9..63cc33fee 100644 --- a/packages/tool-server/test/flows/flow-deviceless.test.ts +++ b/packages/tool-server/test/flows/flow-deviceless.test.ts @@ -3,9 +3,11 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import type { Registry } from "@argent/registry"; +import { zodObjectToJsonSchema } from "@argent/registry"; import { createRunFlowTool, type FlowRunResult } from "../../src/tools/flows/flow-run"; import { serializeFlow, type FlowStep } from "../../src/tools/flows/flow-utils"; import { stepRequiresDevice } from "../../src/tools/flows/flow-device"; +import { createStopAllSimulatorServersTool } from "../../src/tools/simulator/stop-all-simulator-servers"; const DEVICE = "00000000-0000-0000-0000-0000000000ab"; let tmpDir: string; @@ -17,8 +19,11 @@ let tmpDir: string; const TOOLS: Record = { "tap": { inputSchema: { properties: { udid: {}, x: {}, y: {} } } }, "stop-metro": { inputSchema: { properties: { port: {} } } }, - // A real tool that declares no input at all. - "stop-all-simulator-servers": {}, + // Declares a device LIST rather than a single id — the shape the runner has + // to rebind to the run device, and therefore one that makes a step need one. + "stop-all-simulator-servers": { inputSchema: { properties: { devices: {} } } }, + // A tool that declares no input at all. + "gather-workspace-data": {}, // Takes a device without receiving the run's own. "flow-execute": { inputSchema: { properties: { name: {}, device: {} } } }, }; @@ -141,10 +146,10 @@ describe("a flow that touches no device", () => { it("runs a tool step whose tool declares no input at all", async () => { // A tool with no schema must not be mistaken for one that needs a device, // and reading its absent schema must not throw. - await writeFlow("stop-all", [{ kind: "tool", name: "stop-all-simulator-servers", args: {} }]); + await writeFlow("no-schema", [{ kind: "tool", name: "gather-workspace-data", args: {} }]); const { registry } = mockRegistry({ booted: [] }); - expect(asRun(await runAuto(registry, "stop-all")).ok).toBe(true); + expect(asRun(await runAuto(registry, "no-schema")).ok).toBe(true); }); it("runs an empty flow", async () => { @@ -282,7 +287,51 @@ describe("stepRequiresDevice", () => { expect(stepRequiresDevice(registry, toolStep("tap"))).toBe(true); expect(stepRequiresDevice(registry, toolStep("flow-execute"))).toBe(true); expect(stepRequiresDevice(registry, toolStep("stop-metro"))).toBe(false); - expect(stepRequiresDevice(registry, toolStep("stop-all-simulator-servers"))).toBe(false); + expect(stepRequiresDevice(registry, toolStep("gather-workspace-data"))).toBe(false); expect(stepRequiresDevice(registry, toolStep("not-a-tool"))).toBe(true); }); + + it("counts the REAL stop-all-simulator-servers schema as acting on a device", () => { + // Against the derived JSON schema, not the mock above: the mock is only as + // good as its agreement with the tool, and the failure this guards is + // exactly a drift between the two — the tool declaring a device key that + // `DEVICE_ARG_KEYS` does not list. Catches a rename of `devices` too. + const schema = zodObjectToJsonSchema( + createStopAllSimulatorServersTool({} as unknown as Registry).zodSchema! + ); + expect(Object.keys((schema as { properties: Record }).properties)).toContain( + "devices" + ); + const registry = { getTool: () => ({ inputSchema: schema }) } as unknown as Registry; + expect( + stepRequiresDevice(registry, { kind: "tool", name: "stop-all-simulator-servers", args: {} }) + ).toBe(true); + }); + + it("counts a device LIST argument as acting on a device", () => { + // `stop-all-simulator-servers` spells its scope `devices`, the only tool + // that does. Missing it here is not a missing injection but a wrong one: + // the run resolves no device, and the binding then rebinds the recorded + // scope to `[""]` — a teardown that reaps nothing and still reports pass. + const { registry } = mockRegistry(); + expect( + stepRequiresDevice(registry, { kind: "tool", name: "stop-all-simulator-servers", args: {} }) + ).toBe(true); + }); +}); + +describe("a recorded teardown step", () => { + it("replays against the run device, not against an empty scope", async () => { + await writeFlow("teardownonly", [ + // What the recorder writes for a scoped `stop-all-simulator-servers`: + // the `devices` key is stripped at record time and re-injected here. + { kind: "tool", name: "stop-all-simulator-servers", args: {} }, + ]); + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE] }); + const run = asRun(await runAuto(registry, "teardownonly")); + + expect(run.device).toBe(DEVICE); + expect(run.ok).toBe(true); + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { devices: [DEVICE] }); + }); }); From 197c99ade9e419c69fdd86a58236ed8c180768ae Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 08:23:39 +0200 Subject: [PATCH 25/98] fix(screen-recording,profiler): say a teardown reaped the session, not that none existed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widening the teardown from 6 namespaces to 13 made `ScreenRecordingSession` and `NativeProfilerSession` reachable from `stop-all-simulator-servers` for the first time. Reaping them is the point — each owns a spawned encoder or trace daemon that must not outlive the session — but `Registry._teardown` nulls the node's instance, so the owner's next `screen-recording-stop` resolved a brand new session and answered: No active screen recording on device . Call `screen-recording-start` first. while a finalized, playable video sat orphaned on disk. `native-profiler-stop` answered the same for a capture that had been running seconds earlier. That is the one thing that is certainly false. Nothing else in the process still knew the capture had happened, or where its output landed. Have the disposer leave a breadcrumb — only when it is destroying something unretrieved — and have the stop guards report the teardown in place of the absence, naming the salvaged video where there is one and saying plainly that there is nothing to salvage where there is not. Breadcrumbs are consumed by the read, and dropped by a subsequent start, so one can never be left to blame a later, genuine "you never started one". --- .../src/blueprints/native-profiler-session.ts | 29 +++++ .../blueprints/screen-recording-session.ts | 24 ++++ .../native-profiler/platforms/android.ts | 12 +- .../profiler/native-profiler/platforms/ios.ts | 13 +- .../src/tools/screen-recording/capture.ts | 5 + .../tools/screen-recording/session-guards.ts | 16 ++- .../tool-server/src/utils/reaped-sessions.ts | 97 ++++++++++++++ .../native-profiler-reaped-session.test.ts | 121 ++++++++++++++++++ .../tool-server/test/screen-recording.test.ts | 82 ++++++++++++ 9 files changed, 395 insertions(+), 4 deletions(-) create mode 100644 packages/tool-server/src/utils/reaped-sessions.ts create mode 100644 packages/tool-server/test/native-profiler-reaped-session.test.ts diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index e8ad1a503..5618b6950 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -11,6 +11,7 @@ import type { ChildProcess } from "child_process"; import type { CpuSample, UiHang, MemoryLeak, CpuHotspot } from "../utils/ios-profiler/types"; import { waitForChildExit } from "../utils/profiler-shared/lifecycle"; import { adbShell } from "../utils/adb"; +import { recordReapedSession } from "../utils/reaped-sessions"; import { disposeWarmEngine } from "@argent/native-devtools-android"; // Cross-platform session for the `native-profiler-*` tools: iOS uses an xctrace @@ -179,6 +180,13 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< clearTimeout(state.recordingTimeout); state.recordingTimeout = null; } + // Read before the teardown below clears it. A capture killed here is + // destroyed rather than salvaged — no SIGINT finalize grace, and on + // Android the on-device trace is removed outright — so the breadcrumb + // exists purely so `native-profiler-stop` stops answering "call + // native-profiler-start first" for a session that really did run. + const abandonedCapture = state.profilingActive; + const abandonedTrace = state.traceFile; if (state.platform === "ios") { const child = state.captureProcess; @@ -193,6 +201,17 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< } } finally { clearLiveState(state); + if (abandonedCapture) { + recordReapedSession( + "native-profiler", + state.deviceId, + abandonedTrace + ? `xctrace was killed without its finalize pass, so the partial bundle at ` + + `${abandonedTrace} is very likely unreadable — re-profile rather than ` + + `trying to salvage it.` + : undefined + ); + } } return; } @@ -214,6 +233,16 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< } } finally { clearLiveState(state); + if (abandonedCapture) { + recordReapedSession( + "native-profiler", + state.deviceId, + // The on-device .pftrace is removed above, and nothing was pulled + // to the host yet, so there is genuinely nothing to point at. + "The perfetto daemon was killed and its on-device trace removed, so no trace " + + "survived — re-profile to capture again." + ); + } } // ANDROID: Free this trace's warm Perfetto engine (trace memory + wasm heap) now diff --git a/packages/tool-server/src/blueprints/screen-recording-session.ts b/packages/tool-server/src/blueprints/screen-recording-session.ts index 2c4ef8816..9e01f4af0 100644 --- a/packages/tool-server/src/blueprints/screen-recording-session.ts +++ b/packages/tool-server/src/blueprints/screen-recording-session.ts @@ -11,6 +11,7 @@ import type { ChildProcess } from "child_process"; import { promises as fs } from "fs"; import { waitForChildExit } from "../utils/profiler-shared/lifecycle"; import { clearActiveScreenRecording } from "../utils/screen-recording-reminder"; +import { recordReapedSession } from "../utils/reaped-sessions"; // Session for the `screen-recording-*` tools. One shape for every platform: // frames come from simulator-server's MJPEG stream and are paced into an ffmpeg @@ -205,6 +206,13 @@ export const screenRecordingSessionBlueprint: ServiceBlueprint< // await will observe this and abort instead of spawning an orphan the // teardown below can no longer reap. state.disposed = true; + // Whether this dispose is destroying an unretrieved capture, decided + // BEFORE the teardown below clears the flags it is read from. Both + // states owe the caller a video: one is still encoding, the other + // finished and is waiting to be handed over. + const hadUnretrievedCapture = + state.recordingActive || state.startPending || state.pendingRetrieval; + const abandonedOutput = state.outputFile; if (state.recordingTimeout) { clearTimeout(state.recordingTimeout); state.recordingTimeout = null; @@ -262,6 +270,22 @@ export const screenRecordingSessionBlueprint: ServiceBlueprint< clearLiveState(state); // The reminder must not outlive the process that owns the capture. clearActiveScreenRecording(state.deviceId); + // Leave a breadcrumb so the owner's `screen-recording-stop` reports + // the teardown instead of "you never started a recording". The stdin + // close above is ffmpeg's normal finalize path, so the file usually + // is playable — but nothing else would ever say it exists, and the + // next resolve builds a session that has never heard of it. + if (hadUnretrievedCapture) { + recordReapedSession( + "screen-recording", + state.deviceId, + abandonedOutput + ? `ffmpeg was given a moment to finalize the container first, so the video ` + + `captured up to that point is usually playable at ${abandonedOutput} — ` + + `check it before re-recording.` + : undefined + ); + } } }, events, diff --git a/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts b/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts index db3890101..b0d460975 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts @@ -1,6 +1,7 @@ import * as path from "path"; import { FAILURE_CODES, FailureError } from "@argent/registry"; import type { NativeProfilerSessionApi } from "../../../../blueprints/native-profiler-session"; +import { describeReapedSession, takeReapedSession } from "../../../../utils/reaped-sessions"; import { getDebugDir } from "../../../../utils/react-profiler/debug/dump"; import { startPerfetto, stopPerfetto } from "../../../../utils/android-profiler/capture"; import { @@ -82,6 +83,10 @@ export async function startNativeProfilerAndroid( api.androidOnDeviceTracePath = onDeviceTracePath; api.profilingActive = true; api.wallClockStartMs = Date.now(); + // This capture's own stop will succeed, so an earlier teardown breadcrumb + // would never be consumed — and would go on to blame a much later, genuine + // "no active session" on a teardown that had nothing to do with it. + takeReapedSession("native-profiler", api.deviceId); api.recordingTimeout = setTimeout(() => { // Best-effort SIGTERM to the on-device perfetto daemon; stop tool will pull @@ -117,8 +122,13 @@ export async function stopNativeProfilerAndroid( ): Promise { const recoveringPartialTrace = api.recordingTimedOut || api.recordingExitedUnexpectedly; if (!api.profilingActive && !recoveringPartialTrace) { + // See the iOS twin: a teardown leaves a fresh session behind, which is + // indistinguishable from one that never started without this breadcrumb. + const reaped = takeReapedSession("native-profiler", api.deviceId); throw new FailureError( - "No active native profiling session found. Call native-profiler-start first.", + reaped + ? describeReapedSession(reaped, "native profiling session") + : "No active native profiling session found. Call native-profiler-start first.", { error_code: FAILURE_CODES.NATIVE_PROFILER_NO_ACTIVE_SESSION, failure_stage: "android_native_profiler_stop", diff --git a/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts b/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts index 25e09a27a..58232c584 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts @@ -4,6 +4,7 @@ import { promises as fs } from "fs"; import { existsSync } from "node:fs"; import * as path from "path"; import type { NativeProfilerSessionApi } from "../../../../blueprints/native-profiler-session"; +import { describeReapedSession, takeReapedSession } from "../../../../utils/reaped-sessions"; import { deviceSetForUdid, simctlArgsForUdidSync } from "../../../../utils/ios-device-sets"; import { getDebugDir } from "../../../../utils/react-profiler/debug/dump"; import { @@ -777,6 +778,9 @@ export async function startNativeProfilerIos( api.cpuFilterPid = strategy ? strategy.cpuFilterPid(detected!) : null; api.profilingActive = true; api.wallClockStartMs = Date.now(); + // See the Android twin: a live capture makes any earlier teardown breadcrumb + // unconsumable, and therefore a future false accusation. + takeReapedSession("native-profiler", api.deviceId); api.recordingTimeout = setTimeout(() => { try { xctraceProcess.kill("SIGINT"); @@ -834,8 +838,15 @@ export async function stopNativeProfilerIos(api: NativeProfilerSessionApi): Prom } if (!api.profilingActive || !api.captureProcess || !api.traceFile) { + // A teardown reaps NativeProfilerSession and the registry nulls the + // instance, so `api` here can be a fresh session that never saw the capture + // this caller started. Say that happened rather than "you never started + // one" — the trace really is gone, but the reason is not the caller's. + const reaped = takeReapedSession("native-profiler", api.deviceId); throw new FailureError( - "No active native profiling session found. Call native-profiler-start first.", + reaped + ? describeReapedSession(reaped, "native profiling session") + : "No active native profiling session found. Call native-profiler-start first.", { error_code: FAILURE_CODES.NATIVE_PROFILER_NO_ACTIVE_SESSION, failure_stage: "native_profiler_stop_session_state", diff --git a/packages/tool-server/src/tools/screen-recording/capture.ts b/packages/tool-server/src/tools/screen-recording/capture.ts index 72c0f1772..74f87092d 100644 --- a/packages/tool-server/src/tools/screen-recording/capture.ts +++ b/packages/tool-server/src/tools/screen-recording/capture.ts @@ -10,6 +10,7 @@ import { markScreenRecordingFinalized, registerActiveScreenRecording, } from "../../utils/screen-recording-reminder"; +import { takeReapedSession } from "../../utils/reaped-sessions"; import { openMjpegStream, readJpegDimensions, type MjpegStream } from "./mjpeg-stream"; import { assertNoActiveRecording, @@ -393,6 +394,10 @@ async function startCaptureLocked( api.wallClockEndMs = null; api.timeLimitSeconds = params.timeLimitSeconds; registerActiveScreenRecording(api.deviceId, api.wallClockStartMs, params.timeLimitSeconds); + // A live capture makes any earlier teardown breadcrumb unreportable: this + // recording's own stop will succeed, so nothing would ever consume it, and it + // would be left to blame a much later, genuine "no active recording". + takeReapedSession("screen-recording", api.deviceId); startPump(api, stream); // Arm the exit handler BEFORE the pointer-enable await below. readiness diff --git a/packages/tool-server/src/tools/screen-recording/session-guards.ts b/packages/tool-server/src/tools/screen-recording/session-guards.ts index 8683c2768..3e4e3786b 100644 --- a/packages/tool-server/src/tools/screen-recording/session-guards.ts +++ b/packages/tool-server/src/tools/screen-recording/session-guards.ts @@ -1,6 +1,7 @@ import { promises as fs } from "fs"; import { FAILURE_CODES, FailureError } from "@argent/registry"; import type { ScreenRecordingSessionApi } from "../../blueprints/screen-recording-session"; +import { describeReapedSession, takeReapedSession } from "../../utils/reaped-sessions"; export interface StartRecordingResult { status: "recording"; @@ -106,10 +107,21 @@ export function assertStoppableSession(api: ScreenRecordingSessionApi, stage: st } const recoverable = api.pendingRetrieval && api.outputFile !== null; if (!api.recordingActive && !recoverable) { + // A teardown reaps this device's ScreenRecordingSession, and the registry + // nulls the instance — so the session resolved above is a brand new one + // that has never heard of the capture that was running a moment ago. Absent + // the breadcrumb, the only thing distinguishing "your recording was + // destroyed, here is where the video landed" from "you never started one" + // is gone, and this reports the second. + const reaped = takeReapedSession("screen-recording", api.deviceId); throw new FailureError( - `No active screen recording on device ${api.deviceId}. Call \`screen-recording-start\` first.`, + reaped + ? describeReapedSession(reaped, "screen recording") + : `No active screen recording on device ${api.deviceId}. Call \`screen-recording-start\` first.`, { - error_code: FAILURE_CODES.SCREEN_RECORDING_NO_ACTIVE_SESSION, + error_code: reaped + ? FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN + : FAILURE_CODES.SCREEN_RECORDING_NO_ACTIVE_SESSION, failure_stage: stage, failure_area: "tool_server", // Session-state, not caller input — matches the profiler family's diff --git a/packages/tool-server/src/utils/reaped-sessions.ts b/packages/tool-server/src/utils/reaped-sessions.ts new file mode 100644 index 000000000..63edb8e72 --- /dev/null +++ b/packages/tool-server/src/utils/reaped-sessions.ts @@ -0,0 +1,97 @@ +/** + * Process-global record of capture sessions a teardown reaped while they still + * held data nobody had retrieved. + * + * `stop-all-simulator-servers` disposes every device-owned service, which since + * the `devices` scope landed includes the three that hold captured output — + * `ScreenRecordingSession` (a video), `NativeProfilerSession` (a trace) and + * `JsRuntimeDebugger` (a console-log file). Disposing them is deliberate: each + * owns a spawned process or an open fd that must not outlive the session. + * + * What is not deliberate is what the owner is then told. `Registry._teardown` + * nulls the node's instance, so the next tool call resolves a FRESH service + * whose api is indistinguishable from one that never ran — and the stop tools + * answer "no active session, call start first" for a capture that did run and + * whose output may still be on disk. That reads as "you never started one", + * which is the one thing that is certainly false. + * + * So the disposer leaves a breadcrumb here and the tool that would otherwise + * report absence reports the teardown instead. Module-global for the same + * reason as `screen-recording-reminder`: it has to outlive the service instance + * it describes, which is exactly what teardown destroys. + * + * Entries are CONSUMED by the read ({@link takeReapedSession}) — the breadcrumb + * explains one confusing answer, once. Leaving it would make a genuine later + * "you never started a recording" blame a teardown from an hour ago. + */ + +/** Which session kind was reaped; scopes the key so two kinds can't collide. */ +export type ReapedSessionKind = "screen-recording" | "native-profiler" | "js-runtime-debugger"; + +export interface ReapedSession { + kind: ReapedSessionKind; + deviceId: string; + /** When the teardown ran, for "…N seconds ago" phrasing. */ + atMs: number; + /** + * What survived, as a ready-to-read clause (e.g. naming a salvaged file), or + * undefined when nothing did. Built by the disposer, which is the only place + * that still knows. + */ + salvage?: string; +} + +const reaped = new Map(); + +function key(kind: ReapedSessionKind, deviceId: string): string { + return `${kind}:${deviceId.toLowerCase()}`; +} + +/** + * Note that `kind`'s session for `deviceId` was disposed with data unretrieved. + * + * Call ONLY when there was something to lose: a dispose of an idle session is + * routine cleanup, and recording it would make the next honest "no active + * session" answer claim a teardown destroyed something. + */ +export function recordReapedSession( + kind: ReapedSessionKind, + deviceId: string, + salvage?: string +): void { + const entry: ReapedSession = { kind, deviceId, atMs: Date.now() }; + if (salvage) entry.salvage = salvage; + reaped.set(key(kind, deviceId), entry); +} + +/** Read and consume the breadcrumb for `kind`/`deviceId`, if there is one. */ +export function takeReapedSession( + kind: ReapedSessionKind, + deviceId: string +): ReapedSession | undefined { + const k = key(kind, deviceId); + const entry = reaped.get(k); + if (entry) reaped.delete(k); + return entry; +} + +/** + * The sentence a tool shows in place of "no active session". Names the cause, + * says it is not necessarily this agent's own doing (one tool-server serves + * every agent), and points at whatever survived. + */ +export function describeReapedSession(entry: ReapedSession, what: string): string { + const secondsAgo = Math.max(0, Math.round((Date.now() - entry.atMs) / 1000)); + return ( + `The ${what} for device ${entry.deviceId} was torn down ${secondsAgo}s ago by a ` + + `stop-all-simulator-servers, which reaps every service a device owns — one tool-server ` + + `serves every agent using this argent install, so this may have been another agent ending ` + + `its session. It was not a session that never started.` + + (entry.salvage ? ` ${entry.salvage}` : "") + ); +} + +/** Test-only: drop all breadcrumbs so cases don't leak across tests. */ +export function __resetReapedSessionsForTesting(): void { + reaped.clear(); +} diff --git a/packages/tool-server/test/native-profiler-reaped-session.test.ts b/packages/tool-server/test/native-profiler-reaped-session.test.ts new file mode 100644 index 000000000..010f1d60f --- /dev/null +++ b/packages/tool-server/test/native-profiler-reaped-session.test.ts @@ -0,0 +1,121 @@ +/** + * `stop-all-simulator-servers` reaps every device-owned service, and since the + * `devices` scope landed that set includes `NativeProfilerSession`. Its dispose + * SIGKILLs the capture with no finalize grace — on Android it also removes the + * on-device trace — so the trace really is destroyed. + * + * What must not also happen is the tool-server denying it ever ran. + * `Registry._teardown` nulls the node's instance, so the next + * `native-profiler-stop` resolves a fresh session and used to answer + * "No active native profiling session found. Call native-profiler-start first." + * for a capture that had been running seconds earlier. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { EventEmitter } from "events"; +import type { ChildProcess } from "child_process"; +import type { DeviceInfo } from "@argent/registry"; + +vi.mock("../src/utils/adb", () => ({ adbShell: vi.fn(async () => "") })); +vi.mock("@argent/native-devtools-android", () => ({ + disposeWarmEngine: vi.fn(async () => {}), + TraceProcessorUnavailableError: class extends Error {}, +})); + +import { + nativeProfilerSessionBlueprint, + type NativeProfilerSessionApi, +} from "../src/blueprints/native-profiler-session"; +import { stopNativeProfilerIos } from "../src/tools/profiler/native-profiler/platforms/ios"; +import { stopNativeProfilerAndroid } from "../src/tools/profiler/native-profiler/platforms/android"; +import { __resetReapedSessionsForTesting } from "../src/utils/reaped-sessions"; + +const iosDevice = { id: "6DBF83B4-0000-0000-0000-000000000000", platform: "ios" } as DeviceInfo; +const androidDevice = { id: "emulator-5554", platform: "android" } as DeviceInfo; + +class FakeChild extends EventEmitter { + kill = vi.fn((_signal?: NodeJS.Signals) => { + queueMicrotask(() => this.emit("exit", null, "SIGKILL")); + return true; + }); +} + +async function session(device: DeviceInfo) { + return nativeProfilerSessionBlueprint.factory({}, device, { device } as never); +} + +beforeEach(() => { + __resetReapedSessionsForTesting(); +}); + +describe("a native profiling session reaped by stop-all-simulator-servers", () => { + it("iOS: names the teardown, and says the partial bundle is not worth salvaging", async () => { + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + api.profilingActive = true; + api.captureProcess = new FakeChild() as unknown as ChildProcess; + api.traceFile = "/tmp/argent-fake.trace"; + + await instance.dispose(); + + // The registry nulls the instance, so the stop below resolves a new one. + const fresh = (await session(iosDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerIos(fresh).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).not.toMatch(/Call native-profiler-start first/); + expect(message).toContain("torn down"); + expect(message).toContain("stop-all-simulator-servers"); + expect(message).toContain("/tmp/argent-fake.trace"); + }); + + it("Android: says outright that no trace survived", async () => { + const instance = await session(androidDevice); + const api = instance.api as NativeProfilerSessionApi; + api.profilingActive = true; + api.capturePid = 4242; + api.androidOnDeviceTracePath = "/data/misc/perfetto-traces/fake.pftrace"; + + await instance.dispose(); + + const fresh = (await session(androidDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerAndroid(fresh).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).not.toMatch(/Call native-profiler-start first/); + expect(message).toContain("torn down"); + expect(message).toContain("no trace survived"); + }); + + it("leaves a plain absence alone when the disposed session was idle", async () => { + // Disposing a session nobody was profiling with is routine cleanup. If that + // left a breadcrumb, the next honest "you never started one" would accuse a + // teardown of destroying a capture that never existed. + const instance = await session(iosDevice); + await instance.dispose(); + + const fresh = (await session(iosDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerIos(fresh).catch((e: unknown) => e); + + expect((err as Error).message).toBe( + "No active native profiling session found. Call native-profiler-start first." + ); + }); + + it("is consumed by the report, so it cannot blame a later unrelated absence", async () => { + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + api.profilingActive = true; + api.captureProcess = new FakeChild() as unknown as ChildProcess; + api.traceFile = "/tmp/argent-fake.trace"; + await instance.dispose(); + + const fresh = (await session(iosDevice)).api as NativeProfilerSessionApi; + await stopNativeProfilerIos(fresh).catch(() => {}); + const again = (await session(iosDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerIos(again).catch((e: unknown) => e); + + expect((err as Error).message).toBe( + "No active native profiling session found. Call native-profiler-start first." + ); + }); +}); diff --git a/packages/tool-server/test/screen-recording.test.ts b/packages/tool-server/test/screen-recording.test.ts index 04a3f0336..2e18d4fec 100644 --- a/packages/tool-server/test/screen-recording.test.ts +++ b/packages/tool-server/test/screen-recording.test.ts @@ -46,6 +46,7 @@ import { __resetActiveScreenRecordingsForTesting, getActiveScreenRecordings, } from "../src/utils/screen-recording-reminder"; +import { __resetReapedSessionsForTesting } from "../src/utils/reaped-sessions"; const mockSpawn = vi.mocked(spawn); const mockOpenStream = vi.mocked(openMjpegStream); @@ -192,6 +193,7 @@ const androidDevice: DeviceInfo = { beforeEach(() => { __resetActiveScreenRecordingsForTesting(); + __resetReapedSessionsForTesting(); mockSpawn.mockReset(); mockOpenStream.mockReset(); mockResolveFfmpeg.mockReset(); @@ -277,6 +279,86 @@ describe("screen-recording session blueprint", () => { await expect(fs.access(logo)).rejects.toThrow(); expect(api.logoFile).toBeNull(); }); + + describe("a capture reaped by stop-all-simulator-servers", () => { + // The teardown sequence from the review: start a recording, let + // `stop-all-simulator-servers` reap the device (which is what disposes this + // service), then call `screen-recording-stop`. `Registry._teardown` nulls + // the node's instance, so that stop resolves a BRAND NEW session — modelled + // here by building a second one for the same device. + async function reapDuringCapture(): Promise<{ + output: string; + fresh: ScreenRecordingSessionApi; + }> { + const instance = await screenRecordingSessionBlueprint.factory({}, iosDevice, { + device: iosDevice, + } as never); + fakeStream(); + fakeChild().exitOnStdinEnd(); + await startAndSettle(instance.api); + const output = instance.api.outputFile!; + + await instance.dispose(); + + return { output, fresh: await makeSession(iosDevice) }; + } + + it("tells the owner the recording was torn down, and where the video landed", async () => { + const { output, fresh } = await reapDuringCapture(); + + const err = await stopCapture(fresh).catch((e: unknown) => e); + + const message = (err as Error).message; + // The bug: this used to be "No active screen recording … Call + // `screen-recording-start` first." while a finalized video sat on disk. + expect(message).not.toMatch(/Call `screen-recording-start` first/); + expect(message).toContain("torn down"); + expect(message).toContain("stop-all-simulator-servers"); + // Nothing else in the process still knows this path exists. + expect(message).toContain(output); + expect(getFailureSignal(err)?.error_code).toBe( + FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN + ); + }); + + it("still reports a plain absence when no capture was reaped", async () => { + // The breadcrumb must not turn every "you never started one" into an + // accusation: disposing an idle session leaves nothing behind. + const instance = await screenRecordingSessionBlueprint.factory({}, iosDevice, { + device: iosDevice, + } as never); + await instance.dispose(); + + const err = await stopCapture(await makeSession(iosDevice)).catch((e: unknown) => e); + + expect((err as Error).message).toContain("No active screen recording"); + expect(getFailureSignal(err)?.error_code).toBe( + FAILURE_CODES.SCREEN_RECORDING_NO_ACTIVE_SESSION + ); + }); + + it("is consumed once, so it cannot blame a later unrelated absence", async () => { + const { fresh } = await reapDuringCapture(); + await stopCapture(fresh).catch(() => {}); + + const err = await stopCapture(await makeSession(iosDevice)).catch((e: unknown) => e); + + expect((err as Error).message).toContain("No active screen recording"); + }); + + it("is dropped by a new recording, which would otherwise never consume it", async () => { + const { fresh } = await reapDuringCapture(); + fakeStream(); + fakeChild().exitOnStdinEnd(); + await startAndSettle(fresh); + await fs.writeFile(fresh.outputFile!, Buffer.alloc(16, 1)); + await stopCapture(fresh); + + const err = await stopCapture(await makeSession(iosDevice)).catch((e: unknown) => e); + + expect((err as Error).message).toContain("No active screen recording"); + }); + }); }); describe("screen recording capture", () => { From 987dbe00f3f374b48ea88cc96a5041a59ce8ff2f Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 08:28:20 +0200 Subject: [PATCH 26/98] fix(debugger): explain a console history the teardown deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JsRuntimeDebugger` joined the teardown's namespace set in this branch, so `stop-all-simulator-servers` now disposes it — and its dispose calls `logWriter.close()`, which unlinks the console-log file holding up to 50,000 captured entries. Deleting it is right; nothing can read that file again, since the next resolve builds a new writer over a new path. Being silent about it is not. The victim's `debugger-log-registry` reconnected transparently and reported `totalEntries: 0` with no error and no warning — indistinguishable from an app that logged nothing, and the opposite conclusion to draw when debugging one. Record what was lost at dispose (only when there was history to lose, and under both ids the device answers to, since `forgetDeviceAlias` removes what joins them) and report it as a `note` on the empty registry. The note is attached only to an empty result and consumed by the read, so it can never explain away a healthy one or resurface against a later, unrelated empty read. Also name this teardown in the two react-profiler messages that could only blame a Metro reload or another tool-server: that session rides on the debugger this reaps, so it dies in the same cascade. --- .../src/blueprints/js-runtime-debugger.ts | 25 +++ .../tools/debugger/debugger-log-registry.ts | 30 ++- .../profiler/react/react-profiler-status.ts | 2 +- .../profiler/react/react-profiler-stop.ts | 6 +- .../test/metro/teardown-log-history.test.ts | 181 ++++++++++++++++++ 5 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 packages/tool-server/test/metro/teardown-log-history.test.ts diff --git a/packages/tool-server/src/blueprints/js-runtime-debugger.ts b/packages/tool-server/src/blueprints/js-runtime-debugger.ts index 609bbc899..4d8d72db9 100644 --- a/packages/tool-server/src/blueprints/js-runtime-debugger.ts +++ b/packages/tool-server/src/blueprints/js-runtime-debugger.ts @@ -10,6 +10,7 @@ import { classifyDevice } from "../utils/device-info"; import { proxyStart } from "../utils/sim-remote"; import { selectTarget } from "../utils/debugger/target-selection"; import { rememberDeviceAlias, forgetDeviceAlias } from "../utils/debugger/device-alias"; +import { recordReapedSession } from "../utils/reaped-sessions"; import { CDPClient, type ConsoleAPICalledParams } from "../utils/debugger/cdp-client"; import { createSourceResolver, type SourceResolver } from "../utils/debugger/source-resolver"; import { SourceMapsRegistry } from "../utils/debugger/source-maps"; @@ -284,6 +285,30 @@ export const jsRuntimeDebuggerBlueprint: ServiceBlueprint { + // `logWriter.close()` below unlinks the log file — up to 50,000 + // captured console entries. That is correct as cleanup (nothing can + // read it again: the next resolve builds a new writer over a new path) + // but it is invisible, and since `JsRuntimeDebugger` joined the + // teardown's namespace set this dispose is routinely triggered by + // another agent's `stop-all-simulator-servers`. Leave a breadcrumb so + // `debugger-log-registry`'s otherwise silent `totalEntries: 0` can say + // what happened to the history. + // + // Only when there IS history to lose, and under both ids this device + // answers to: the caller may read back with either the id it connected + // with or the `logicalDeviceId` Metro echoed, and `forgetDeviceAlias` + // below removes the only thing that joins them. + const captured = logWriter.getStats().totalEntries; + if (captured > 0) { + const salvage = + `The ${captured} captured console ${captured === 1 ? "entry" : "entries"} went with ` + + `it — the log file is deleted on teardown, so this registry starts empty rather ` + + `than the app having logged nothing.`; + recordReapedSession("js-runtime-debugger", deviceId, salvage); + if (api.logicalDeviceId && api.logicalDeviceId !== deviceId) { + recordReapedSession("js-runtime-debugger", api.logicalDeviceId, salvage); + } + } forgetDeviceAlias(api.logicalDeviceId); await consoleServer.close(); logWriter.close(); diff --git a/packages/tool-server/src/tools/debugger/debugger-log-registry.ts b/packages/tool-server/src/tools/debugger/debugger-log-registry.ts index 38117c88b..b974fdb53 100644 --- a/packages/tool-server/src/tools/debugger/debugger-log-registry.ts +++ b/packages/tool-server/src/tools/debugger/debugger-log-registry.ts @@ -3,12 +3,22 @@ import type { ToolDefinition } from "@argent/registry"; import type { JsRuntimeDebuggerApi } from "../../blueprints/js-runtime-debugger"; import type { LogStats, MessageCluster } from "../../utils/debugger/log-file-writer"; import { DEBUGGER_TOOL_CAPABILITY, debuggerServiceRef } from "./debugger-service-ref"; +import { canonicalDeviceId } from "../../utils/debugger/device-alias"; +import { describeReapedSession, takeReapedSession } from "../../utils/reaped-sessions"; interface LogRegistryResponse extends LogStats { clusters: MessageCluster[]; deviceName: string; appName: string; logicalDeviceId: string | undefined; + /** + * Why this registry is empty when it should not be — present only when the + * previous debugger session for this device was torn down by a + * `stop-all-simulator-servers` with console history captured. Without it an + * empty registry reads as "the app logged nothing", which is the wrong + * conclusion to hand an agent debugging a silent app. + */ + note?: string; } const zodSchema = z.object({ @@ -32,23 +42,37 @@ export const debuggerLogRegistryTool: ToolDefinition< }, description: `Get a summary of all console logs captured from the app's JS runtime. Returns the log file path, entry counts by level, and message clusters (grouped by similarity). Works against Hermes (iOS / Android / Vega) and V8 (Chromium). -Use when investigating warnings, errors, or unexpected output — call this first for an overview, then read the returned file for details. Returns empty stats if no log data has been captured yet.`, +Use when investigating warnings, errors, or unexpected output — call this first for an overview, then read the returned file for details. Returns empty stats if no log data has been captured yet — but check { note }, which is present only when the stats are empty BECAUSE a stop-all-simulator-servers tore the previous debugger session down and deleted its log file. Absent that note, empty really does mean the app has logged nothing.`, zodSchema, capability: DEBUGGER_TOOL_CAPABILITY, services: (params) => ({ debugger: debuggerServiceRef(params), }), - async execute(services) { + async execute(services, params) { const api = services.debugger as JsRuntimeDebuggerApi; const stats = api.logWriter.getStats(); const clusters = api.logWriter.getClusters(20); - return { + const response: LogRegistryResponse = { ...stats, clusters, deviceName: api.deviceName, appName: api.appName, logicalDeviceId: api.logicalDeviceId, }; + + // Resolving the service above silently RECONNECTED if a teardown had reaped + // the previous session, so an empty registry here is ambiguous: either the + // app has logged nothing, or a `stop-all-simulator-servers` deleted the log + // file. Only the empty case is ambiguous — a registry with entries in it is + // reporting this session's own capture, and consuming a breadcrumb there + // would attach a stale explanation to a healthy result. + if (stats.totalEntries === 0) { + const reaped = + takeReapedSession("js-runtime-debugger", canonicalDeviceId(params.device_id)!) ?? + takeReapedSession("js-runtime-debugger", params.device_id); + if (reaped) response.note = describeReapedSession(reaped, "JS-runtime debugger session"); + } + return response; }, }; diff --git a/packages/tool-server/src/tools/profiler/react/react-profiler-status.ts b/packages/tool-server/src/tools/profiler/react/react-profiler-status.ts index 0e20034d6..67da927eb 100644 --- a/packages/tool-server/src/tools/profiler/react/react-profiler-status.ts +++ b/packages/tool-server/src/tools/profiler/react/react-profiler-status.ts @@ -174,7 +174,7 @@ export function createReactProfilerStatusTool( session_status: isMine ? "active" : "taken_over", note: isMine ? "Your profiling session is still running. Call react-profiler-stop to collect the data, or continue profiling." - : "A different profiling session is running (another tool-server instance took over, or this process restarted after start). Data from the prior session is lost at the takeover moment. Use react-profiler-start { force: true } to reclaim.", + : "A different profiling session is running (another tool-server instance took over, this process restarted after start, or a stop-all-simulator-servers reaped this device's JS-runtime debugger and took this session down with it, leaving the in-app owner behind). Data from the prior session is lost at the takeover moment. Use react-profiler-start { force: true } to reclaim.", }; }, }; diff --git a/packages/tool-server/src/tools/profiler/react/react-profiler-stop.ts b/packages/tool-server/src/tools/profiler/react/react-profiler-stop.ts index 137f54b09..c1a059599 100644 --- a/packages/tool-server/src/tools/profiler/react/react-profiler-stop.ts +++ b/packages/tool-server/src/tools/profiler/react/react-profiler-stop.ts @@ -180,8 +180,10 @@ Fails if no active profiling session exists or the CDP connection was lost durin if (!entry || entry.state !== ServiceState.RUNNING) { throw new FailureError( - "No active profiling session. The session may have been lost due to a Metro reload. " + - "Call react-profiler-start to begin a new session.", + "No active profiling session. The session may have been lost to a Metro reload, or " + + "torn down by a stop-all-simulator-servers — this session rides on the device's " + + "JS-runtime debugger, which that teardown reaps, and one tool-server serves every " + + "agent using this argent install. Call react-profiler-start to begin a new session.", { error_code: FAILURE_CODES.REACT_PROFILER_NO_ACTIVE_SESSION, failure_stage: "react_profiler_stop_session_lookup", diff --git a/packages/tool-server/test/metro/teardown-log-history.test.ts b/packages/tool-server/test/metro/teardown-log-history.test.ts new file mode 100644 index 000000000..f9ed4ab3b --- /dev/null +++ b/packages/tool-server/test/metro/teardown-log-history.test.ts @@ -0,0 +1,181 @@ +/** + * `stop-all-simulator-servers` reaps every device-owned service, and since the + * `devices` scope landed that set includes `JsRuntimeDebugger`. Its dispose + * calls `logWriter.close()`, which unlinks the console-log file — up to 50,000 + * captured entries. + * + * The deletion itself is fine: the next resolve builds a new writer over a new + * path, so nothing could ever read the old file again. What was not fine is + * that the victim's `debugger-log-registry` transparently reconnected and + * reported `totalEntries: 0` with no error and no warning — indistinguishable + * from an app that has logged nothing, which is the opposite conclusion. + * + * Drives the real Registry → JsRuntimeDebugger → debugger-log-registry path + * against a mock Metro, disposing the service exactly as the teardown does. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import { WebSocketServer, WebSocket } from "ws"; +import * as http from "node:http"; +import { Registry } from "@argent/registry"; +import { + jsRuntimeDebuggerBlueprint, + type JsRuntimeDebuggerApi, +} from "../../src/blueprints/js-runtime-debugger"; +import { debuggerConnectTool } from "../../src/tools/debugger/debugger-connect"; +import { debuggerLogRegistryTool } from "../../src/tools/debugger/debugger-log-registry"; +import { __resetReapedSessionsForTesting } from "../../src/utils/reaped-sessions"; + +let mockServer: http.Server; +let wss: WebSocketServer; +let mockPort: number; +let registry: Registry; + +const LOGICAL_ID = "logical-only-device"; + +function handleCDPMessage(ws: WebSocket, raw: string) { + const { id } = JSON.parse(raw) as { id: number; method: string }; + ws.send(JSON.stringify({ id, result: {} })); +} + +beforeAll(async () => { + await new Promise((resolve) => { + mockServer = http.createServer((req, res) => { + if (req.url === "/status") { + res.setHeader("X-React-Native-Project-Root", "/mock/project"); + res.end("packager-status:running"); + return; + } + if (req.url === "/json/list") { + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify([ + { + id: "page-0", + title: "app (Test Device)", + description: "[C++ connection]", + webSocketDebuggerUrl: `ws://localhost:${mockPort}/inspector/debug?device=${LOGICAL_ID}&page=1`, + deviceName: "Test Device", + reactNative: { + logicalDeviceId: LOGICAL_ID, + capabilities: { prefersFuseboxFrontend: true }, + }, + }, + ]) + ); + return; + } + res.statusCode = 404; + res.end("Not found"); + }); + + wss = new WebSocketServer({ server: mockServer }); + wss.on("connection", (ws) => ws.on("message", (raw) => handleCDPMessage(ws, raw.toString()))); + + mockServer.listen(0, () => { + mockPort = (mockServer.address() as { port: number }).port; + resolve(); + }); + }); + + registry = new Registry(); + registry.registerBlueprint(jsRuntimeDebuggerBlueprint); + registry.registerTool(debuggerConnectTool); + registry.registerTool(debuggerLogRegistryTool); +}); + +afterAll(async () => { + await registry.dispose(); + await new Promise((resolve) => wss.close(() => mockServer.close(() => resolve()))); +}); + +beforeEach(async () => { + // The registry caches the service, so a session a previous case left + // connected would be reused — carrying its entry count into the next case. + await registry.disposeService(`JsRuntimeDebugger:${mockPort}:${LOGICAL_ID}`).catch(() => {}); + __resetReapedSessionsForTesting(); +}); + +async function connectAndCapture(deviceId: string, entries: number): Promise { + await registry.invokeTool("debugger-connect", { port: mockPort, device_id: deviceId }); + const urn = `JsRuntimeDebugger:${mockPort}:${deviceId}`; + const api = await registry.resolveService(urn); + for (let i = 0; i < entries; i++) { + api.logWriter.write({ + id: i, + timestamp: new Date(1710000000000 + i * 1000).toISOString(), + level: "log", + message: `captured ${i}`, + }); + } + expect(api.logWriter.getStats().totalEntries).toBe(entries); + return urn; +} + +describe("a debugger session reaped by stop-all-simulator-servers", () => { + it("says the console history was deleted rather than reporting a silent app", async () => { + const urn = await connectAndCapture(LOGICAL_ID, 60); + + // Exactly what the teardown does to this device's debugger. + await registry.disposeService(urn); + + // The registry reconnects transparently — a brand new writer over a new + // file. The count really is 0; the question is whether anything says why. + const result = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { totalEntries: number; note?: string }; + + expect(result.totalEntries).toBe(0); + expect(result.note).toBeDefined(); + expect(result.note).toContain("60 captured console entries"); + expect(result.note).toContain("stop-all-simulator-servers"); + expect(result.note).toContain("torn down"); + }); + + it("stays silent when the previous session had captured nothing", async () => { + // A teardown that destroyed no history has nothing to explain, and saying + // otherwise would make every empty registry look like a lost one. + await registry.invokeTool("debugger-connect", { port: mockPort, device_id: LOGICAL_ID }); + await registry.disposeService(`JsRuntimeDebugger:${mockPort}:${LOGICAL_ID}`); + + const result = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { totalEntries: number; note?: string }; + + expect(result.totalEntries).toBe(0); + expect(result.note).toBeUndefined(); + }); + + it("does not attach the explanation to a registry that has its own entries", async () => { + const urn = await connectAndCapture(LOGICAL_ID, 5); + await registry.disposeService(urn); + // Reconnect and capture fresh history before reading. + await connectAndCapture(LOGICAL_ID, 3); + + const result = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { totalEntries: number; note?: string }; + + expect(result.totalEntries).toBe(3); + expect(result.note).toBeUndefined(); + }); + + it("reports the loss once, not on every later empty read", async () => { + const urn = await connectAndCapture(LOGICAL_ID, 12); + await registry.disposeService(urn); + + const first = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { note?: string }; + const second = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { note?: string }; + + expect(first.note).toBeDefined(); + expect(second.note).toBeUndefined(); + }); +}); From c95ffe227d28a7ca21196885f8838241f5881f1f Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 08:34:38 +0200 Subject: [PATCH 27/98] fix(flow): keep the scratch path out of a write failure, and blame the right thing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems with what an agent reads when an append cannot write the flow file. The message correctly named only the flow file, but the raw errno went along as `{ cause }` — and `formatErrorForAgent` walks the cause chain and appends it, so the string an agent actually reads ended in: — caused by: EACCES: permission denied, open '…/.argent-flow-28020-31.tmp' naming a scratch file that was deleted moments earlier and never existed as far as the caller is concerned. Scrub the temp path out of the cause and keep the rest of the errno, which is the part worth having. Second, the explanation "…so must be writable" was appended for every errno. A 252-character flow name fails in `rename` with ENAMETOOLONG, and that reported a directory-permissions problem the user would go and not find. Pick the explanation from the code: permissions for EACCES/EPERM/EROFS, space for ENOSPC/EDQUOT, name length for ENAMETOOLONG, and a plain statement of the swap otherwise. The temp-path guard also asserted against `err.message` alone, so it passed while the rendered string violated the invariant. Assert on `formatErrorForAgent(err)` — the string the invariant is actually about — and cover the read-only-directory repro and the mis-attributed errno. --- .../tool-server/src/tools/flows/flow-utils.ts | 65 +++++++++++++++---- .../flows/flow-concurrent-recording.test.ts | 60 ++++++++++++++++- 2 files changed, 112 insertions(+), 13 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index cab56ef62..43741395a 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2423,6 +2423,49 @@ let flowWriteSeq = 0; * file, and it replaces the inode, so a chmod on the flow file or a hardlink to * it does not survive an append. */ +/** + * What actually went wrong, per errno. The swap needs write permission on the + * DIRECTORY, which is the surprising part and worth stating — but only when + * that is the failure. Stating it for every code turned an over-long flow name + * (`ENAMETOOLONG` out of `rename`) into a report of a directory-permissions + * problem the user would then go and not find. + */ +function writeFailureHint(code: string | undefined, filePath: string): string { + const dir = path.dirname(filePath); + switch (code) { + case "EACCES": + case "EPERM": + case "EROFS": + return ( + `an append replaces the file via a sibling temp file and rename, so ${dir} must be ` + + `writable — permission on the flow file itself is not enough.` + ); + case "ENOSPC": + case "EDQUOT": + return `the filesystem holding ${dir} is out of space (or over quota).`; + case "ENAMETOOLONG": + return `the flow name makes ${path.basename(filePath)} longer than this filesystem allows — use a shorter name.`; + case "ENOENT": + return `${dir} does not exist.`; + default: + return `an append replaces the file via a sibling temp file and rename in ${dir}.`; + } +} + +/** + * The original error with the internal scratch path rewritten to the flow file, + * so the cause chain `formatErrorForAgent` renders never names a temp file that + * was already deleted. Everything else about the errno — code, syscall, the + * kernel's own wording — is kept. + */ +function scrubTempPath(err: unknown, tmpPath: string, filePath: string): Error { + if (!(err instanceof Error)) return new Error(String(err)); + if (!err.message.includes(tmpPath)) return err; + const scrubbed = new Error(err.message.split(tmpPath).join(filePath)); + scrubbed.name = err.name; + return scrubbed; +} + async function writeFlowFile(filePath: string, content: string): Promise { const tmpPath = path.join( path.dirname(filePath), @@ -2441,24 +2484,24 @@ async function writeFlowFile(filePath: string, content: string): Promise { // Rethrow against the flow file, never the scratch path. The temp name is // an internal detail — pid+counter suffixed, and already removed above — so // surfacing its raw errno (`EACCES: … open '.argent-flow--.tmp'`) - // would name a file that no longer exists and never mention the flow. The - // swap writes a sibling and renames, so the actual cause is write - // permission (or space) on the DIRECTORY: name the flow file and the - // directory, and keep the original errno as `cause`. - const code = - err instanceof Error && typeof (err as NodeJS.ErrnoException).code === "string" - ? (err as NodeJS.ErrnoException).code - : undefined; + // would name a file that no longer exists and never mention the flow. + // + // That applies to the CAUSE as much as to this message: + // `formatErrorForAgent` walks the cause chain and appends each new message, + // so attaching the raw errno puts the scratch path in front of the agent + // anyway — through the one string it actually reads. Scrub the path out of + // the cause and keep the rest, which is the part worth having. + const errno = err instanceof Error ? (err as NodeJS.ErrnoException) : undefined; + const code = typeof errno?.code === "string" ? errno.code : undefined; throw new FailureError( - `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — an append replaces ` + - `the file via a sibling temp file and rename, so ${path.dirname(filePath)} must be writable.`, + `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — ${writeFailureHint(code, filePath)}`, { error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, failure_stage: "flow_file_write", failure_area: "tool_server", error_kind: "unknown", }, - { cause: err instanceof Error ? err : new Error(String(err)) } + { cause: scrubTempPath(err, tmpPath, filePath) } ); } } diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index ef11aee76..66f6e89a3 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -11,6 +11,7 @@ import { flowFinishRecordingTool } from "../../src/tools/flows/flow-finish-recor import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step"; import { createRunFlowTool } from "../../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite"; +import { formatErrorForAgent } from "../../src/utils/format-error"; import { __resetRecordingsForTesting, getRecordingSession, @@ -690,10 +691,16 @@ describe("flow-file writes as seen by a concurrent reader", () => { spy.mockRestore(); expect(err).toBeInstanceOf(Error); - const message = (err as Error).message; - // Names the flow file and the directory (the real cause), not the scratch path. + // Against the string an agent actually READS, not `err.message`: + // `formatErrorForAgent` appends the cause chain, so asserting on the + // message alone passed while the rendered text still named the scratch + // file. The invariant is about what is disclosed, so assert on what is. + const message = formatErrorForAgent(err); expect(message).toContain(target); expect(message).not.toMatch(/\.argent-flow-\d+-\d+\.tmp/); + // The errno itself is worth keeping — only the phantom path is not. + expect(message).toContain("ENOSPC"); + expect(message).toContain("out of space"); expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); // The half-written scratch file must not survive in the committed flows dir. @@ -702,6 +709,55 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(listActiveRecordings()).toEqual([]); }); + it("names only the flow file when a read-only flows dir fails an append", async () => { + // The review's own repro: chmod 500 the flows dir, then append to a live + // recording. This fails at the temp OPEN — earlier than either case above — + // and the errno names a scratch file that has never existed on disk. + const root = await makeRoot("readonly-dir"); + const target = flowPath(root, "alpha"); + await start(root, "alpha"); + const flowsDir = path.dirname(target); + await fs.chmod(flowsDir, 0o500); + try { + const err = await addEcho(root, "alpha", "note").catch((e: unknown) => e); + + const message = formatErrorForAgent(err); + expect(message).toContain(target); + expect(message).not.toMatch(/\.argent-flow-\d+-\d+\.tmp/); + // Here the directory-permission explanation IS the right one. + expect(message).toContain("must be writable"); + } finally { + await fs.chmod(flowsDir, 0o700); + } + }); + + it("explains the errno it actually got, not directory permissions every time", async () => { + // "so must be writable" used to be appended to every failure. An + // over-long flow name fails in `rename` with ENAMETOOLONG — a writable + // directory does not help, and sending someone to check permissions on one + // they will find perfectly writable is a wrong lead, not a vague one. + const root = await makeRoot("nametoolong"); + const target = flowPath(root, "alpha"); + await fs.mkdir(path.dirname(target), { recursive: true }); + + const realRename = fs.rename; + const spy = vi.spyOn(fs, "rename").mockImplementationOnce(async () => { + const err: NodeJS.ErrnoException = new Error( + `ENAMETOOLONG: name too long, rename '${target}'` + ); + err.code = "ENAMETOOLONG"; + throw err; + }); + const err = await start(root, "alpha").catch((e: unknown) => e); + spy.mockRestore(); + void realRename; + + const message = formatErrorForAgent(err); + expect(message).toContain("ENAMETOOLONG"); + expect(message).toContain("use a shorter name"); + expect(message).not.toContain("must be writable"); + }); + it("never exposes an empty or unparseable file while appends are in flight", async () => { // The property the two inode assertions above encode, observed the way a // reader actually experiences it: poll the path as fast as the event loop From 6dccd7cd3cedf21b9f3af948c51f77a09f810197 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 08:40:37 +0200 Subject: [PATCH 28/98] test: pin the five behaviours the suite was not holding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of these was verified by mutating the code it covers and watching the suite stay green, then re-running against the mutation once the assertion was in place. - The scoped teardown's `devices` parameter was never driven through its own schema: every case hands `execute` a hand-built params object, and the sole schema assertion was the negative `udids` rejection. Changing `.optional()` to `.default([])` therefore passed, making `scoped` permanently true so the machine-wide sweep reaped nothing while answering `{ stopped: [] }`. Parse, then execute what the parse produced, on both the scoped and omitted shapes. - The flow-file containment guard exempts an upload via `fileInput?.viaUpload`, but every containment case passed `fileInput: undefined` and the one case carrying a file input used `viaUpload: true`. Relaxing the test to `if (fileInput)` opened the bypass to the COMMON shape — `viaUpload: false`, which `resolveOne` returns for every same-machine `flow_file` — undetected. - The `devices` telemetry branch had no test. Adding one showed why: both consumers are gated on the tool declaring a capability, and the one tool spelling the parameter that way declares none, so the branch is latent rather than telemetry-only. Test it through a capability-bearing tool and correct the two comments that claimed it feeds telemetry today. - The rewritten screen-recording teardown message was asserted nowhere; only its error code was, and that enum still says "shutting down". Reverting the whole rewrite to the old one-liner left the suite green. - The CLI help assertion compared the rendered help against the fixture it was rendered from — `x.toContain(x)`, which holds however the renderer places the description. Pin the placement instead, which is what the renderer decides. --- packages/argent-cli/test/run-help.test.ts | 11 +++-- packages/tool-server/src/http.ts | 14 +++++-- .../test/flows/flow-remote-recording.test.ts | 26 ++++++++++++ .../tool-server/test/http-tools-meta.test.ts | 40 +++++++++++++++++++ .../tool-server/test/screen-recording.test.ts | 13 ++++++ packages/tool-server/test/stop-tools.test.ts | 33 +++++++++++++++ 6 files changed, 131 insertions(+), 6 deletions(-) diff --git a/packages/argent-cli/test/run-help.test.ts b/packages/argent-cli/test/run-help.test.ts index 4cb9a9c0f..ab26911d8 100644 --- a/packages/argent-cli/test/run-help.test.ts +++ b/packages/argent-cli/test/run-help.test.ts @@ -121,9 +121,14 @@ describe("argent run --help — whole-payload --args advertisement", () => { await run(["flow-add-step", "--help"], { paths: {} as never }); const help = capturedHelp(); - // The tool's own prose is printed above the flag block, so the help names - // the recording a step is being added to rather than an implicit active one. - expect(help).toContain(flowAddStepMeta.description); + // The tool's own prose is printed ABOVE the flag block. Asserting mere + // containment says almost nothing here — `help` is rendered from this same + // fixture, so it reduces to `x.toContain(x)` and holds for any renderer + // that emits the description anywhere at all, including below the flags. + // Pin the placement, which is the part the renderer decides. + const descriptionAt = help.indexOf(flowAddStepMeta.description); + expect(descriptionAt).toBeGreaterThanOrEqual(0); + expect(descriptionAt).toBeLessThan(help.indexOf("--name ")); // The recording identity is required alongside `command`: omitting either // flag fails the server's zod validation, so the help has to say so up front // instead of presenting them as optional extras. diff --git a/packages/tool-server/src/http.ts b/packages/tool-server/src/http.ts index afeaa07bf..35b3f8937 100644 --- a/packages/tool-server/src/http.ts +++ b/packages/tool-server/src/http.ts @@ -146,8 +146,14 @@ function extractDeviceArg(data: unknown): string | null { // `devices: string[]` is a third spelling, used only by // `stop-all-simulator-servers`' scoped teardown. A call can name several // devices of different platforms; the first is enough for the coarse - // telemetry platform. It never reaches the capability gate — that tool - // declares no capability — so this is a telemetry-only refinement. + // telemetry platform. + // + // Latent today: BOTH consumers of this function are gated on the tool + // declaring a capability — the gate directly, and `extractInvocationMeta` + // through its `hasCapability` argument — and the one tool that spells the + // parameter this way declares none. Kept so the reading of `devices` is + // defined in one place if a capability-bearing tool ever takes a device list, + // rather than being rediscovered then. if (Array.isArray(record.devices) && typeof record.devices[0] === "string") { return record.devices[0]; } @@ -753,7 +759,9 @@ export function createHttpApp(registry: Registry, options?: HttpAppOptions): Htt // iOS-only device_id-tool is rejected at the gate instead of falling // through to the deeper blueprint error (which surfaces as a generic 500). // Only the first two ever reach this gate — the `devices` tool declares no - // capability — but the third is read the same way for telemetry platform. + // capability, and neither does telemetry read it for the same reason (see + // `extractDeviceArg`). The third is defined here so it behaves like the + // others the day a capability-bearing tool takes a device list. const deviceArg = extractDeviceArg(parsedData); if (def.capability && deviceArg) { try { diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index 6af42bb0c..754d5c333 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -757,6 +757,32 @@ describe("flow_file containment", () => { ).toBe(uploaded); }); + it("exempts an upload, and ONLY an upload, from containment", async () => { + // The exemption keys on `viaUpload`, not on the mere presence of a file + // input — but nothing pinned that discrimination: every containment case + // above passes `fileInput: undefined`, and the one case that supplies one + // uses `viaUpload: true`. So relaxing the guard to + // + // if (fileInput?.viaUpload) -> if (fileInput) + // + // left the whole suite green while opening the containment bypass to the + // COMMON shape: `resolveOne` returns `viaUpload: false` for any wire path + // that already exists on the host, i.e. every same-machine `flow-execute` + // carrying a `flow_file`. + const hostPath = { clientPath: CLIENT_FLOW_PATH, presentOnHost: true, viaUpload: false }; + + await expect(resolveFlowSource(params("/etc/anything.yaml"), hostPath)).rejects.toThrow( + "Invalid flow_file" + ); + // The same input with the upload flag set is the trusted case, and must + // still pass — otherwise this test would also hold for a guard that simply + // ignored `fileInput` altogether. + expect( + (await resolveFlowSource(params("/etc/anything.yaml"), { ...hostPath, viaUpload: true })) + .filePath + ).toBe("/etc/anything.yaml"); + }); + it("rejects a relative flow_file", async () => { await expect(resolveFlowSource(params(".argent/flows/remote-flow.yaml"))).rejects.toThrow( "Invalid flow_file" diff --git a/packages/tool-server/test/http-tools-meta.test.ts b/packages/tool-server/test/http-tools-meta.test.ts index 483beac2e..c61be5f04 100644 --- a/packages/tool-server/test/http-tools-meta.test.ts +++ b/packages/tool-server/test/http-tools-meta.test.ts @@ -190,6 +190,46 @@ describe("GET /tools progressive-loading metadata", () => { expect(recordInvocation).toHaveBeenCalledWith(expect.any(String), { platform: "android" }); }); + it("records the platform of a scoped teardown, whose device arg is a LIST", async () => { + // `devices` is the third device-arg spelling and the only one that is an + // array — `stop-all-simulator-servers`' scope. The other two spellings are + // pinned above; deleting the `devices` branch of `extractDeviceArg` left + // the whole suite green, so a scoped teardown silently lost its platform. + // Driven through `device-tool` because `extractInvocationMeta` derives a + // platform only for a tool that declares a capability. `stop-all-simulator- + // servers`, the sole tool spelling `devices` today, declares none — so this + // branch is latent in production and only a capability-bearing tool can + // exercise it. + let seenMeta: Record | undefined; + const recordInvocation = vi.fn((_id: string, meta: Record) => { + seenMeta = meta; + return vi.fn(); + }); + handle.dispose(); + handle = createHttpApp(stubRegistry(), { recordInvocation }); + + await request(handle.app) + .post("/tools/device-tool") + .send({ devices: ["emulator-5554", "11111111-1111-1111-1111-111111111111"] }) + .expect(200); + + // The first id is enough for the coarse platform; a mixed-platform scope + // is not something this dimension tries to represent. + expect(seenMeta).toEqual({ platform: "android" }); + }); + + it("ignores a devices list that holds no usable id", async () => { + const recordInvocation = vi.fn(() => vi.fn()); + handle.dispose(); + handle = createHttpApp(stubRegistry(), { recordInvocation }); + + await request(handle.app).post("/tools/device-tool").send({ devices: [] }).expect(200); + + // An empty scope yields no device arg, so no platform — and with nothing + // else to record, no invocation metadata at all. + expect(recordInvocation).not.toHaveBeenCalled(); + }); + it("refines an iOS device to `tvos` when its cached runtime kind is tv", async () => { tvKinds.ios = "tv"; let seenMeta: Record | undefined; diff --git a/packages/tool-server/test/screen-recording.test.ts b/packages/tool-server/test/screen-recording.test.ts index 2e18d4fec..a7932f2b6 100644 --- a/packages/tool-server/test/screen-recording.test.ts +++ b/packages/tool-server/test/screen-recording.test.ts @@ -616,6 +616,19 @@ describe("screen recording capture", () => { expect(getFailureSignal(err)?.error_code).toBe( FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN ); + // The error CODE is not the behaviour here — its enum name still says + // "shutting down", which is exactly the claim the message stopped making. + // A dispose is now far more often a `stop-all-simulator-servers` reaping + // this device than a process shutdown, and the two are indistinguishable + // from `api.disposed`. So the message must name both, and must not tell + // the caller a retry is pointless: on the teardown branch the device is + // usually still up. Asserted here because reverting the whole rewrite to + // the old one-liner otherwise leaves the suite green. + const message = (err as Error).message; + expect(message).toContain("stop-all-simulator-servers"); + expect(message).toContain("nothing was recorded"); + expect(message).toContain("start the recording again"); + expect(message).toContain(IOS_UDID); } expect(mockSpawn).not.toHaveBeenCalled(); expect(stream.close).toHaveBeenCalled(); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index bd15718bd..1d2fe8a97 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -606,6 +606,39 @@ describe("stop-all-simulator-servers device scoping", () => { }); }); + it("drives the scope through its own schema, not just past it", async () => { + // Every other case here hands `execute` a hand-built params object, so zod + // is never in the loop and the ONLY schema assertion is a negative (the + // `udids` rejection above). That leaves the parse itself unpinned: changing + // + // devices: z.array(z.string()).optional() -> .default([]) + // + // typechecks, keeps all 3255 tests green, and makes `params.devices` always + // `[]` — so `scoped` is permanently true and the machine-wide sweep reaps + // nothing while answering `{ stopped: [] }`, which the tool documents as + // "only means nothing was still running". Parse, then execute what the + // parse produced, on both shapes. + const registry = createMockRegistry(twoAgentServices()); + const tool = createStopAllSimulatorServersTool(registry); + const schema = tool.zodSchema!; + + // A scoped call is accepted and reaches execute as the ids it was given. + expect(schema.safeParse({ devices: [MINE] }).success).toBe(true); + const scoped = await tool.execute!({}, schema.parse({ devices: [MINE] })); + expect(scoped).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE}`], + }); + + // And an omitted scope still parses to "absent" — the machine-wide sweep — + // rather than to an empty list that would scope to nothing. + const swept = createMockRegistry(twoAgentServices()); + const sweepTool = createStopAllSimulatorServersTool(swept); + expect(schema.parse({}).devices).toBeUndefined(); + const unscoped = await sweepTool.execute!({}, schema.parse({})); + expect(unscoped.stopped).toHaveLength(5); + expect(unscoped).not.toHaveProperty("unmatched"); + }); + it("does not match a device id that is a prefix of another device's id", async () => { const services = new Map([ ["SimulatorServer:AAAA", { state: ServiceState.RUNNING, dependents: [] }], From fec70d0f34489e31ecc0994c6083745cac828cb9 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 11:03:21 +0200 Subject: [PATCH 29/98] fix(flow): reconcile the recorder with main's run: gate and symlinked flows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration fixes this branch needs against the main it now sits on. Every one is a place where main gained behaviour after this branch forked, and the two had to be brought together rather than either side dropped. The atomic swap replaced symlinked flow files. flow-start-recording and every append now go through a temp file + rename, and rename(2) replaces the path it is handed — so a saved flow that is a symlink into a shared vault lost its link on the first write, and the vault copy was stranded with the pre-recording content. The plain writeFile it replaced followed the link. Resolve the target first, and resolve its directory separately so the very first write (which has no realpath of its own yet) lands on the same canonical spelling as every later append. Pinned by the two symlink cases in flow-tools.test.ts, which fail against the unresolved swap. The cross-project run: capture defers to main. Both this branch and #567/#568 fixed the same defect — a nested flow-execute that ran another project's same-named fragment would record `run: `, which replays THIS project's copy. Main refuses the substitution and keeps the raw call (which replays via name + project_root, i.e. the file that actually ran); this branch recorded the directive and warned. Main's is the stricter rule and already ships, so the branch's warn-and-compose path and its safeFlowsDir helper are dropped, and the two tests that pinned them now pin the refusal. The resolved branch no longer forwards a warning either — under main's gate `flow` and `warning` are exclusive, so that passthrough was dead. Tests main added since the fork still drove the recorder through the module globals this branch removes: http-flow-path-boundary.test.ts reset them in its hooks, and the run:-composition cases called flow-add-step without the name + project_root the tools now require. Threaded through, hooks dropped. --- .../tool-server/src/tools/flows/flow-utils.ts | 19 +++- .../flows/flow-concurrent-recording.test.ts | 40 ++++++--- .../test/flows/flow-remote-recording.test.ts | 2 + .../tool-server/test/flows/flow-tools.test.ts | 90 +++++++++++++++++-- .../test/http-flow-path-boundary.test.ts | 9 +- 5 files changed, 130 insertions(+), 30 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 43741395a..688e77682 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2467,15 +2467,30 @@ function scrubTempPath(err: unknown, tmpPath: string, filePath: string): Error { } async function writeFlowFile(filePath: string, content: string): Promise { + // Swap onto the flow file's REAL path. A saved flow may be a symlink into a + // shared vault — the runner canonicalizes before reading, and `run:` + // composition anchors on the real file — and rename(2) replaces the path it + // is handed, so renaming onto the link's own spelling would swap the symlink + // for a regular file and strand the vault copy with the pre-recording + // content. A plain write follows the link; resolving first keeps that + // behavior while keeping the swap atomic. + // + // The directory is resolved separately so that a flow file which does not + // exist yet (the first write of a recording, which has no realpath of its + // own) still lands on the same canonical spelling as every later append — + // otherwise the first swap and the rest would disagree wherever an ancestor + // is itself a symlink, which is the default for the temp dir on macOS. + const dir = await fs.realpath(path.dirname(filePath)).catch(() => path.dirname(filePath)); + const target = await fs.realpath(filePath).catch(() => path.join(dir, path.basename(filePath))); const tmpPath = path.join( - path.dirname(filePath), + path.dirname(target), `.argent-flow-${process.pid}-${++flowWriteSeq}.tmp` ); try { await fs.writeFile(tmpPath, content, "utf8"); // Atomic within a filesystem, and the temp file is a sibling of the target, // so it is always the same one. - await fs.rename(tmpPath, filePath); + await fs.rename(tmpPath, target); } catch (err) { // Leave no scratch file behind, whichever half failed. The write itself can // fail with the file already created (ENOSPC, EIO), so this has to cover it diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 66f6e89a3..3a8a8dea2 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -606,13 +606,18 @@ describe("flow-file writes as seen by a concurrent reader", () => { // `os.tmpdir()` itself is caught by the directory comparison below on any // host, without ever needing two real filesystems to reproduce EXDEV. const root = await makeRoot("scratch-sibling"); - const flowsDir = path.dirname(flowPath(root, "alpha")); vi.mocked(fs.rename).mockClear(); await start(root, "alpha"); // writeNewFlowFile → writeFlowFile → 1 rename await addStep(root, "alpha", "a1"); // appendStep → writeFlowFile → 1 rename await addEcho(root, "alpha", "a2"); // appendStep → writeFlowFile → 1 rename + // Canonical, because the writer swaps onto the flow file's REAL path so a + // symlinked flow keeps its link (see writeFlowFile). The directory that + // must hold the scratch file is therefore the resolved one — still a strict + // subdirectory of the temp root, so the os.tmpdir() relocation this case + // exists to catch is caught exactly as before. + const flowsDir = await fs.realpath(path.dirname(flowPath(root, "alpha"))); const renameCalls = vi.mocked(fs.rename).mock.calls; expect(renameCalls).toHaveLength(3); for (const [from, to] of renameCalls) { @@ -1549,12 +1554,16 @@ describe("recording a flow-execute step while several projects are in play", () ]); }); - it("records run: when the target sits next to the recording", async () => { + it("keeps the raw step when the executed project has no file to compare against", async () => { const recordingRoot = await makeRoot("run-target-sibling"); const executedRoot = await makeRoot("run-target-elsewhere"); // Mirror image: the fragment is a sibling of the flow being recorded and is - // absent from the executed project. + // absent from the executed project. Being a sibling is necessary for `run:` + // but not sufficient — the recorded directive must replay the file that + // just RAN, and nothing verifiable ran from the executed project's flows + // dir, so the two cannot be shown to be one file. The raw step, which + // replays via name + project_root, is then the only honest record. await writeSavedFlow(recordingRoot, "helper", fragment); await fs.mkdir(path.join(executedRoot, ".argent", "flows"), { recursive: true }); @@ -1565,17 +1574,22 @@ describe("recording a flow-execute step while several projects are in play", () udid: IOS_DEVICE, }); - expect(res.message).not.toContain("kept the raw flow-execute step"); - expect(await readSteps(recordingRoot, "wrapper")).toEqual([{ kind: "run", flow: "helper" }]); + expect(res.message).toContain("could not verify which file the live flow-execute ran"); + expect(await readSteps(recordingRoot, "wrapper")).toEqual([ + { kind: "tool", name: "flow-execute", args: { name: "helper", project_root: executedRoot } }, + ]); }); - it("warns when a same-named fragment exists in BOTH projects", async () => { + it("keeps the raw step when a same-named fragment exists in BOTH projects", async () => { const recordingRoot = await makeRoot("run-target-both"); const executedRoot = await makeRoot("run-target-both-other"); // The ambiguous case concurrent recording makes routine: a generic fragment // name that exists in two projects. `run: helper` resolves against the - // recording, so replay runs a DIFFERENT file than the one that just ran. + // recording, so replay would run a DIFFERENT file than the one that just + // ran — same name, different flow, both green and nothing said. The + // recorder refuses the substitution and keeps the raw call, which names + // both files and reproduces exactly what ran. await writeSavedFlow(recordingRoot, "helper", fragment); await writeSavedFlow(executedRoot, "helper", { executionPrerequisite: "", @@ -1589,13 +1603,14 @@ describe("recording a flow-execute step while several projects are in play", () udid: IOS_DEVICE, }); - // Still recorded as composition — that is what `run:` means — but the - // substitution is stated rather than silent. - expect(await readSteps(recordingRoot, "wrapper")).toEqual([{ kind: "run", flow: "helper" }]); - expect(res.message).toContain("replays THIS project's helper.yaml"); + expect(res.message).toContain("not the file the live flow-execute ran"); expect(res.message).toContain(executedRoot); + expect(await readSteps(recordingRoot, "wrapper")).toEqual([ + { kind: "tool", name: "flow-execute", args: { name: "helper", project_root: executedRoot } }, + ]); - // Same project on both sides is the unambiguous case and stays quiet. + // Same project on both sides is the unambiguous case: the file that ran and + // the sibling that would replay are one file, so it composes and stays quiet. await start(recordingRoot, "quiet"); const same = await addRawStep(recordingRoot, "quiet", "flow-execute", { name: "helper", @@ -1603,6 +1618,7 @@ describe("recording a flow-execute step while several projects are in play", () udid: IOS_DEVICE, }); expect(same.message).toBe('Step added to "quiet" flow'); + expect(await readSteps(recordingRoot, "quiet")).toEqual([{ kind: "run", flow: "helper.yaml" }]); }); }); diff --git a/packages/tool-server/test/flows/flow-remote-recording.test.ts b/packages/tool-server/test/flows/flow-remote-recording.test.ts index 754d5c333..726135421 100644 --- a/packages/tool-server/test/flows/flow-remote-recording.test.ts +++ b/packages/tool-server/test/flows/flow-remote-recording.test.ts @@ -167,6 +167,8 @@ describe("flow recording with a remote client (probe miss)", () => { addStep.execute( {}, { + name: "remote-flow", + project_root: CLIENT_ROOT, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(CLIENT_ROOT, ".argent", "flows", "login.yaml"), diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index bb254f570..8f468469e 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -647,7 +647,15 @@ describe("flow-add-step", () => { await fs.writeFile(otherTwin, "steps:\n - echo: theirs\n", "utf8"); const args = { name: "twin", project_root: otherRoot }; - const result = await tool.execute({}, { command: "flow-execute", args: JSON.stringify(args) }); + const result = await tool.execute( + {}, + { + name: "compose-twin", + project_root: tmpDir, + command: "flow-execute", + args: JSON.stringify(args), + } + ); // The live invoke ran the other project's copy… expect(registry.invokeTool).toHaveBeenCalledWith("flow-execute", args); @@ -685,7 +693,15 @@ describe("flow-add-step", () => { await writeSiblingFlow("frag", "steps:\n - echo: hi\n"); const args = { name: "Frag", project_root: tmpDir }; - const result = await tool.execute({}, { command: "flow-execute", args: JSON.stringify(args) }); + const result = await tool.execute( + {}, + { + name: "compose-name-casing", + project_root: tmpDir, + command: "flow-execute", + args: JSON.stringify(args), + } + ); // `run: Frag` names a flow no case-sensitive checkout can find, so the raw // step is kept and the warning hands back the recordable spelling. @@ -712,7 +728,15 @@ describe("flow-add-step", () => { ); const args = { name: "frag", project_root: tmpDir }; - const result = await tool.execute({}, { command: "flow-execute", args: JSON.stringify(args) }); + const result = await tool.execute( + {}, + { + name: "compose-name-rename", + project_root: tmpDir, + command: "flow-execute", + args: JSON.stringify(args), + } + ); expect(result.message).toContain('case-insensitively to "frag.YAML"'); expect(result.message).toContain( @@ -738,6 +762,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-name-mixed", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ name: "MixedCase", project_root: tmpDir }), } @@ -774,7 +800,12 @@ describe("flow-add-step", () => { } const result = await tool.execute( {}, - { command: "flow-execute", args: JSON.stringify(args) } + { + name: "compose-unanchored", + project_root: tmpDir, + command: "flow-execute", + args: JSON.stringify(args), + } ); expect(result.message).toContain(`project_root must be an absolute path ${detail}`); @@ -833,7 +864,12 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, - { command: "flow-execute", args: JSON.stringify({ name: "frag", project_root: base }) } + { + name: "rec", + project_root: base, + command: "flow-execute", + args: JSON.stringify({ name: "frag", project_root: base }), + } ); // Anchored beside the symlink's spelling this would miss the fragment and @@ -861,7 +897,12 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, - { command: "flow-execute", args: JSON.stringify({ name: "frag", project_root: base }) } + { + name: "rec", + project_root: base, + command: "flow-execute", + args: JSON.stringify({ name: "frag", project_root: base }), + } ); expect(result.message).toMatch(/could not resolve/i); @@ -893,7 +934,12 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, - { command: "flow-execute", args: JSON.stringify({ name: "frag", project_root: base }) } + { + name: "rec", + project_root: base, + command: "flow-execute", + args: JSON.stringify({ name: "frag", project_root: base }), + } ); expect(result.message).toMatch(/not the file the live flow-execute ran/i); @@ -915,6 +961,8 @@ describe("flow-add-step", () => { const result = await tool.execute( {}, { + name: "compose-path", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: sibling, project_root: tmpDir }), } @@ -952,6 +1000,8 @@ describe("flow-add-step", () => { .execute( {}, { + name: "compose-casing", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(tmpDir, ".argent", "flows", "Sibling.yaml"), @@ -994,6 +1044,8 @@ describe("flow-add-step", () => { .execute( {}, { + name: "compose-rename", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(tmpDir, ".argent", "flows", "frag.yaml"), @@ -1028,6 +1080,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-outside", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: outside, project_root: tmpDir }), } @@ -1056,6 +1110,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-dotdot", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: dotdot, project_root: tmpDir }), } @@ -1084,6 +1140,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-stemless", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: stemless, project_root: tmpDir }), } @@ -1110,6 +1168,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-cased", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(tmpDir, ".argent", "flows", "Login.YAML"), @@ -1135,6 +1195,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-mismatch", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(tmpDir, ".argent", "flows", "login.yaml"), @@ -1184,6 +1246,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-relative", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: sibling, project_root: root }), } @@ -1213,6 +1277,8 @@ describe("flow-add-step", () => { tool.execute( {}, { + name: "compose-rootless", + project_root: tmpDir, command: "flow-execute", args: JSON.stringify({ flow_path: path.join(tmpDir, ".argent", "flows", "login.yaml"), @@ -1254,7 +1320,15 @@ describe("flow-add-step", () => { const args = buildArgs(path.join(tmpDir, ".argent", "flows", "login.yaml"), tmpDir); await expect( - tool.execute({}, { command: "flow-execute", args: JSON.stringify(args) }) + tool.execute( + {}, + { + name: "compose-ambiguous", + project_root: tmpDir, + command: "flow-execute", + args: JSON.stringify(args), + } + ) ).rejects.toThrow(); // The nested call must reach flow-execute exactly as written — no flow_path diff --git a/packages/tool-server/test/http-flow-path-boundary.test.ts b/packages/tool-server/test/http-flow-path-boundary.test.ts index ac3871ec4..ff18ce3b0 100644 --- a/packages/tool-server/test/http-flow-path-boundary.test.ts +++ b/packages/tool-server/test/http-flow-path-boundary.test.ts @@ -7,11 +7,7 @@ import { ArtifactStore, type Registry, type ToolContext } from "@argent/registry import { createHttpApp, type HttpAppHandle } from "../src/http"; import { createRunFlowTool } from "../src/tools/flows/flow-run"; import { flowReadPrerequisiteTool } from "../src/tools/flows/flow-read-prerequisite"; -import { - clearActiveFlow, - clearActiveProjectRoot, - serializeFlow, -} from "../src/tools/flows/flow-utils"; +import { serializeFlow } from "../src/tools/flows/flow-utils"; vi.mock("../src/utils/update-checker", () => ({ getUpdateState: vi.fn(() => ({ updateInstallable: false, currentVersion: "1.0.0" })), @@ -99,13 +95,10 @@ beforeEach(async () => { ); steps = stepRegistry(); handle = createHttpApp(httpRegistry(steps)); - clearActiveFlow(); }); afterEach(async () => { handle?.dispose(); - clearActiveFlow(); - clearActiveProjectRoot(); await fs.rm(tmpDir, { recursive: true, force: true }); if (originalToken === undefined) delete process.env.ARGENT_AUTH_TOKEN; else process.env.ARGENT_AUTH_TOKEN = originalToken; From a7ce547520e0e49d6d0a873406da199ba7ec770f Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:11:03 +0200 Subject: [PATCH 30/98] fix(flow): key a recording by the file the filesystem resolves, not the spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recording sessions and their locks were keyed by getFlowPath — pure path.join string math — while the write resolved the target through the filesystem. Two distinct, correctly spelled keys landing on one real file therefore got two independent sessions and two independent locks, and nothing detected it: each session genuinely was the current one for its own key, so assertSessionStillLive never fired, the second start reported no `restarted`, and both agents were told they finished successfully over a mixture of each other's steps. Three configurations reach it, all supported: a flow file symlinked into a shared vault from two projects, `.argent/flows` itself symlinked from two packages of a monorepo, and two flow names differing only in case on a case-insensitive volume (APFS by default). Resolve the key through the filesystem instead, sharing the exact resolution writeFlowFile already performs — so the identity a recording is keyed by and the file its steps land in cannot disagree. macOS answers the case question too: realpath("login.yaml") returns "Login.yaml" when that is what is on disk, while a case-sensitive volume simply fails to find it and the two stay two, which is correct there. The earlier objection to normalizing held only for a hand-rolled normalization; asking the filesystem was always available on the host path, and "client" mode needs no special case since both realpath calls fail there and the fallback returns the old pure-path spelling unchanged. A session found under a key someone else spelled differently is no longer handed over: requireRecordingSession compares the caller's flow path against the holder's and reports the collision, naming both spellings and the take that now owns the file, rather than silently enrolling the caller in another agent's recording. Key resolution is realpath, which completes on libuv's threadpool in an order unrelated to the order requested — and every recording tool resolves before joining its flow file's lock queue. Callers spelling one path the same way share one in-flight resolution so their continuations run in subscription order and the queue stays FIFO; without that, a restart could land behind the append it is meant to discard. --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-add-step.ts | 2 +- .../src/tools/flows/flow-finish-recording.ts | 4 +- .../src/tools/flows/flow-insert-echo.ts | 2 +- .../src/tools/flows/flow-start-recording.ts | 4 +- .../tool-server/src/tools/flows/flow-utils.ts | 226 +++++++--- .../flows/flow-concurrent-recording.test.ts | 194 ++++++-- .../tool-server/test/flows/flow-tools.test.ts | 4 +- .../tool-server/test/flows/flow-utils.test.ts | 418 +++++++++--------- 9 files changed, 547 insertions(+), 309 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index fb542762f..ebf1eb49f 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -141,7 +141,7 @@ Every tool during recording returns the current flow file contents, so you can t - **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. - **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. - **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` (in the return value) counts the discarded take, but can be absent even on a restart. Starting a _different_ flow abandons nothing. -- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. +- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. A name that only _resolves_ to the same file — a differently-cased one on macOS/Windows, or a flow (or `.argent/flows`) symlinked into a shared vault from two projects — is the same key, because the key is the file the filesystem resolves to, not the spelling you passed. That collision is reported rather than silent: the second start says `restarted` with a `discardedSteps` count, and the first recording's next call fails with `… are the same file on this filesystem …` naming both spellings. - **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. (A takeover by another agent is different: it resolves to _their_ recording and succeeds — see the previous bullet — rather than reaching this error.) The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. - **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index f7eae0e3b..7dac842f4 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -455,7 +455,7 @@ If a step was recorded by mistake, edit the .yaml file directly to remove it.`, zodSchema, services: () => ({}), async execute(_services, params, ctx) { - const session = requireRecordingSession(params.project_root, params.name); + const session = await requireRecordingSession(params.project_root, params.name); const args: Record = params.args ? JSON.parse(params.args) : {}; // A nested flow-execute must never carry a raw flow_path into the live diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index 96d2e0362..ad1bebfb5 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -90,7 +90,7 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps params.project_root, params.name, async () => { - const session = requireRecordingSession(params.project_root, params.name); + const session = await requireRecordingSession(params.project_root, params.name); // Host mode re-reads the file so manual edits made during the recording // survive into the summary; in client mode this host never has the file, @@ -119,7 +119,7 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps // {@link renderToolArgs}; keeping the order is what makes the next one // recoverable rather than fatal. const summary = summarizeSteps(flow); - clearRecordingSession(params.project_root, params.name); + await clearRecordingSession(params.project_root, params.name); return { filePath, flowFile, savedTo, flow, summary }; } ); diff --git a/packages/tool-server/src/tools/flows/flow-insert-echo.ts b/packages/tool-server/src/tools/flows/flow-insert-echo.ts index 90f1772a9..4aec50dec 100644 --- a/packages/tool-server/src/tools/flows/flow-insert-echo.ts +++ b/packages/tool-server/src/tools/flows/flow-insert-echo.ts @@ -33,7 +33,7 @@ Returns { message, flowFile, savedTo }. Fails if that flow has no recording in p zodSchema, services: () => ({}), async execute(_services, params) { - const session = requireRecordingSession(params.project_root, params.name); + const session = await requireRecordingSession(params.project_root, params.name); const { flowFile, savedTo } = await appendStepToFlow(session, { kind: "echo", diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index d5699e9dd..1d18eb57f 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -143,7 +143,7 @@ to remove or reorder steps.`, // restart — the file is already truncated — as a plain fresh start, // discarding the count computed here. This read is inside our own key's // lock, so it and the count agree. - const replaced = getRecordingSession(params.project_root, params.name) ?? null; + const replaced = (await getRecordingSession(params.project_root, params.name)) ?? null; const discardedSteps = replaced === null ? undefined @@ -158,7 +158,7 @@ to remove or reorder steps.`, } else { savedTo = clientFileDirective(filePath, flowFile); } - startRecordingSession({ + await startRecordingSession({ name: params.name, projectRoot: params.project_root, persist, diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 688e77682..279433f82 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -95,32 +95,18 @@ export function assertSafeFlowName(name: string): void { } /** - * The flow file `/.argent/flows/.yaml`. Pure path math over - * two validated inputs, and the recording-session key (see - * {@link startRecordingSession}). + * The flow file `/.argent/flows/.yaml`, as the CALLER + * spelled it. Pure path math over two validated inputs — this is the path + * reported back to the agent, not the recording-session key (that is + * {@link resolveFlowKey}, which asks the filesystem instead). * - * Two different projects can never collide on one key. The converse is only - * true up to `path.join`, which folds a trailing slash, `//` and `.` segments - * but NOT symlinks or case. Two spellings that the filesystem considers one - * path therefore mint two sessions — and two independent locks — over one file, - * which bypasses every guarantee here: neither session is ever "superseded", so - * a restart silently truncates the other's live take and both agents are told - * they finished successfully with a mixture of each other's steps. - * - * Two ways in, and the second is the likelier one: - * - the ROOT spelled two ways (`/tmp/p` vs `/private/tmp/p` on macOS). Needs - * two callers disagreeing about one directory; roots are cwd-derived, so this - * is rare. - * - the NAME cased two ways (`Login` vs `login`) on a case-insensitive volume, - * which APFS is by default. The name is agent-chosen free text, so this needs - * only two agents naming the same flow differently — hence the "pick a name - * unique to your task" warning on `flow-start-recording`. - * - * Neither is normalized away, because the correct normalization is the - * filesystem's and we cannot ask it: case-folding the key would wrongly merge - * two genuinely distinct flows on a case-SENSITIVE volume (ext4), and resolving - * symlinks is impossible in "client" mode, where the root does not exist on this - * host at all. + * `path.join` folds a trailing slash, `//` and `.` segments but NOT symlinks or + * case, so two callers can spell one real file two ways here: the ROOT spelled + * two ways (`/tmp/p` vs `/private/tmp/p` on macOS), the flows dir or the flow + * file symlinked into a shared vault from two projects, or the NAME cased two + * ways (`Login` vs `login`) on a case-insensitive volume, which APFS is by + * default. Keying sessions on this string would mint two sessions — and two + * independent locks — over one file, so nothing here is a session key. */ export function getFlowPath(projectRoot: string, name: string): string { const flowsDir = getFlowsDir(projectRoot); @@ -140,6 +126,66 @@ export function getFlowPath(projectRoot: string, name: string): string { return filePath; } +/** + * The flow file's identity as the FILESYSTEM sees it, which is what a recording + * session and its lock are keyed by. Two callers who spell one real file two + * ways — a symlink into a shared vault, a symlinked `.argent/flows`, a root + * spelled `/tmp` vs `/private/tmp`, a name cased two ways on APFS — resolve to + * one key here, so the collision reads as the restart it actually is instead of + * minting a second session that silently truncates the first. + * + * This is the same resolution {@link writeFlowFile} performs before its swap, + * deliberately: the key and the write agree by construction, so wherever the + * filesystem declines to answer (a dangling symlink, a flows dir that does not + * exist yet) both fall back to the same pure-path spelling and the two remain + * two — which is correct, because two files is what the write then produces. + * + * It costs one `realpath` pair per recording tool call, on a path already doing + * file I/O. + * + * The earlier objection to normalizing — that the correct normalization is the + * filesystem's and we cannot ask it — held only for a hand-rolled one. Asking + * the filesystem is exactly what this does, so a case-SENSITIVE volume (ext4) + * keeps `Login` and `login` apart on its own: `realpath` there simply fails to + * find the variant spelling. + * + * "client" mode needs no special case. The caller's root does not exist on this + * host, so both `realpath` calls fail and the fallback returns + * {@link getFlowPath} unchanged — the old behavior, and the only one available + * when the file is on another machine. Two clients that share a flow file + * across that boundary are beyond this process's reach, as they were before. + */ +export function resolveFlowKey(projectRoot: string, name: string): Promise { + const spelled = getFlowPath(projectRoot, name); + const inFlight = keyResolutions.get(spelled); + if (inFlight) return inFlight; + const resolving = canonicalFlowPath(spelled).finally(() => { + if (keyResolutions.get(spelled) === resolving) keyResolutions.delete(spelled); + }); + keyResolutions.set(spelled, resolving); + return resolving; +} + +/** + * Canonical-key resolutions currently IN FLIGHT, keyed by the spelled path. + * Not a cache — the entry is dropped the moment it settles, so a symlink + * repointed between two tool calls is seen — but a sequencer. + * + * Resolution is `realpath`, which runs on libuv's threadpool and therefore + * completes in an order unrelated to the order it was requested in. Every + * recording tool resolves its key before joining its flow file's lock queue, so + * without this, which of two tool calls acquires the lock first would be + * decided by threadpool scheduling rather than by which was issued first — a + * restart could land behind the append it is supposed to discard. Callers that + * spell one path the same way share one promise, so their continuations run in + * subscription order and the queue they join stays FIFO. + * + * Two DIFFERENT spellings of one file resolve independently and so race, as + * they did before. Nothing depends on their order: mutual exclusion comes from + * the resolved key, which is the same for both. + */ +const keyResolutions = new Map>(); + /** * How the flow file a caller addressed is spelled in its own directory. * `listed`: the directory carries that basename byte-for-byte — or its listing @@ -207,6 +253,12 @@ export interface RecordingSession { name: string; /** Caller-supplied project root, as passed to every recording tool. */ projectRoot: string; + /** + * The {@link resolveFlowKey} this session is registered under. Stored rather + * than re-derived, so {@link assertSessionStillLive} asks about the key the + * session actually holds — and needs no filesystem round trip to do it. + */ + key: string; persist: FlowPersistMode; /** * Absolute path of the flow file as the CALLER knows it. A real host path in @@ -221,11 +273,21 @@ export interface RecordingSession { } /** - * Live recordings, keyed by {@link getFlowPath} — the identity of the artifact - * being built. Two sessions on one key mean two writers on one output file (a - * genuine collision); two different keys are independent, so concurrent agents - * recording different flows — in one project or across projects, against one - * device or several — never write into each other's take. + * Live recordings, keyed by {@link resolveFlowKey} — the identity of the + * artifact being built, as the FILESYSTEM resolves it rather than as a caller + * spelled it. Two sessions on one key mean two writers on one output file (a + * genuine collision, reported as a restart); two different keys are two + * different files, so concurrent agents recording different flows — in one + * project or across projects, against one device or several — never write into + * each other's take. + * + * The one window the key does not close: two starts BOTH in flight before + * either has created its file. Neither realpath can see a file that is not + * there yet, so two spellings of one not-yet-existing file resolve apart, and + * the two writes then land on one file. It closes itself on the next call — + * the file exists by then, so both spellings resolve together and the loser + * finds its key held by the other session, which fails loudly in + * {@link requireRecordingSession} rather than silently mixing takes. * * Isolation of the recorded artifact, not of the fact that a recording exists: * the not-found path of {@link requireRecordingSession} deliberately names the @@ -284,12 +346,12 @@ async function withFlowLock(key: string, fn: () => Promise): Promise { * truncate-then-register, `flow-finish-recording`'s read-then-clear — hold the * same lock that {@link appendStepToFlow} takes. */ -export function withFlowFileLock( +export async function withFlowFileLock( projectRoot: string, name: string, fn: () => Promise ): Promise { - return withFlowLock(getFlowPath(projectRoot, name), fn); + return withFlowLock(await resolveFlowKey(projectRoot, name), fn); } /** @@ -348,19 +410,21 @@ export interface RecordingSessionInit { * take), or null — the common case, including starting a second, unrelated * recording while others are in progress. */ -export function startRecordingSession(init: RecordingSessionInit): RecordingSession | null { - const key = getFlowPath(init.projectRoot, init.name); +export async function startRecordingSession( + init: RecordingSessionInit +): Promise { + const key = await resolveFlowKey(init.projectRoot, init.name); const previous = recordings.get(key) ?? null; - recordings.set(key, { ...init, lastTouchedSeq: touch() }); + recordings.set(key, { ...init, key, lastTouchedSeq: touch() }); evictIfOverCapacity(); return previous; } -export function getRecordingSession( +export async function getRecordingSession( projectRoot: string, name: string -): RecordingSession | undefined { - return recordings.get(getFlowPath(projectRoot, name)); +): Promise { + return recordings.get(await resolveFlowKey(projectRoot, name)); } /** @@ -375,8 +439,11 @@ export function listActiveRecordings(): { name: string; projectRoot: string; ste })); } -export function requireRecordingSession(projectRoot: string, name: string): RecordingSession { - const session = getRecordingSession(projectRoot, name); +export async function requireRecordingSession( + projectRoot: string, + name: string +): Promise { + const session = await getRecordingSession(projectRoot, name); if (!session) { // Name what was asked for AND what is live, so the agent can self-correct: // with concurrent recordings the usual cause is a typo in `name` or the @@ -424,17 +491,45 @@ export function requireRecordingSession(projectRoot: string, name: string): Reco } ); } + // The key is the file's identity, so a session found under it may have been + // registered by a caller who spells that one file differently — a symlink + // into a shared vault, a symlinked `.argent/flows`, a name cased two ways on + // APFS. Handing it over would silently enrol this caller in the OTHER take: + // its steps would land in a file it never addressed, under a prerequisite it + // never declared, and its finish would report the other agent's steps as its + // own. That collision is what the restart already destroyed this caller's + // take for, so report it as the loss it is rather than papering over it. A + // root spelled with a trailing slash is not one of these — `getFlowPath` + // normalizes both sides before they are compared. + const asked = getFlowPath(projectRoot, name); + const held = getFlowPath(session.projectRoot, session.name); + if (asked !== held) { + throw new FailureError( + `Recording of "${name}" in ${projectRoot} is no longer active — ${held} and ${asked} ` + + `are the same file on this filesystem (a symlink, or a case-insensitive volume), and ` + + `"${session.name}" in ${session.projectRoot} is the take that now holds it. Starting ` + + `that recording truncated this one. Record under a name that resolves to its own file, ` + + `or coordinate with the other caller — restarting here would destroy their take in turn.`, + { + error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, + failure_stage: "flow_recording_key_aliased", + failure_area: "tool_server", + error_kind: "validation", + } + ); + } session.lastTouchedSeq = touch(); return session; } -export function clearRecordingSession(projectRoot: string, name: string): void { - recordings.delete(getFlowPath(projectRoot, name)); +export async function clearRecordingSession(projectRoot: string, name: string): Promise { + recordings.delete(await resolveFlowKey(projectRoot, name)); } export function __resetRecordingsForTesting(): void { recordings.clear(); flowFileLocks.clear(); + keyResolutions.clear(); } /** @@ -2466,22 +2561,30 @@ function scrubTempPath(err: unknown, tmpPath: string, filePath: string): Error { return scrubbed; } -async function writeFlowFile(filePath: string, content: string): Promise { - // Swap onto the flow file's REAL path. A saved flow may be a symlink into a - // shared vault — the runner canonicalizes before reading, and `run:` - // composition anchors on the real file — and rename(2) replaces the path it - // is handed, so renaming onto the link's own spelling would swap the symlink - // for a regular file and strand the vault copy with the pre-recording - // content. A plain write follows the link; resolving first keeps that - // behavior while keeping the swap atomic. - // - // The directory is resolved separately so that a flow file which does not - // exist yet (the first write of a recording, which has no realpath of its - // own) still lands on the same canonical spelling as every later append — - // otherwise the first swap and the rest would disagree wherever an ancestor - // is itself a symlink, which is the default for the temp dir on macOS. +/** + * A flow file's REAL path. A saved flow may be a symlink into a shared vault — + * the runner canonicalizes before reading, and `run:` composition anchors on + * the real file — and rename(2) replaces the path it is handed, so renaming + * onto the link's own spelling would swap the symlink for a regular file and + * strand the vault copy with the pre-recording content. A plain write follows + * the link; resolving first keeps that behavior while keeping the swap atomic. + * + * The directory is resolved separately so that a flow file which does not exist + * yet (the first write of a recording, which has no realpath of its own) still + * lands on the same canonical spelling as every later append — otherwise the + * first swap and the rest would disagree wherever an ancestor is itself a + * symlink, which is the default for the temp dir on macOS. + * + * Shared with {@link resolveFlowKey}, so the identity a recording is keyed by + * and the file its steps land in can never disagree. + */ +async function canonicalFlowPath(filePath: string): Promise { const dir = await fs.realpath(path.dirname(filePath)).catch(() => path.dirname(filePath)); - const target = await fs.realpath(filePath).catch(() => path.join(dir, path.basename(filePath))); + return await fs.realpath(filePath).catch(() => path.join(dir, path.basename(filePath))); +} + +async function writeFlowFile(filePath: string, content: string): Promise { + const target = await canonicalFlowPath(filePath); const tmpPath = path.join( path.dirname(target), `.argent-flow-${process.pid}-${++flowWriteSeq}.tmp` @@ -2606,7 +2709,7 @@ export type FlowSavedTo = string | ClientFileDirective; * NEXT call on the key reports the recording gone. */ function assertSessionStillLive(session: RecordingSession, step: FlowStep): void { - const current = recordings.get(getFlowPath(session.projectRoot, session.name)); + const current = recordings.get(session.key); if (current === session) return; // A key that is occupied by a DIFFERENT session was restarted; an empty key // was either finished or evicted by the MAX_RECORDINGS backstop, which the @@ -2660,7 +2763,12 @@ export async function appendStepToFlow( session: RecordingSession, step: FlowStep ): Promise<{ flowFile: string; savedTo: FlowSavedTo }> { - return withFlowFileLock(session.projectRoot, session.name, async () => { + // The session's OWN key, not a fresh resolution of it: the lock this append + // takes and the identity {@link assertSessionStillLive} checks must be the + // same one, or a key that moved under the session (a symlink repointed + // mid-recording) would let the append hold one lock while asserting about + // another. + return withFlowLock(session.key, async () => { assertSessionStillLive(session, step); session.lastTouchedSeq = touch(); if (session.persist === "host") { diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 3a8a8dea2..3151fc368 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -255,8 +255,8 @@ describe("two recordings in one project", () => { expect(finished.steps).toBe(1); // Only alpha's key was cleared. - expect(getRecordingSession(root, "alpha")).toBeUndefined(); - expect(getRecordingSession(root, "beta")?.filePath).toBe(flowPath(root, "beta")); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); + expect((await getRecordingSession(root, "beta"))?.filePath).toBe(flowPath(root, "beta")); // beta keeps recording into its own file. await addEcho(root, "beta", "b2"); @@ -288,8 +288,8 @@ describe("the same flow name under two project roots", () => { expect(await readMarkers(rootB, "checkout")).toEqual(["tool:b1", "tool:b2"]); // Sessions carry their own project root and prerequisite, not the other's. - expect(getRecordingSession(rootA, "checkout")?.projectRoot).toBe(rootA); - expect(getRecordingSession(rootB, "checkout")?.projectRoot).toBe(rootB); + expect((await getRecordingSession(rootA, "checkout"))?.projectRoot).toBe(rootA); + expect((await getRecordingSession(rootB, "checkout"))?.projectRoot).toBe(rootB); const finishedA = await finish(rootA, "checkout"); expect(finishedA.path).toBe(flowPath(rootA, "checkout")); @@ -303,6 +303,126 @@ describe("the same flow name under two project roots", () => { }); }); +// ── Two keys the filesystem considers one file ─────────────────────── + +/** + * The isolation above is stated per KEY, and the key is `path.join` string + * math while the write resolves through the filesystem. Everything here is a + * pair of distinct, correctly spelled keys that land on ONE real file — via a + * symlink, or via a case-insensitive volume. Nothing may treat those as + * independent: the second start must read as the restart it actually is + * (discarding the first take, counted), and the first recording's next append + * must fail loudly rather than land in a take that is no longer its own. + */ +describe("two recording keys that resolve to one file", () => { + /** Skipped on a case-sensitive volume, where the two names ARE two files. */ + async function fsFoldsCase(dir: string): Promise { + const probe = path.join(dir, "ArgentCaseProbe"); + await fs.writeFile(probe, "", "utf8"); + try { + await fs.stat(path.join(dir, "argentcaseprobe")); + return true; + } catch { + return false; + } finally { + await fs.rm(probe, { force: true }); + } + } + + it("treats a second project's symlink to the same flow file as a restart", async () => { + const vault = await makeRoot("vault"); + const rootA = await makeRoot("symlink-a"); + const rootB = await makeRoot("symlink-b"); + const shared = path.join(vault, "checkout.yaml"); + await fs.writeFile(shared, "steps: []\n", "utf8"); + for (const root of [rootA, rootB]) { + await fs.mkdir(path.dirname(flowPath(root, "checkout")), { recursive: true }); + await fs.symlink(shared, flowPath(root, "checkout")); + } + + await start(rootA, "checkout"); + await addEcho(rootA, "checkout", "h1-a"); + await addEcho(rootA, "checkout", "h1-b"); + await addEcho(rootA, "checkout", "h1-c"); + + // B addresses the same real file under its own spelling. That is a + // restart, and it destroys A's three-step take — so it must say so. + const restarted = await start(rootB, "checkout"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(3); + + // A's session lost the key; its next append fails instead of landing in + // B's take. + const err = await captureFailure(addEcho(rootA, "checkout", "h1-d")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect(formatErrorForAgent(err)).toContain("no longer active"); + + await addEcho(rootB, "checkout", "h2-a"); + // A's finish reports the same loss, rather than handing back B's take as + // if it were A's own. + const finishErr = await captureFailure(finish(rootA, "checkout")); + expect(getFailureSignal(finishErr)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + + const finishedB = await finish(rootB, "checkout"); + expect(markers(parseFlow(finishedB.flowFile).steps)).toEqual(["echo:h2-a"]); + // The link survived the swap: one real file, holding only B's take. + expect(markers(parseFlow(await fs.readFile(shared, "utf8")).steps)).toEqual(["echo:h2-a"]); + expect((await fs.lstat(flowPath(rootA, "checkout"))).isSymbolicLink()).toBe(true); + }); + + it("treats a shared symlinked flows DIRECTORY the same way", async () => { + const vault = await makeRoot("vault-dir"); + const rootA = await makeRoot("symdir-a"); + const rootB = await makeRoot("symdir-b"); + for (const root of [rootA, rootB]) { + await fs.mkdir(path.join(root, ".argent"), { recursive: true }); + await fs.symlink(vault, path.join(root, ".argent", "flows")); + } + + await start(rootA, "checkout"); + await addEcho(rootA, "checkout", "a1"); + await addEcho(rootA, "checkout", "a2"); + + const restarted = await start(rootB, "checkout"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(2); + + const err = await captureFailure(addEcho(rootA, "checkout", "a3")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + }); + + it("treats two case-variant flow names on a case-folding volume as one key", async () => { + const root = await makeRoot("case-variant"); + await fs.mkdir(path.dirname(flowPath(root, "Login")), { recursive: true }); + if (!(await fsFoldsCase(path.dirname(flowPath(root, "Login"))))) return; + + await start(root, "Login"); + await addEcho(root, "Login", "l1"); + await addEcho(root, "Login", "l2"); + + // `login` is a different key by string math, the same file by this volume. + const restarted = await start(root, "login"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(2); + + const err = await captureFailure(addEcho(root, "Login", "l3")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + }); + + it("keeps two genuinely distinct flows independent", async () => { + // The control: no symlink, no case variance, so nothing is canonicalized + // together and the isolation guarantee holds exactly as stated. + const rootA = await makeRoot("control-a"); + const rootB = await makeRoot("control-b"); + await start(rootA, "checkout"); + await start(rootB, "checkout"); + await addEcho(rootA, "checkout", "a1"); + await addEcho(rootB, "checkout", "b1"); + expect(await readMarkers(rootA, "checkout")).toEqual(["echo:a1"]); + expect(await readMarkers(rootB, "checkout")).toEqual(["echo:b1"]); + }); +}); + // ── Addressing a key that isn't live ───────────────────────────────── describe("addressing an unknown recording key", () => { @@ -371,7 +491,7 @@ describe("concurrent flow-add-step calls on one recording", () => { expect([...recorded].sort()).toEqual(tags.map((t) => `tool:${t}`).sort()); // The in-memory copy the session serves to flow-finish-recording agrees. - expect(getRecordingSession(root, "burst")?.flow.steps).toHaveLength(tags.length); + expect((await getRecordingSession(root, "burst"))?.flow.steps).toHaveLength(tags.length); const finished = await finish(root, "burst"); expect(finished.steps).toBe(tags.length); }); @@ -513,6 +633,9 @@ describe("the flow-file lock", () => { // assertion above is about release, not about the map never being used. const gate = openGate(); const held = withFlowFileLock(root, "alpha", () => gate.promise); + // `settle` first: the lock is taken on the CANONICAL key, so the entry + // appears only once that resolution has come back from the filesystem. + await settle(); expect(__flowFileLockCountForTesting()).toBe(before + 1); gate.open(); await held; @@ -868,8 +991,8 @@ describe("running a flow in a third project while two recordings are live", () = expect(runResult).toHaveProperty("ok", true); // Both sessions still point at their own files… - expect(getRecordingSession(rootA, "alpha")?.filePath).toBe(flowPath(rootA, "alpha")); - expect(getRecordingSession(rootB, "beta")?.filePath).toBe(flowPath(rootB, "beta")); + expect((await getRecordingSession(rootA, "alpha"))?.filePath).toBe(flowPath(rootA, "alpha")); + expect((await getRecordingSession(rootB, "beta"))?.filePath).toBe(flowPath(rootB, "beta")); // …and subsequent steps still land there. await addStep(rootA, "alpha", "a2"); @@ -908,7 +1031,7 @@ describe("restarting a recording on one key", () => { // beta neither lost its steps nor its session. expect(await readMarkers(root, "beta")).toEqual(["echo:b1"]); - expect(getRecordingSession(root, "beta")?.flow.steps).toHaveLength(1); + expect((await getRecordingSession(root, "beta"))?.flow.steps).toHaveLength(1); await addEcho(root, "beta", "b2"); expect(await readMarkers(root, "beta")).toEqual(["echo:b1", "echo:b2"]); @@ -935,7 +1058,7 @@ describe("restarting a recording on one key", () => { // …while the same name under the other root — a different key — is not // touched: that recording kept its step and its session. expect(await readMarkers(rootA, "alpha")).toEqual(["tool:a1"]); - expect(getRecordingSession(rootA, "alpha")?.flow.steps).toHaveLength(1); + expect((await getRecordingSession(rootA, "alpha"))?.flow.steps).toHaveLength(1); }); it("counts the steps the FILE held, not the ones this session appended", async () => { @@ -1029,7 +1152,7 @@ describe("a restart that lands while a step is still running", () => { // The new take is empty — no step from the discarded one leaked into it. expect(await readMarkers(root, "alpha")).toEqual([]); - expect(getRecordingSession(root, "alpha")?.flow.steps).toHaveLength(0); + expect((await getRecordingSession(root, "alpha"))?.flow.steps).toHaveLength(0); // …and the restarted recording still works. await addStep(root, "alpha", "a3"); @@ -1072,7 +1195,7 @@ describe("a restart that lands while a step is still running", () => { const root = await makeRoot("restart-lock"); await start(root, "alpha"); await addStep(root, "alpha", "a1"); - const firstSession = getRecordingSession(root, "alpha"); + const firstSession = await getRecordingSession(root, "alpha"); // Stand in for an append that is mid read-modify-write on alpha's file. const order: string[] = []; @@ -1089,7 +1212,7 @@ describe("a restart that lands while a step is still running", () => { // while another writer holds the file. expect(order).toEqual([]); expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); - expect(getRecordingSession(root, "alpha")).toBe(firstSession); + expect(await getRecordingSession(root, "alpha")).toBe(firstSession); order.push("lock-released"); lock.open(); @@ -1100,14 +1223,14 @@ describe("a restart that lands while a step is still running", () => { expect(restarted.restarted).toBe(true); expect(restarted.discardedSteps).toBe(1); expect(await readMarkers(root, "alpha")).toEqual([]); - expect(getRecordingSession(root, "alpha")).not.toBe(firstSession); + expect(await getRecordingSession(root, "alpha")).not.toBe(firstSession); }); it("keeps a step queued behind the restart out of the new take", async () => { const root = await makeRoot("restart-queued-append"); await start(root, "alpha"); await addStep(root, "alpha", "a1"); - const discarded = getRecordingSession(root, "alpha"); + const discarded = await getRecordingSession(root, "alpha"); // Park a holder on alpha's file lock. Everything issued below queues behind // it, so the interleaving is fixed by the lock's arrival order rather than @@ -1117,13 +1240,14 @@ describe("a restart that lands while a step is still running", () => { // Second in the queue: the restart — truncate the file, swap the session. const restarting = start(root, "alpha"); - // Third: a step for the take the restart is discarding. flow-add-echo - // resolves its session and takes the lock in one synchronous block, so this + // Third: a step for the take the restart is discarding. Both calls resolve + // the same spelled path, so they share one in-flight key resolution and + // join the lock queue in the order issued (see `keyResolutions`) — this // append is bound to the OLD session and enters the lock the instant the - // restart's critical section ends — the window a truncate that is not fused - // to the session swap leaves open, onto a file that is already empty. + // restart's critical section ends, which is the window a truncate that is + // not fused to the session swap leaves open, onto a file already empty. const appending = addEcho(root, "alpha", "stray"); - expect(getRecordingSession(root, "alpha")).toBe(discarded); + expect(await getRecordingSession(root, "alpha")).toBe(discarded); holder.open(); const [restartResult, appendResult] = await Promise.allSettled([restarting, appending]); @@ -1143,7 +1267,7 @@ describe("a restart that lands while a step is still running", () => { // The invariant: the new take's file is what the new take says it is, and // carries nothing from the discarded one. - const session = getRecordingSession(root, "alpha"); + const session = await getRecordingSession(root, "alpha"); expect(session).toBeDefined(); expect(session).not.toBe(discarded); const onDisk = await readMarkers(root, "alpha"); @@ -1199,7 +1323,7 @@ describe("a finish that lands while a step is still running", () => { } // Either way the recording is gone, and nothing can be appended to it. - expect(getRecordingSession(root, "alpha")).toBeUndefined(); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); } }); @@ -1220,7 +1344,7 @@ describe("a finish that lands while a step is still running", () => { await settle(); expect(order).toEqual([]); // The session is still live: resolve-read-clear is one critical section. - expect(getRecordingSession(root, "alpha")).toBeDefined(); + expect(await getRecordingSession(root, "alpha")).toBeDefined(); order.push("lock-released"); lock.open(); @@ -1230,7 +1354,7 @@ describe("a finish that lands while a step is still running", () => { expect(order).toEqual(["lock-released", "finish-returned"]); expect(finished.steps).toBe(1); expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1"]); - expect(getRecordingSession(root, "alpha")).toBeUndefined(); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); }); it("rejects a step whose recording was already finished", async () => { @@ -1282,7 +1406,7 @@ describe("a finish on a flow file that no longer parses", () => { await start(root, "alpha"); await addStep(root, "alpha", "a1"); await addEcho(root, "alpha", "a2"); - const session = getRecordingSession(root, "alpha"); + const session = await getRecordingSession(root, "alpha"); const repaired = await fs.readFile(flowPath(root, "alpha"), "utf8"); await fs.writeFile(flowPath(root, "alpha"), NOT_A_LIST, "utf8"); @@ -1296,7 +1420,7 @@ describe("a finish on a flow file that no longer parses", () => { expect(await fs.readFile(flowPath(root, "alpha"), "utf8")).toBe(NOT_A_LIST); // The take survived the failure, as the same session object. - expect(getRecordingSession(root, "alpha")).toBe(session); + expect(await getRecordingSession(root, "alpha")).toBe(session); expect(session?.flow.steps).toHaveLength(2); // A retry while the file is still broken fails the same way — the recording @@ -1314,7 +1438,7 @@ describe("a finish on a flow file that no longer parses", () => { expect(finished.steps).toBe(3); expect(finished.summary).toHaveLength(3); expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["tool:a1", "echo:a2", "echo:a3"]); - expect(getRecordingSession(root, "alpha")).toBeUndefined(); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); }); it("leaves a concurrent recording — and its own key — exactly as they were", async () => { @@ -1340,7 +1464,7 @@ describe("a finish on a flow file that no longer parses", () => { .map((r) => r.name) .sort() ).toEqual(["alpha", "beta"]); - expect(getRecordingSession(root, "alpha")?.filePath).toBe(flowPath(root, "alpha")); + expect((await getRecordingSession(root, "alpha"))?.filePath).toBe(flowPath(root, "alpha")); // beta neither lost its file nor its ability to finish. expect(await readMarkers(root, "beta")).toEqual(["echo:b1"]); @@ -1348,7 +1472,7 @@ describe("a finish on a flow file that no longer parses", () => { expect(markers(parseFlow(finishedBeta.flowFile).steps)).toEqual(["echo:b1"]); // alpha outlived beta's finish too, and finishes on the repaired file. - expect(getRecordingSession(root, "alpha")).toBeDefined(); + expect(await getRecordingSession(root, "alpha")).toBeDefined(); await fs.writeFile( flowPath(root, "alpha"), 'executionPrerequisite: ""\nsteps:\n - echo: repaired\n', @@ -1384,9 +1508,9 @@ describe("the concurrent-recording cap", () => { .sort(); expect(live).toHaveLength(MAX_RECORDINGS); expect(live).toEqual([...names.filter((n) => n !== untouched), "overflow"].sort()); - expect(getRecordingSession(root, untouched)).toBeUndefined(); + expect(await getRecordingSession(root, untouched)).toBeUndefined(); // The oldest registration survived, because it was still being used. - expect(getRecordingSession(root, names[0])).toBeDefined(); + expect(await getRecordingSession(root, names[0])).toBeDefined(); // The survivors are still usable — eviction dropped one, not the table. await addEcho(root, names[0], "still-live"); expect(await readMarkers(root, names[0])).toEqual(["echo:touch", "echo:still-live"]); @@ -1425,8 +1549,8 @@ describe("the concurrent-recording cap", () => { // The recording that just appended survives; the victim is the one whose // last use really is the oldest. Without the stamp on land, rec-0 is the // one dropped here. - expect(getRecordingSession(root, "rec-0")).toBeDefined(); - expect(getRecordingSession(root, names[1])).toBeUndefined(); + expect(await getRecordingSession(root, "rec-0")).toBeDefined(); + expect(await getRecordingSession(root, names[1])).toBeUndefined(); // And it is still usable, not merely present. await addEcho(root, "rec-0", "after"); expect(await readMarkers(root, "rec-0")).toEqual(["tool:slow", "echo:after"]); @@ -1446,7 +1570,7 @@ describe("the concurrent-recording cap", () => { // under the running step. for (const name of names.slice(1)) await addEcho(root, name, "touch"); await start(root, "overflow"); - expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + expect(await getRecordingSession(root, "rec-0")).toBeUndefined(); gate.release(); const err = await captureFailure(appending); @@ -1507,7 +1631,7 @@ describe("the concurrent-recording cap", () => { // A 33rd recording overflows the cap and evicts the LRU — rec-0's key — // while the restart is parked with rec-0's live session already captured. await start(root, "overflow"); - expect(getRecordingSession(root, "rec-0")).toBeUndefined(); + expect(await getRecordingSession(root, "rec-0")).toBeUndefined(); held.open(); const res = await restarting; @@ -1643,6 +1767,6 @@ describe("finishing a recording whose YAML was hand-edited into an unrenderable expect(finished.steps).toBe(1); expect(finished.summary).toEqual(["1. tool: keyboard [cyclic args]"]); // The recording is properly closed, not left dangling by a thrown summary. - expect(getRecordingSession(root, "alpha")).toBeUndefined(); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); }); }); diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 8f468469e..259a81c15 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -2124,7 +2124,7 @@ describe("flow-execute", () => { {}, { name: "recording", project_root: tmpDir, executionPrerequisite: PREREQ } ); - const before = getRecordingSession(tmpDir, "recording"); + const before = await getRecordingSession(tmpDir, "recording"); expect(before).toBeDefined(); // Execute saved flows — neither should affect the active recording @@ -2133,7 +2133,7 @@ describe("flow-execute", () => { // The recording still points at the flow it was opened for, in its own // project — a replay elsewhere must not rebind name/root/file. - const after = getRecordingSession(tmpDir, "recording"); + const after = await getRecordingSession(tmpDir, "recording"); expect(after).toBe(before); expect(after).toMatchObject({ name: "recording", diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 51888d2a0..6a6ddfa4a 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -25,7 +25,7 @@ import { // ── serializeFlow ──────────────────────────────────────────────────── describe("serializeFlow", () => { - it("serializes an empty flow with prerequisite", () => { + it("serializes an empty flow with prerequisite", async () => { const flow: FlowFile = { executionPrerequisite: "App on home screen", steps: [], @@ -35,7 +35,7 @@ describe("serializeFlow", () => { expect(result).toContain("steps: []"); }); - it("serializes echo steps", () => { + it("serializes echo steps", async () => { const flow: FlowFile = { executionPrerequisite: "Fresh reload", steps: [{ kind: "echo", message: "Hello" }], @@ -44,7 +44,7 @@ describe("serializeFlow", () => { expect(result).toContain("- echo: Hello"); }); - it("serializes tool steps with args", () => { + it("serializes tool steps with args", async () => { const flow: FlowFile = { executionPrerequisite: "", steps: [{ kind: "tool", name: "tap", args: { x: 0.5, y: 0.3 } }], @@ -55,7 +55,7 @@ describe("serializeFlow", () => { expect(result).toContain(" y: 0.3"); }); - it("serializes tool steps with empty args (omits args key)", () => { + it("serializes tool steps with empty args (omits args key)", async () => { const flow: FlowFile = { executionPrerequisite: "", steps: [{ kind: "tool", name: "screenshot", args: {} }], @@ -65,7 +65,7 @@ describe("serializeFlow", () => { expect(result).not.toContain("args:"); }); - it("rejects gesture targets that cannot round-trip through the parser", () => { + it("rejects gesture targets that cannot round-trip through the parser", async () => { const serializeStep = (step: FlowFile["steps"][number]) => serializeFlow({ executionPrerequisite: "", steps: [step] }); @@ -85,19 +85,19 @@ describe("serializeFlow", () => { // ── describeSelector ───────────────────────────────────────────────── describe("describeSelector", () => { - it("spells identifier as id, the flow-YAML spelling", () => { + it("spells identifier as id, the flow-YAML spelling", async () => { expect(describeSelector({ identifier: "submit" })).toBe('id="submit"'); }); - it("renders a text selector", () => { + it("renders a text selector", async () => { expect(describeSelector({ text: "Login" })).toBe('text="Login"'); }); - it("drops the internal loose flag", () => { + it("drops the internal loose flag", async () => { expect(describeSelector({ text: "Login", loose: true })).toBe('text="Login"'); }); - it("joins multiple keys with spaces", () => { + it("joins multiple keys with spaces", async () => { expect(describeSelector({ text: "Login", role: "button" })).toBe('text="Login" role="button"'); }); }); @@ -105,27 +105,27 @@ describe("describeSelector", () => { // ── parseFlow ──────────────────────────────────────────────────────── describe("parseFlow", () => { - it("parses a flow with executionPrerequisite and echo steps", () => { + it("parses a flow with executionPrerequisite and echo steps", async () => { const content = "executionPrerequisite: App on home screen\nsteps:\n - echo: Hello\n"; const flow = parseFlow(content); expect(flow.executionPrerequisite).toBe("App on home screen"); expect(flow.steps).toEqual([{ kind: "echo", message: "Hello" }]); }); - it("parses tool entries with args", () => { + it("parses tool entries with args", async () => { const content = 'executionPrerequisite: ""\nsteps:\n - tool: tap\n args:\n x: 0.5\n y: 0.3\n'; const flow = parseFlow(content); expect(flow.steps).toEqual([{ kind: "tool", name: "tap", args: { x: 0.5, y: 0.3 } }]); }); - it("parses tool entries with no args", () => { + it("parses tool entries with no args", async () => { const content = 'executionPrerequisite: ""\nsteps:\n - tool: screenshot\n'; const flow = parseFlow(content); expect(flow.steps).toEqual([{ kind: "tool", name: "screenshot", args: {} }]); }); - it("parses a multi-step flow", () => { + it("parses a multi-step flow", async () => { const content = [ "executionPrerequisite: Settings open", "steps:", @@ -149,32 +149,32 @@ describe("parseFlow", () => { ]); }); - it("returns empty steps for empty content", () => { + it("returns empty steps for empty content", async () => { const flow = parseFlow(""); expect(flow.executionPrerequisite).toBe(""); expect(flow.steps).toEqual([]); }); - it("defaults executionPrerequisite to empty string when missing", () => { + it("defaults executionPrerequisite to empty string when missing", async () => { const content = "steps:\n - echo: Hello\n"; const flow = parseFlow(content); expect(flow.executionPrerequisite).toBe(""); expect(flow.steps).toEqual([{ kind: "echo", message: "Hello" }]); }); - it("throws on unrecognized entries", () => { + it("throws on unrecognized entries", async () => { const content = 'executionPrerequisite: ""\nsteps:\n - bogus: line\n'; expect(() => parseFlow(content)).toThrow("Unrecognized flow entry"); }); - it("renders a small unrecognized entry in full", () => { + it("renders a small unrecognized entry in full", async () => { // The common authoring error: a short mistyped step. The echo cap must // leave it untouched — seeing the whole entry is what makes it fixable. const content = 'executionPrerequisite: ""\nsteps:\n - bogus: line\n'; expect(() => parseFlow(content)).toThrow(': {"bogus":"line"}'); }); - it("caps the echoed entry so an oversized value cannot ride the diagnostic", () => { + it("caps the echoed entry so an oversized value cannot ride the diagnostic", async () => { // A mistyped run: path can point parseFlow at any in-project YAML file, // and this message flows verbatim to stdout and into agent context — so // the render must be bounded, and the tail of the value must not appear. @@ -191,11 +191,11 @@ describe("parseFlow", () => { expect(message.length).toBeLessThan(400); }); - it("throws when content is not an object with steps", () => { + it("throws when content is not an object with steps", async () => { expect(() => parseFlow("- echo: Hello\n")).toThrow("expected an object with a steps array"); }); - it("classifies a YAML syntax error as a validation failure with the parser's detail", () => { + it("classifies a YAML syntax error as a validation failure with the parser's detail", async () => { let thrown: unknown; try { parseFlow("steps: ][\n"); @@ -212,35 +212,35 @@ describe("parseFlow", () => { expect((thrown as Error).message).toContain("line 1"); }); - it("throws a validation error (not a TypeError) on a primitive step entry", () => { + it("throws a validation error (not a TypeError) on a primitive step entry", async () => { const content = 'executionPrerequisite: ""\nsteps:\n - tap\n'; expect(() => parseFlow(content)).toThrow("Unrecognized flow entry"); }); - it("throws a validation error on a null step entry", () => { + it("throws a validation error on a null step entry", async () => { const content = 'executionPrerequisite: ""\nsteps:\n - ~\n'; expect(() => parseFlow(content)).toThrow("Unrecognized flow entry"); }); - it("sugars a bare-string selector into a loose { text } for tap", () => { + it("sugars a bare-string selector into a loose { text } for tap", async () => { const flow = parseFlow("steps:\n - tap: Settings\n"); // Bare string ⇒ loose: resolves identifier-first, then falls back to text. expect(flow.steps).toEqual([{ kind: "tap", selector: { text: "Settings", loose: true } }]); }); - it("sugars a bare-string selector for type.into", () => { + it("sugars a bare-string selector for type.into", async () => { const flow = parseFlow('steps:\n - type: { into: email, text: "a@b.com" }\n'); expect(flow.steps).toEqual([ { kind: "type", into: { text: "email", loose: true }, text: "a@b.com" }, ]); }); - it("defaults type.submit to on (no submit key in the parsed model)", () => { + it("defaults type.submit to on (no submit key in the parsed model)", async () => { const flow = parseFlow('steps:\n - type: { into: email, text: "a@b.com" }\n'); expect(flow.steps[0]).not.toHaveProperty("submit"); }); - it("parses and round-trips an explicit type.submit: false opt-out", () => { + it("parses and round-trips an explicit type.submit: false opt-out", async () => { const flow = parseFlow('steps:\n - type: { into: email, text: "a@b.com", submit: false }\n'); expect(flow.steps).toEqual([ { kind: "type", into: { text: "email", loose: true }, text: "a@b.com", submit: false }, @@ -249,11 +249,11 @@ describe("parseFlow", () => { expect(parseFlow(serializeFlow(flow)).steps).toEqual(flow.steps); }); - it("rejects a non-boolean type.submit", () => { + it("rejects a non-boolean type.submit", async () => { expect(() => parseFlow('steps:\n - type: { into: email, text: "x", submit: 3 }\n')).toThrow(); }); - it("keeps an explicit { text } map strict (no loose fallback)", () => { + it("keeps an explicit { text } map strict (no loose fallback)", async () => { const flow = parseFlow("steps:\n - tap: { text: Settings }\n"); expect(flow.steps).toEqual([{ kind: "tap", selector: { text: "Settings" } }]); }); @@ -279,7 +279,7 @@ describe("parseFlow", () => { expect(parseFlow(yaml).steps).toEqual([expected]); }); - it("accepts a regex selector combined with id and role", () => { + it("accepts a regex selector combined with id and role", async () => { expect( parseFlow( "steps:\n - tap: { text: { matches: '^Order #\\d+$' }, id: order-row, role: button }\n" @@ -340,23 +340,23 @@ describe("parseFlow", () => { expect(() => parseFlow(yaml)).toThrow(`${where} \`matches\` is not a valid regular expression`); }); - it("parses the map form's `id` as the internal identifier field (strict)", () => { + it("parses the map form's `id` as the internal identifier field (strict)", async () => { const flow = parseFlow("steps:\n - tap: { id: submit-btn }\n"); expect(flow.steps).toEqual([{ kind: "tap", selector: { identifier: "submit-btn" } }]); }); - it("accepts `identifier` as a parse-only alias for `id`", () => { + it("accepts `identifier` as a parse-only alias for `id`", async () => { const flow = parseFlow("steps:\n - tap: { identifier: submit-btn }\n"); expect(flow.steps).toEqual([{ kind: "tap", selector: { identifier: "submit-btn" } }]); }); - it("rejects a selector map carrying both `id` and `identifier`", () => { + it("rejects a selector map carrying both `id` and `identifier`", async () => { expect(() => parseFlow("steps:\n - tap: { id: a, identifier: b }\n")).toThrow( /`id` or `identifier`.*not both/ ); }); - it("re-serializes an identifier-spelled flow with the `id` spelling", () => { + it("re-serializes an identifier-spelled flow with the `id` spelling", async () => { // Old files parse via the alias; the next write (appendStep re-serializes // the whole file) migrates them to the canonical `id` spelling. const yaml = serializeFlow(parseFlow("steps:\n - tap: { identifier: submit-btn }\n")); @@ -364,7 +364,7 @@ describe("parseFlow", () => { expect(yaml).not.toContain("identifier:"); }); - it("parses condition-as-key await/assert sugar (visible/exists/hidden)", () => { + it("parses condition-as-key await/assert sugar (visible/exists/hidden)", async () => { const flow = parseFlow( [ "steps:", @@ -380,7 +380,7 @@ describe("parseFlow", () => { ]); }); - it("parses the text sugar { in, contains } as a substring match", () => { + it("parses the text sugar { in, contains } as a substring match", async () => { const flow = parseFlow( 'steps:\n - assert: { text: { in: { id: counter }, contains: "Taps: 0" } }\n' ); @@ -395,7 +395,7 @@ describe("parseFlow", () => { ]); }); - it("parses the text sugar { in, equals } as an exact match", () => { + it("parses the text sugar { in, equals } as an exact match", async () => { const flow = parseFlow( 'steps:\n - assert: { text: { in: { id: counter }, equals: "Taps: 0" } }\n' ); @@ -410,13 +410,13 @@ describe("parseFlow", () => { ]); }); - it("rejects text sugar with both contains and equals", () => { + it("rejects text sugar with both contains and equals", async () => { expect(() => parseFlow("steps:\n - assert: { text: { in: counter, contains: a, equals: b } }\n") ).toThrow(/exactly one of `contains`, `equals`, or `matches`/); }); - it("rejects the explicit { condition, selector, expectedText } form (sugar only)", () => { + it("rejects the explicit { condition, selector, expectedText } form (sugar only)", async () => { expect(() => parseFlow( [ @@ -430,25 +430,25 @@ describe("parseFlow", () => { ).toThrow(/exactly one condition key/); }); - it("rejects an await/assert body with no condition key", () => { + it("rejects an await/assert body with no condition key", async () => { expect(() => parseFlow("steps:\n - assert: { selector: foo }\n")).toThrow( /exactly one condition key/ ); }); - it("rejects text sugar with neither contains nor equals", () => { + it("rejects text sugar with neither contains nor equals", async () => { expect(() => parseFlow("steps:\n - assert: { text: { in: counter } }\n")).toThrow( /exactly one of `contains`, `equals`, or `matches`/ ); }); - it("rejects text sugar with an empty contains", () => { + it("rejects text sugar with an empty contains", async () => { expect(() => parseFlow('steps:\n - assert: { text: { in: counter, contains: "" } }\n') ).toThrow(/non-empty `contains`/); }); - it("serializes await/assert with the condition-as-key sugar (no condition: field)", () => { + it("serializes await/assert with the condition-as-key sugar (no condition: field)", async () => { const yaml = serializeFlow({ executionPrerequisite: "", steps: [ @@ -499,7 +499,7 @@ describe("parseFlow", () => { expect(parseFlow(yaml).steps).toEqual([step]); }); - it("roundtrips the sugared step kinds through YAML", () => { + it("roundtrips the sugared step kinds through YAML", async () => { // The spelling carries the loose bit exactly both ways: a LOOSE text-only // selector serializes to a bare string (which parses back loose); a strict // `{ text }` keeps the map form (which parses back strict). Identifier @@ -546,7 +546,7 @@ describe("parseFlow", () => { expect(parseFlow(serializeFlow(flow)).steps).toEqual(flow.steps); }); - it("keeps a strict { text } selector strict across repeated round-trips (never collapsed to a bare loose string)", () => { + it("keeps a strict { text } selector strict across repeated round-trips (never collapsed to a bare loose string)", async () => { // The recorder derives strict `{ text }` selectors, and every recorded step // re-reads and re-writes the whole file (appendStep) — so a single lossy // serialization would silently promote them to loose, sending them through @@ -563,7 +563,7 @@ describe("parseFlow", () => { expect(parseFlow(serializeFlow(reparsed)).steps).toEqual(flow.steps); }); - it("sugars a bare-string scroll-to target and keeps the within map", () => { + it("sugars a bare-string scroll-to target and keeps the within map", async () => { const flow = parseFlow( ["steps:", " - scroll-to: { target: Account, direction: down }"].join("\n") ); @@ -572,17 +572,17 @@ describe("parseFlow", () => { ]); }); - it("parses a bare-number wait as milliseconds", () => { + it("parses a bare-number wait as milliseconds", async () => { const flow = parseFlow("steps:\n - wait: 750\n"); expect(flow.steps).toEqual([{ kind: "wait", ms: 750 }]); }); - it("rejects a wait that is not a non-negative number", () => { + it("rejects a wait that is not a non-negative number", async () => { expect(() => parseFlow("steps:\n - wait: soon\n")).toThrow("wait needs a non-negative number"); expect(() => parseFlow("steps:\n - wait: -5\n")).toThrow("wait needs a non-negative number"); }); - it("parses an await timeout in milliseconds", () => { + it("parses an await timeout in milliseconds", async () => { const flow = parseFlow("steps:\n - await: { visible: Account, timeout: 10000 }\n"); expect(flow.steps).toEqual([ { @@ -594,7 +594,7 @@ describe("parseFlow", () => { ]); }); - it("rejects an await timeout that is not a positive finite number", () => { + it("rejects an await timeout that is not a positive finite number", async () => { // `.inf`, `.nan`, and an overflowing literal all parse to a typeof-number // value; letting Infinity through would make the runner's poll deadline // unreachable (an unbounded await). @@ -605,7 +605,7 @@ describe("parseFlow", () => { } }); - it("rejects a timeout on an assert step (an assert is an immediate check)", () => { + it("rejects a timeout on an assert step (an assert is an immediate check)", async () => { // The internal assert step has no timeout field, so a YAML `timeout` used // to be silently dropped; reject it loudly instead — a check that needs // time to become true is a wait, spelled `await`. @@ -617,27 +617,27 @@ describe("parseFlow", () => { ).toThrow(/assert has no timeout/); }); - it("rejects a scroll-to with an invalid direction", () => { + it("rejects a scroll-to with an invalid direction", async () => { expect(() => parseFlow("steps:\n - scroll-to: { target: Account, direction: sideways }\n") ).toThrow("scroll-to direction must be one of"); }); - it("defaults scroll-to direction to down", () => { + it("defaults scroll-to direction to down", async () => { const flow = parseFlow("steps:\n - scroll-to: { target: Account }\n"); expect(flow.steps).toEqual([ { kind: "scroll-to", target: { text: "Account", loose: true }, direction: "down" }, ]); }); - it("parses a bare-string scroll-to as a down-scroll to that target", () => { + it("parses a bare-string scroll-to as a down-scroll to that target", async () => { const flow = parseFlow("steps:\n - scroll-to: Account\n"); expect(flow.steps).toEqual([ { kind: "scroll-to", target: { text: "Account", loose: true }, direction: "down" }, ]); }); - it("serializes the default scroll-to back to the bare-string sugar", () => { + it("serializes the default scroll-to back to the bare-string sugar", async () => { const steps = [ { kind: "scroll-to", target: { text: "Account", loose: true }, direction: "down" }, ] as FlowFile["steps"]; @@ -646,12 +646,12 @@ describe("parseFlow", () => { expect(parseFlow(yaml).steps).toEqual(steps); }); - it("parses a bare-string snapshot as its name", () => { + it("parses a bare-string snapshot as its name", async () => { const flow = parseFlow("steps:\n - snapshot: home\n"); expect(flow.steps).toEqual([{ kind: "snapshot", name: "home" }]); }); - it("serializes a name-only snapshot as a bare string, keeps the map with maxMismatch", () => { + it("serializes a name-only snapshot as a bare string, keeps the map with maxMismatch", async () => { const steps = [ { kind: "snapshot", name: "home" }, { kind: "snapshot", name: "cart", maxMismatch: 1.5 }, @@ -662,16 +662,16 @@ describe("parseFlow", () => { expect(parseFlow(yaml).steps).toEqual(steps); }); - it("rejects a snapshot name that is not path-safe", () => { + it("rejects a snapshot name that is not path-safe", async () => { expect(() => parseFlow("steps:\n - snapshot: ../evil\n")).toThrow(/must match/); }); - it("accepts a string-number maxMismatch", () => { + it("accepts a string-number maxMismatch", async () => { const flow = parseFlow('steps:\n - snapshot: { name: home, maxMismatch: "1.5" }\n'); expect(flow.steps).toEqual([{ kind: "snapshot", name: "home", maxMismatch: 1.5 }]); }); - it("rejects a non-numeric, negative, or out-of-range maxMismatch", () => { + it("rejects a non-numeric, negative, or out-of-range maxMismatch", async () => { for (const bad of ['"5%"', "-1", "101", ".nan"]) { expect(() => parseFlow(`steps:\n - snapshot: { name: home, maxMismatch: ${bad} }\n`) @@ -679,7 +679,7 @@ describe("parseFlow", () => { } }); - it("parses snapshot cropOn as a selector (bare-string loose, map strict)", () => { + it("parses snapshot cropOn as a selector (bare-string loose, map strict)", async () => { const flow = parseFlow( "steps:\n" + " - snapshot: { name: home, cropOn: Header }\n" + @@ -691,7 +691,7 @@ describe("parseFlow", () => { ]); }); - it("serializes snapshot cropOn in the map form and round-trips", () => { + it("serializes snapshot cropOn in the map form and round-trips", async () => { const steps = [ { kind: "snapshot", name: "home", cropOn: { text: "Header", loose: true } }, { kind: "snapshot", name: "cart", maxMismatch: 1.5, cropOn: { identifier: "cart-total" } }, @@ -701,13 +701,13 @@ describe("parseFlow", () => { expect(parseFlow(yaml).steps).toEqual(steps); }); - it("rejects a point-form cropOn — a point has no extent to crop to", () => { + it("rejects a point-form cropOn — a point has no extent to crop to", async () => { expect(() => parseFlow("steps:\n - snapshot: { name: home, cropOn: { x: 0.5, y: 0.5 } }\n") ).toThrow(/snapshot\.cropOn: selector has unknown keys `x`, `y`/); }); - it("rejects a tap body mixing a selector with coordinates", () => { + it("rejects a tap body mixing a selector with coordinates", async () => { for (const key of ["id", "identifier"]) { expect(() => parseFlow(`steps:\n - tap: { ${key}: box, x: 0.5, y: 0.5 }\n`)).toThrow( "tap takes a selector or x/y coordinates, not both" @@ -715,7 +715,7 @@ describe("parseFlow", () => { } }); - it("rejects a coordinate tap with a missing or non-numeric x/y", () => { + it("rejects a coordinate tap with a missing or non-numeric x/y", async () => { expect(() => parseFlow("steps:\n - tap: { x: 0.5 }\n")).toThrow( "tap: a coordinate target needs numeric x and y" ); @@ -724,7 +724,7 @@ describe("parseFlow", () => { ); }); - it("round-trips free-text values exactly, including whitespace-only lines", () => { + it("round-trips free-text values exactly, including whitespace-only lines", async () => { // The parser stores every free-text field verbatim — `type.text`, `echo`, // await/assert `contains`/`equals`, and `executionPrerequisite` are never // trimmed — so serialization must be byte-exact too. Default yamlStringify @@ -767,7 +767,7 @@ describe("parseFlow", () => { } }); - it("never serializes a whitespace-only-line value as a block scalar", () => { + it("never serializes a whitespace-only-line value as a block scalar", async () => { const steps = [{ kind: "echo", message: "step one \n \ndone" }] as FlowFile["steps"]; const yaml = serializeFlow({ executionPrerequisite: "", steps }); // Block (|) and folded (>) scalars are not round-trip-safe for this shape; @@ -783,37 +783,37 @@ describe("parseFlow", () => { // surface later as a misleading runtime failure (wrong scroll direction, // lost submit opt-out, lost timeout, lost snapshot tolerance). describe("unknown option keys are rejected at parse time", () => { - it("rejects a misspelled scroll-to direction key with a suggestion", () => { + it("rejects a misspelled scroll-to direction key with a suggestion", async () => { expect(() => parseFlow("steps:\n - scroll-to: { target: Order-1234, directon: up }\n") ).toThrow(/scroll-to has unknown key `directon` \(did you mean `direction`\?\)/); }); - it("rejects a misspelled type.submit key with a suggestion", () => { + it("rejects a misspelled type.submit key with a suggestion", async () => { expect(() => parseFlow('steps:\n - type: { into: email, text: "a@b.com", sumbit: false }\n') ).toThrow(/type has unknown key `sumbit` \(did you mean `submit`\?\)/); }); - it("rejects a misspelled await.timeout key with a suggestion", () => { + it("rejects a misspelled await.timeout key with a suggestion", async () => { expect(() => parseFlow("steps:\n - await: { visible: Account, timeut: 10000 }\n")).toThrow( /await has unknown key `timeut` \(did you mean `timeout`\?\)/ ); }); - it("rejects a misspelled snapshot.maxMismatch key with a suggestion", () => { + it("rejects a misspelled snapshot.maxMismatch key with a suggestion", async () => { expect(() => parseFlow("steps:\n - snapshot: { name: home, maxMissmatch: 1.5 }\n")).toThrow( /snapshot has unknown key `maxMissmatch` \(did you mean `maxMismatch`\?\)/ ); }); - it("rejects a miscased snapshot.cropOn key with a suggestion", () => { + it("rejects a miscased snapshot.cropOn key with a suggestion", async () => { expect(() => parseFlow("steps:\n - snapshot: { name: home, cropon: Header }\n")).toThrow( /snapshot has unknown key `cropon` \(did you mean `cropOn`\?\)/ ); }); - it("rejects an unknown key on a selector map", () => { + it("rejects an unknown key on a selector map", async () => { expect(() => parseFlow("steps:\n - tap: { text: Save, roel: button }\n")).toThrow( /tap: selector has unknown key `roel` \(did you mean `role`\?\)/ ); @@ -827,25 +827,25 @@ describe("parseFlow", () => { ); }); - it("rejects an unknown key without a suggestion when nothing is close", () => { + it("rejects an unknown key without a suggestion when nothing is close", async () => { expect(() => parseFlow("steps:\n - scroll-to: { target: Row, sideways: true }\n")).toThrow( /scroll-to has unknown key `sideways` — allowed keys: target, direction, within/ ); }); - it("rejects an unknown key in an await/assert text body", () => { + it("rejects an unknown key in an await/assert text body", async () => { expect(() => parseFlow('steps:\n - assert: { text: { in: counter, contians: "Taps: 0" } }\n') ).toThrow(/assert.text has unknown key `contians` \(did you mean `contains`\?\)/); }); - it("rejects a stray key on a coordinate tap", () => { + it("rejects a stray key on a coordinate tap", async () => { expect(() => parseFlow("steps:\n - tap: { x: 0.5, y: 0.5, why: 0.6 }\n")).toThrow( /tap: a coordinate target takes only \{ x, y \}/ ); }); - it("rejects an unknown key in a launch map and its chromium value", () => { + it("rejects an unknown key in a launch map and its chromium value", async () => { expect(() => parseFlow("steps:\n - launch: { amdroid: com.acme.app }\n")).toThrow( /launch has unknown key `amdroid` \(did you mean `android`\?\)/ ); @@ -854,7 +854,7 @@ describe("parseFlow", () => { ).toThrow(/launch.chromium has unknown key `arg` \(did you mean `args`\?\)/); }); - it("rejects a step-level sibling key (options belong inside the directive value)", () => { + it("rejects a step-level sibling key (options belong inside the directive value)", async () => { expect(() => parseFlow("steps:\n - await: { visible: Account }\n timeout: 5000\n") ).toThrow( @@ -862,26 +862,26 @@ describe("parseFlow", () => { ); }); - it("rejects a step carrying two directive keys", () => { + it("rejects a step carrying two directive keys", async () => { expect(() => parseFlow("steps:\n - echo: hi\n tap: Save\n")).toThrow( /a step takes exactly one directive key, found `echo`, `tap`/ ); }); - it("suggests the directive key for a misspelled step kind", () => { + it("suggests the directive key for a misspelled step kind", async () => { expect(() => parseFlow("steps:\n - snapshoot: home\n")).toThrow( /unrecognized step kind \(did you mean `snapshot`\?\)/ ); }); - it("rejects an unknown top-level flow file key", () => { + it("rejects an unknown top-level flow file key", async () => { expect(() => parseFlow("executionPrerequisit: Settings open\nsteps:\n - echo: hi\n") ).toThrow(/unknown key `executionPrerequisit` \(did you mean `executionPrerequisite`\?\)/); }); }); - it("roundtrips: serialize then parse", () => { + it("roundtrips: serialize then parse", async () => { const flow: FlowFile = { executionPrerequisite: "App freshly loaded on home screen", steps: [ @@ -899,26 +899,26 @@ describe("parseFlow", () => { // ── chromium launch (app path) ─────────────────────────────────────── describe("chromium launch parsing", () => { - it("parses a chromium launch with a bare-string app path", () => { + it("parses a chromium launch with a bare-string app path", async () => { const flow = parseFlow("steps:\n - launch: { chromium: ./app }\n"); expect(flow.steps).toEqual([{ kind: "launch", app: { chromium: "./app" } }]); }); - it("parses a chromium launch with a { path, args } map", () => { + it("parses a chromium launch with a { path, args } map", async () => { const flow = parseFlow("steps:\n - launch: { chromium: { path: ./app, args: [--e2e] } }\n"); expect(flow.steps).toEqual([ { kind: "launch", app: { chromium: { path: "./app", args: ["--e2e"] } } }, ]); }); - it("parses a mixed per-platform launch (ios id + chromium path)", () => { + it("parses a mixed per-platform launch (ios id + chromium path)", async () => { const flow = parseFlow("steps:\n - launch: { ios: com.acme.app, chromium: ./app }\n"); expect(flow.steps).toEqual([ { kind: "launch", app: { ios: "com.acme.app", chromium: "./app" } }, ]); }); - it("round-trips a chromium { path, args } launch through YAML", () => { + it("round-trips a chromium { path, args } launch through YAML", async () => { const flow: FlowFile = { executionPrerequisite: "", steps: [ @@ -928,13 +928,13 @@ describe("chromium launch parsing", () => { expect(parseFlow(serializeFlow(flow)).steps).toEqual(flow.steps); }); - it("rejects a chromium map with no path", () => { + it("rejects a chromium map with no path", async () => { expect(() => parseFlow("steps:\n - launch: { chromium: { args: [--e2e] } }\n")).toThrow( /launch needs/ ); }); - it("rejects a chromium map with non-string args", () => { + it("rejects a chromium map with non-string args", async () => { expect(() => parseFlow("steps:\n - launch: { chromium: { path: ./app, args: [1, 2] } }\n") ).toThrow(/launch needs/); @@ -942,27 +942,27 @@ describe("chromium launch parsing", () => { }); describe("chromiumLaunchSpec", () => { - it("reads a bare-string launch as the app path", () => { + it("reads a bare-string launch as the app path", async () => { expect(chromiumLaunchSpec("./app")).toEqual({ path: "./app" }); }); - it("reads a chromium string value as the path", () => { + it("reads a chromium string value as the path", async () => { expect(chromiumLaunchSpec({ chromium: "./app" })).toEqual({ path: "./app" }); }); - it("reads a chromium { path, args } value", () => { + it("reads a chromium { path, args } value", async () => { expect(chromiumLaunchSpec({ chromium: { path: "./app", args: ["--e2e"] } })).toEqual({ path: "./app", args: ["--e2e"], }); }); - it("returns null when no chromium target is declared", () => { + it("returns null when no chromium target is declared", async () => { expect(chromiumLaunchSpec({ ios: "com.acme.app" })).toBeNull(); expect(chromiumLaunchSpec(undefined)).toBeNull(); }); - it("appIdForPlatform returns the chromium path (the runner's declared-target guard)", () => { + it("appIdForPlatform returns the chromium path (the runner's declared-target guard)", async () => { expect(appIdForPlatform({ chromium: { path: "./app", args: ["--e2e"] } }, "chromium")).toBe( "./app" ); @@ -974,13 +974,13 @@ describe("chromiumLaunchSpec", () => { // ── native shorthand ───────────────────────────────────────────────── describe("native launch shorthand", () => { - it("parses a native-only launch and round-trips it", () => { + it("parses a native-only launch and round-trips it", async () => { const flow = parseFlow("steps:\n - launch: { native: com.acme.app }\n"); expect(flow.steps).toEqual([{ kind: "launch", app: { native: "com.acme.app" } }]); expect(parseFlow(serializeFlow(flow)).steps).toEqual(flow.steps); }); - it("parses native alongside a per-platform override and a chromium path", () => { + it("parses native alongside a per-platform override and a chromium path", async () => { const flow = parseFlow( "steps:\n - launch: { native: com.acme.app, android: com.acme.app.debug, chromium: ./app }\n" ); @@ -992,11 +992,11 @@ describe("native launch shorthand", () => { ]); }); - it("rejects an empty native id", () => { + it("rejects an empty native id", async () => { expect(() => parseFlow('steps:\n - launch: { native: "" }\n')).toThrow(/launch needs/); }); - it("appIdForPlatform falls back to native for installed platforms, override wins", () => { + it("appIdForPlatform falls back to native for installed platforms, override wins", async () => { const app = { native: "com.acme.app", android: "com.acme.app.debug" }; // native fills in for platforms without a specific key… expect(appIdForPlatform(app, "ios")).toBe("com.acme.app"); @@ -1005,7 +1005,7 @@ describe("native launch shorthand", () => { expect(appIdForPlatform(app, "android")).toBe("com.acme.app.debug"); }); - it("native never applies to chromium (chromium takes a path, not an id)", () => { + it("native never applies to chromium (chromium takes a path, not an id)", async () => { expect(appIdForPlatform({ native: "com.acme.app" }, "chromium")).toBeNull(); expect(chromiumLaunchSpec({ native: "com.acme.app" })).toBeNull(); }); @@ -1035,43 +1035,43 @@ describe("recording sessions", () => { flow, }); - it("throws when the key has no recording", () => { - expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + it("throws when the key has no recording", async () => { + await expect(requireRecordingSession("/tmp/proj-a", "my-flow")).rejects.toThrow( /No active recording for flow "my-flow"/ ); }); - it("classifies the not-found throw as FLOW_NO_ACTIVE_RECORDING", () => { + it("classifies the not-found throw as FLOW_NO_ACTIVE_RECORDING", async () => { let caught: unknown; try { - requireRecordingSession("/tmp/proj-a", "my-flow"); + await requireRecordingSession("/tmp/proj-a", "my-flow"); } catch (err) { caught = err; } expect(getFailureSignal(caught)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); }); - it("names the asked-for key and this project's live recordings in the not-found message", () => { + it("names the asked-for key and this project's live recordings in the not-found message", async () => { // With concurrent recordings the usual cause is a typo or the wrong // project_root; the agent can only self-correct if it sees the live keys. - start("/tmp/proj-a", "checkout"); - start("/tmp/proj-b", "login"); - expect(() => requireRecordingSession("/tmp/proj-a", "chekout")).toThrow( + await start("/tmp/proj-a", "checkout"); + await start("/tmp/proj-b", "login"); + await expect(requireRecordingSession("/tmp/proj-a", "chekout")).rejects.toThrow( /No active recording for flow "chekout" in \/tmp\/proj-a\./ ); - expect(() => requireRecordingSession("/tmp/proj-a", "chekout")).toThrow( + await expect(requireRecordingSession("/tmp/proj-a", "chekout")).rejects.toThrow( /Active recordings: "checkout" \(plus 1 in other projects\)\./ ); }); - it("counts other projects' recordings without naming them", () => { + it("counts other projects' recordings without naming them", async () => { // A tool-server bound beyond loopback serves unrelated callers; another // project's flow names and absolute paths are not this caller's to see. - start("/tmp/proj-b", "login"); - start("/tmp/proj-c", "secret-onboarding"); - const message = (() => { + await start("/tmp/proj-b", "login"); + await start("/tmp/proj-c", "secret-onboarding"); + const message = await (async () => { try { - requireRecordingSession("/tmp/proj-a", "my-flow"); + await requireRecordingSession("/tmp/proj-a", "my-flow"); } catch (err) { return (err as Error).message; } @@ -1086,17 +1086,17 @@ describe("recording sessions", () => { expect(message).not.toContain("/tmp/proj-c"); }); - it("treats a differently-spelled but identical root as THIS project", () => { + it("treats a differently-spelled but identical root as THIS project", async () => { // The partition compares path.join-normalized flows dirs, not raw strings. // A caller that spells its own root with a trailing slash must still be // shown its own live recordings — a strict === would answer "none in this // project (plus 1 in other projects)", degrading the message in exactly the // wrong-project_root case it exists to diagnose. Every other test here // spells both sides identically, so only this one separates the two. - start("/tmp/proj-a", "checkout"); - const message = (() => { + await start("/tmp/proj-a", "checkout"); + const message = await (async () => { try { - requireRecordingSession("/tmp/proj-a/", "chekout"); + await requireRecordingSession("/tmp/proj-a/", "chekout"); } catch (err) { return (err as Error).message; } @@ -1106,15 +1106,15 @@ describe("recording sessions", () => { expect(message).not.toContain("other projects"); }); - it("does not tell the agent to just call flow-start-recording", () => { + it("does not tell the agent to just call flow-start-recording", async () => { // This message is reached for a key that was never started, but equally for // one that was finished, superseded, or dropped by the concurrency cap — // and in those cases the flow file on disk is fully populated. Naming // flow-start-recording as the fix destroys it, because it truncates // unconditionally and reports no `restarted` when no session was replaced. - const message = (() => { + const message = await (async () => { try { - requireRecordingSession("/tmp/proj-a", "finished-earlier"); + await requireRecordingSession("/tmp/proj-a", "finished-earlier"); } catch (err) { return (err as Error).message; } @@ -1125,74 +1125,80 @@ describe("recording sessions", () => { expect(message).not.toMatch(/Call flow-start-recording first/); }); - it('reports "none in this project" when nothing is being recorded', () => { - expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + it('reports "none in this project" when nothing is being recorded', async () => { + await expect(requireRecordingSession("/tmp/proj-a", "my-flow")).rejects.toThrow( /Active recordings: none in this project\./ ); }); - it("returns the session that was started for that key", () => { - start("/tmp/proj-a", "my-flow"); - const session = requireRecordingSession("/tmp/proj-a", "my-flow"); + it("returns the session that was started for that key", async () => { + await start("/tmp/proj-a", "my-flow"); + const session = await requireRecordingSession("/tmp/proj-a", "my-flow"); expect(session.name).toBe("my-flow"); expect(session.projectRoot).toBe("/tmp/proj-a"); expect(session.persist).toBe("host"); expect(session.filePath).toBe(getFlowPath("/tmp/proj-a", "my-flow")); }); - it("getRecordingSession returns undefined for a key with no recording", () => { - expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); + it("getRecordingSession returns undefined for a key with no recording", async () => { + expect(await getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); }); - it("getRecordingSession returns the live session", () => { - start("/tmp/proj-a", "my-flow"); - expect(getRecordingSession("/tmp/proj-a", "my-flow")?.name).toBe("my-flow"); + it("getRecordingSession returns the live session", async () => { + await start("/tmp/proj-a", "my-flow"); + expect((await getRecordingSession("/tmp/proj-a", "my-flow"))?.name).toBe("my-flow"); }); - it("clearRecordingSession removes only that key", () => { - start("/tmp/proj-a", "my-flow"); - start("/tmp/proj-a", "other-flow"); - clearRecordingSession("/tmp/proj-a", "my-flow"); - expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); - expect(() => requireRecordingSession("/tmp/proj-a", "my-flow")).toThrow( + it("clearRecordingSession removes only that key", async () => { + await start("/tmp/proj-a", "my-flow"); + await start("/tmp/proj-a", "other-flow"); + await clearRecordingSession("/tmp/proj-a", "my-flow"); + expect(await getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); + await expect(requireRecordingSession("/tmp/proj-a", "my-flow")).rejects.toThrow( /No active recording for flow "my-flow"/ ); // The unrelated recording is untouched. - expect(requireRecordingSession("/tmp/proj-a", "other-flow").name).toBe("other-flow"); + expect((await requireRecordingSession("/tmp/proj-a", "other-flow")).name).toBe("other-flow"); }); - it("keeps same-named recordings under different project roots independent", () => { - start("/tmp/proj-a", "my-flow", { executionPrerequisite: "A", steps: [] }); - start("/tmp/proj-b", "my-flow", { executionPrerequisite: "B", steps: [] }); - expect(requireRecordingSession("/tmp/proj-a", "my-flow").flow.executionPrerequisite).toBe("A"); - expect(requireRecordingSession("/tmp/proj-b", "my-flow").flow.executionPrerequisite).toBe("B"); + it("keeps same-named recordings under different project roots independent", async () => { + await start("/tmp/proj-a", "my-flow", { executionPrerequisite: "A", steps: [] }); + await start("/tmp/proj-b", "my-flow", { executionPrerequisite: "B", steps: [] }); + expect( + (await requireRecordingSession("/tmp/proj-a", "my-flow")).flow.executionPrerequisite + ).toBe("A"); + expect( + (await requireRecordingSession("/tmp/proj-b", "my-flow")).flow.executionPrerequisite + ).toBe("B"); // Finishing one leaves the other recording. - clearRecordingSession("/tmp/proj-a", "my-flow"); - expect(getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); - expect(requireRecordingSession("/tmp/proj-b", "my-flow").flow.executionPrerequisite).toBe("B"); + await clearRecordingSession("/tmp/proj-a", "my-flow"); + expect(await getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); + expect( + (await requireRecordingSession("/tmp/proj-b", "my-flow")).flow.executionPrerequisite + ).toBe("B"); }); - it("returns null when starting a recording on a free key", () => { - expect(start("/tmp/proj-a", "my-flow")).toBeNull(); + it("returns null when starting a recording on a free key", async () => { + expect(await start("/tmp/proj-a", "my-flow")).toBeNull(); // A second, unrelated recording is the common concurrent case — not a replace. - expect(start("/tmp/proj-a", "other-flow")).toBeNull(); - expect(start("/tmp/proj-b", "my-flow")).toBeNull(); + expect(await start("/tmp/proj-a", "other-flow")).toBeNull(); + expect(await start("/tmp/proj-b", "my-flow")).toBeNull(); }); - it("returns the replaced session when re-recording the same key", () => { - start("/tmp/proj-a", "my-flow", { executionPrerequisite: "first take", steps: [] }); - const replaced = start("/tmp/proj-a", "my-flow", { + it("returns the replaced session when re-recording the same key", async () => { + await start("/tmp/proj-a", "my-flow", { executionPrerequisite: "first take", steps: [] }); + const replaced = await start("/tmp/proj-a", "my-flow", { executionPrerequisite: "second take", steps: [], }); expect(replaced?.flow.executionPrerequisite).toBe("first take"); // The later take wins — one key, one writer. - expect(requireRecordingSession("/tmp/proj-a", "my-flow").flow.executionPrerequisite).toBe( - "second take" - ); + expect( + (await requireRecordingSession("/tmp/proj-a", "my-flow")).flow.executionPrerequisite + ).toBe("second take"); }); - it("evicts the least recently USED recording, not the oldest one", () => { + it("evicts the least recently USED recording, not the oldest one", async () => { // The cap is a leak backstop, but which entry it drops matters: evicting a // recording an agent is actively using would strand its steps. Fill past // the cap, touching the first-registered key just before the overflow — it @@ -1204,11 +1210,11 @@ describe("recording sessions", () => { // holds when this file runs alone but not under full-suite load. The // counter's tie-freedom is argued at `touch()` rather than pinned here. const cap = MAX_RECORDINGS; - for (let i = 0; i < cap; i++) start("/tmp/proj-a", `flow-${i}`); + for (let i = 0; i < cap; i++) await start("/tmp/proj-a", `flow-${i}`); expect(listActiveRecordings()).toHaveLength(cap); - requireRecordingSession("/tmp/proj-a", "flow-0"); // now most-recently-used - start("/tmp/proj-a", "overflow"); + await requireRecordingSession("/tmp/proj-a", "flow-0"); // now most-recently-used + await start("/tmp/proj-a", "overflow"); const live = new Set(listActiveRecordings().map((r) => r.name)); expect(live.size).toBe(cap); @@ -1217,18 +1223,18 @@ describe("recording sessions", () => { expect(live.has("overflow")).toBe(true); }); - it("listActiveRecordings reflects what is live", () => { + it("listActiveRecordings reflects what is live", async () => { expect(listActiveRecordings()).toEqual([]); - start("/tmp/proj-a", "my-flow", { + await start("/tmp/proj-a", "my-flow", { executionPrerequisite: "", steps: [{ kind: "echo", message: "hi" }], }); - start("/tmp/proj-b", "my-flow"); + await start("/tmp/proj-b", "my-flow"); expect(listActiveRecordings()).toEqual([ { name: "my-flow", projectRoot: "/tmp/proj-a", steps: 1 }, { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, ]); - clearRecordingSession("/tmp/proj-a", "my-flow"); + await clearRecordingSession("/tmp/proj-a", "my-flow"); expect(listActiveRecordings()).toEqual([ { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, ]); @@ -1236,10 +1242,10 @@ describe("recording sessions", () => { expect(listActiveRecordings()).toEqual([]); }); - it("keys a session by the normalized flow path, so a trailing slash rejoins it", () => { - start("/tmp/proj-a", "my-flow"); - expect(requireRecordingSession("/tmp/proj-a/", "my-flow").name).toBe("my-flow"); - expect(start("/tmp/proj-a/", "my-flow")).not.toBeNull(); + it("keys a session by the normalized flow path, so a trailing slash rejoins it", async () => { + await start("/tmp/proj-a", "my-flow"); + expect((await requireRecordingSession("/tmp/proj-a/", "my-flow")).name).toBe("my-flow"); + expect(await start("/tmp/proj-a/", "my-flow")).not.toBeNull(); expect(listActiveRecordings()).toHaveLength(1); }); }); @@ -1251,35 +1257,35 @@ describe("getFlowPath name validation", () => { // shared state, so two callers naming two projects can never collide. const root = "/tmp/argent-flow-name-test"; - it("accepts plain alphanumeric names", () => { + it("accepts plain alphanumeric names", async () => { expect(getFlowPath(root, "my-flow_1")).toBe( path.join(root, ".argent", "flows", "my-flow_1.yaml") ); }); - it("normalizes a trailing slash on the project root", () => { + it("normalizes a trailing slash on the project root", async () => { // The flow path doubles as the recording-session key: a trailing slash must // not mint a second identity for the same file. expect(getFlowPath("/tmp/x/", "f")).toBe(getFlowPath("/tmp/x", "f")); }); - it("rejects path-traversal segments", () => { + it("rejects path-traversal segments", async () => { expect(() => getFlowPath(root, "../../etc/passwd")).toThrow(/Invalid flow name/); expect(() => getFlowPath(root, "../foo")).toThrow(/Invalid flow name/); }); - it("rejects path separators", () => { + it("rejects path separators", async () => { expect(() => getFlowPath(root, "foo/bar")).toThrow(/Invalid flow name/); expect(() => getFlowPath(root, "/abs/path")).toThrow(/Invalid flow name/); }); - it("rejects names with spaces or shell metacharacters", () => { + it("rejects names with spaces or shell metacharacters", async () => { expect(() => getFlowPath(root, "foo bar")).toThrow(/Invalid flow name/); expect(() => getFlowPath(root, "foo;bar")).toThrow(/Invalid flow name/); expect(() => getFlowPath(root, "foo$(id)")).toThrow(/Invalid flow name/); }); - it("rejects empty names", () => { + it("rejects empty names", async () => { expect(() => getFlowPath(root, "")).toThrow(/Invalid flow name/); }); }); @@ -1287,18 +1293,18 @@ describe("getFlowPath name validation", () => { // PR #194 follow-up C: project_root must be absolute AND free of ".." // segments (path.join collapses ".." and would relocate the flows dir). describe("assertValidProjectRoot validation", () => { - it("rejects a relative project_root", () => { + it("rejects a relative project_root", async () => { expect(() => assertValidProjectRoot("relative/path")).toThrow(/absolute path/); }); - it('rejects an absolute project_root containing ".." segments', () => { + it('rejects an absolute project_root containing ".." segments', async () => { expect(() => assertValidProjectRoot("/a/../../../etc")).toThrow(/must not contain "\.\."/); expect(() => assertValidProjectRoot("/home/user/../../root")).toThrow( /must not contain "\.\."/ ); }); - it("accepts a clean absolute project_root", () => { + it("accepts a clean absolute project_root", async () => { expect(() => assertValidProjectRoot("/tmp/argent-pr194-c-test")).not.toThrow(); }); }); @@ -1306,7 +1312,7 @@ describe("assertValidProjectRoot validation", () => { // ── within (descendant) selector scoping ───────────────────────────── describe("within selector scoping", () => { - it("parses a within scope on a tap selector", () => { + it("parses a within scope on a tap selector", async () => { const flow = parseFlow("steps:\n - tap: { text: Delete, within: { id: profile-card } }\n"); expect(flow.steps).toEqual([ { @@ -1316,7 +1322,7 @@ describe("within selector scoping", () => { ]); }); - it("a bare-string within stays loose (identifier-first, then text)", () => { + it("a bare-string within stays loose (identifier-first, then text)", async () => { const flow = parseFlow("steps:\n - tap: { text: Delete, within: profile-card }\n"); expect(flow.steps).toEqual([ { @@ -1326,7 +1332,7 @@ describe("within selector scoping", () => { ]); }); - it("within chains outward and round-trips exactly", () => { + it("within chains outward and round-trips exactly", async () => { const flow: FlowFile = { executionPrerequisite: "", steps: [ @@ -1354,7 +1360,7 @@ describe("within selector scoping", () => { expect(parseFlow(serializeFlow(flow))).toEqual(flow); }); - it("serializes a loose within back to its bare-string spelling", () => { + it("serializes a loose within back to its bare-string spelling", async () => { const yaml = serializeFlow({ executionPrerequisite: "", steps: [ @@ -1367,7 +1373,7 @@ describe("within selector scoping", () => { expect(yaml).toContain("within: profile-card"); }); - it("accepts the regex text matcher inside a within scope", () => { + it("accepts the regex text matcher inside a within scope", async () => { const flow = parseFlow( "steps:\n - assert: { visible: { text: Delete, within: { text: { matches: '^Card \\d+$' } } } }\n" ); @@ -1380,42 +1386,42 @@ describe("within selector scoping", () => { ]); }); - it("rejects a selector that is ONLY a within scope", () => { + it("rejects a selector that is ONLY a within scope", async () => { expect(() => parseFlow("steps:\n - tap: { within: { id: card } }\n")).toThrow( /still needs its own text\/id\/role/ ); }); - it("rejects unknown keys inside a within scope, naming the nested slot", () => { + it("rejects unknown keys inside a within scope, naming the nested slot", async () => { expect(() => parseFlow("steps:\n - tap: { text: Delete, within: { idd: card } }\n")).toThrow( /tap\.within: selector has unknown key `idd` \(did you mean `id`\?\)/ ); }); - it("rejects id+identifier both set inside a within scope", () => { + it("rejects id+identifier both set inside a within scope", async () => { expect(() => parseFlow("steps:\n - tap: { text: A, within: { id: x, identifier: x } }\n") ).toThrow(/`id` or `identifier` \(its alias\), not both/); }); - it("rejects a cyclic within alias via the depth cap", () => { + it("rejects a cyclic within alias via the depth cap", async () => { const yaml = "steps:\n - tap: &s { text: Delete, within: *s }\n"; expect(() => parseFlow(yaml)).toThrow(/nest deeper than|cyclic YAML alias/); }); - it("rejects a within selector mixed with coordinates", () => { + it("rejects a within selector mixed with coordinates", async () => { expect(() => parseFlow("steps:\n - tap: { within: { id: card }, x: 0.5, y: 0.5 }\n")).toThrow( /takes a selector or x\/y coordinates, not both/ ); }); - it("rejects a within key beside the tap options form", () => { + it("rejects a within key beside the tap options form", async () => { expect(() => parseFlow("steps:\n - tap: { on: Photo, times: 2, within: { id: card } }\n") ).toThrow(/the tap options form takes a nested selector/); }); - it("within works in scroll-to's target while scroll-to's own within stays the container anchor", () => { + it("within works in scroll-to's target while scroll-to's own within stays the container anchor", async () => { const flow = parseFlow( [ "steps:", @@ -1435,7 +1441,7 @@ describe("within selector scoping", () => { ]); }); - it("describeSelector renders the scope chain in parentheses", () => { + it("describeSelector renders the scope chain in parentheses", async () => { expect( describeSelector({ text: "Delete", @@ -1444,7 +1450,7 @@ describe("within selector scoping", () => { ).toBe('text="Delete" within (id="cards" within (text="Settings"))'); }); - it("when guards reject a {{secret:…}} placeholder hidden in a within scope", () => { + it("when guards reject a {{secret:…}} placeholder hidden in a within scope", async () => { expect(() => parseFlow( [ @@ -1461,7 +1467,7 @@ describe("within selector scoping", () => { // ── sibling scopes (`after`/`next`) and the `any` universal selector ── describe("sibling selector scopes and the universal selector", () => { - it("parses `after` (CSS ~) and `next` (CSS +) scopes", () => { + it("parses `after` (CSS ~) and `next` (CSS +) scopes", async () => { const flow = parseFlow( [ "steps:", @@ -1479,7 +1485,7 @@ describe("sibling selector scopes and the universal selector", () => { ]); }); - it("parses `any: true` paired with a scope and round-trips exactly", () => { + it("parses `any: true` paired with a scope and round-trips exactly", async () => { const yaml = [ "steps:", @@ -1501,7 +1507,7 @@ describe("sibling selector scopes and the universal selector", () => { expect(parseFlow(serializeFlow(flow))).toEqual(flow); }); - it("a bare-string sibling scope stays loose, and serializes back to the bare spelling", () => { + it("a bare-string sibling scope stays loose, and serializes back to the bare spelling", async () => { const flow = parseFlow("steps:\n - tap: { role: Switch, next: wifi-row }\n"); expect(flow.steps).toEqual([ { kind: "tap", selector: { role: "Switch", next: { text: "wifi-row", loose: true } } }, @@ -1510,7 +1516,7 @@ describe("sibling selector scopes and the universal selector", () => { expect(parseFlow(serializeFlow(flow))).toEqual(flow); }); - it("scopes combine and nest, round-tripping through YAML", () => { + it("scopes combine and nest, round-tripping through YAML", async () => { const flow = parseFlow( [ "steps:", @@ -1533,7 +1539,7 @@ describe("sibling selector scopes and the universal selector", () => { expect(parseFlow(serializeFlow(flow))).toEqual(flow); }); - it("accepts the regex text matcher inside a sibling scope", () => { + it("accepts the regex text matcher inside a sibling scope", async () => { const flow = parseFlow( "steps:\n - tap: { role: Switch, next: { text: { matches: '^Row \\d+$' } } }\n" ); @@ -1542,25 +1548,25 @@ describe("sibling selector scopes and the universal selector", () => { ]); }); - it("rejects a selector that is ONLY a sibling scope", () => { + it("rejects a selector that is ONLY a sibling scope", async () => { expect(() => parseFlow("steps:\n - tap: { after: { text: Danger } }\n")).toThrow( /`after` only scopes where to look — the selector still needs its own text\/id\/role/ ); }); - it("rejects `any: true` alongside the fields it would make redundant", () => { + it("rejects `any: true` alongside the fields it would make redundant", async () => { expect(() => parseFlow("steps:\n - tap: { any: true, role: Button, next: { text: Wi-Fi } }\n") ).toThrow(/already matches every element — drop it, or drop the `role`/); }); - it("rejects a bare `any: true` with no scope to narrow it", () => { + it("rejects a bare `any: true` with no scope to narrow it", async () => { expect(() => parseFlow("steps:\n - tap: { any: true }\n")).toThrow( /matches every element on screen — pair it with a scope \(within\/after\/next\)/ ); }); - it("rejects a non-`true` any value rather than reading it as a locator", () => { + it("rejects a non-`true` any value rather than reading it as a locator", async () => { // Falsy AND truthy: a truthiness check would wave `any: 1` / `any: yes` // through as the universal selector — a spelling no reader can predict and // the serializer cannot reproduce. @@ -1571,19 +1577,19 @@ describe("sibling selector scopes and the universal selector", () => { } }); - it("rejects unknown keys inside a sibling scope, naming the nested slot", () => { + it("rejects unknown keys inside a sibling scope, naming the nested slot", async () => { expect(() => parseFlow("steps:\n - tap: { role: Switch, next: { roel: Button } }\n")).toThrow( /tap\.next: selector has unknown key `roel` \(did you mean `role`\?\)/ ); }); - it("rejects a cyclic sibling alias via the scope budget", () => { + it("rejects a cyclic sibling alias via the scope budget", async () => { expect(() => parseFlow("steps:\n - tap: &s { text: Delete, after: *s }\n")).toThrow( /more than \d+ scopes|cyclic YAML alias/ ); }); - it("bounds a selector's whole scope TREE, not just its depth", () => { + it("bounds a selector's whole scope TREE, not just its depth", async () => { // Three relations per level means a depth cap alone still admits 3^depth // scopes — and the runner expands one alternative per combination of // bare-string scopes, so a few hundred bytes of YAML would exhaust the heap @@ -1607,7 +1613,7 @@ describe("sibling selector scopes and the universal selector", () => { ).toThrow(/more than 6 scopes/); }); - it("serializeFlow refuses an `any` selector the parser would reject on read-back", () => { + it("serializeFlow refuses an `any` selector the parser would reject on read-back", async () => { // appendStep re-parses the whole file on every recorded step, so a selector // that violates the parser's `any` rules must fail where it was built, not // on some later append. @@ -1632,7 +1638,7 @@ describe("sibling selector scopes and the universal selector", () => { ]); }); - it("rejects a sibling-scoped selector mixed with coordinates or tap options", () => { + it("rejects a sibling-scoped selector mixed with coordinates or tap options", async () => { expect(() => parseFlow("steps:\n - tap: { after: { id: card }, x: 0.5, y: 0.5 }\n")).toThrow( /takes a selector or x\/y coordinates, not both/ ); @@ -1641,7 +1647,7 @@ describe("sibling selector scopes and the universal selector", () => { ).toThrow(/the tap options form takes a nested selector/); }); - it("a loose bare-string selector cannot carry a scope through serialization", () => { + it("a loose bare-string selector cannot carry a scope through serialization", async () => { expect(() => serializeFlow({ executionPrerequisite: "", @@ -1652,7 +1658,7 @@ describe("sibling selector scopes and the universal selector", () => { ).toThrow(/incompatible fields: after/); }); - it("names the missing scroll-to target instead of leaking a schema message", () => { + it("names the missing scroll-to target instead of leaking a schema message", async () => { // `within` is a selector key now, so this body reads like a scoped selector // — it is actually the options map, missing its target. for (const body of ["{ within: { id: list } }", "{ direction: up }", "{}"]) { @@ -1672,7 +1678,7 @@ describe("sibling selector scopes and the universal selector", () => { ).not.toThrow(); }); - it("describeSelector renders each scope, and `*` for the universal selector", () => { + it("describeSelector renders each scope, and `*` for the universal selector", async () => { expect(describeSelector({ role: "Switch", next: { text: "Wi-Fi" } })).toBe( 'role="Switch" next (text="Wi-Fi")' ); @@ -1681,7 +1687,7 @@ describe("sibling selector scopes and the universal selector", () => { ).toBe('* within (id="row") after (text="Name")'); }); - it("when guards reject a {{secret:…}} placeholder hidden in ANY scope", () => { + it("when guards reject a {{secret:…}} placeholder hidden in ANY scope", async () => { // Every relation, so no branch of the walk can be skipped unnoticed. for (const scope of ["within", "after", "next"]) { expect(() => From c17405bf68e6b6e6ff28df760c4a75469e779d85 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:23:00 +0200 Subject: [PATCH 31/98] fix(flow): let a cleanup flow run again, and scope its teardown when it can MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding `devices` to DEVICE_ARG_KEYS made `toolRequiresDevice` true for stop-all-simulator-servers, so resolveRunDevice demanded a resolved device where it previously returned none. On main the tool had no schema at all, so `!props` made a teardown-only flow device-free. The two situations such a flow actually runs in both broke: with several booted it failed FLOW_DEVICE_RESOLUTION ("pass --device or --platform to disambiguate"), and with none matching it failed "No booted device found" — neither a question a machine-wide sweep has an answer to. A flow that also contains a device step already required one and was unaffected. `devices` is a scope, not a target, and that is the distinction the key sets now draw: a screenshot with no `udid` has nothing to point at and cannot run, while the teardown with no `devices` is the sweep itself — a complete call, and the whole content of a cleanup flow. So DEVICE_ARG_KEYS covers targets only, and a flow that merely SCOPES to a device resolves one opportunistically: it takes the device when one is unambiguous, keeping the narrowing that stops a replayed teardown reaping what another agent is mid-session on, and runs unscoped when resolution has no single answer rather than failing the flow. bindDeviceArgs no longer binds a scope key with no device behind it. `[""]` would be a teardown scoped to an id that owns nothing — reaping nothing while reporting pass, the exact failure the binding exists to prevent. flow-deviceless covered this in the one configuration that still worked (exactly one booted). It now covers nothing booted, several booted, an explicit --device, a --platform that still matches several, and the mixed flow whose device step keeps the scope bound. --- .../src/tools/flows/flow-device.ts | 62 +++++++--- .../tool-server/src/tools/flows/flow-run.ts | 37 ++++-- .../test/flows/flow-deviceless.test.ts | 115 +++++++++++++++--- 3 files changed, 172 insertions(+), 42 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index c34b3351a..5542fc352 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -57,21 +57,26 @@ const DEVICE_BIND_KEYS = ["udid", "device_id", "device"] as const; const DEVICE_BIND_LIST_KEYS = ["devices"] as const; /** - * Keys that mean a tool acts on a device — every key either bind set covers. + * Keys that mean a tool needs a device to act on at all — the TARGET keys, and + * deliberately not the scope keys in {@link DEVICE_BIND_LIST_KEYS}. * * `toolRequiresDevice` consults this, and `resolveRunDevice` skips resolving a - * device for a flow no step here matches. So a key that is BOUND but not listed - * here is worse than an unbound one: the run resolves `device: null`, and - * `bindDeviceArgs(…, device?.id ?? "", …)` then rebinds the recorded value to - * the empty string rather than leaving it alone. For `devices` that is - * `{ devices: [""] }` — a teardown scoped to an id that owns nothing, which - * reaps nothing and still reports pass, exactly the failure - * {@link DEVICE_BIND_LIST_KEYS} exists to prevent. + * device for a flow no step here matches. The distinction is what a missing + * device does to the step: a `screenshot` with no `udid` has nothing to point + * at and cannot run, while `stop-all-simulator-servers` with no `devices` is + * the machine-wide sweep — a complete, meaningful call, and the whole content + * of a cleanup flow. Listing `devices` here made such a flow demand a device it + * has no use for, so the two situations a cleanup flow actually runs in — none + * booted, or several — failed it outright. * - * Both sets, therefore, and not a hand-maintained superset: a key added to - * either one is covered here by construction. + * A scope key is therefore bound OPPORTUNISTICALLY: {@link bindDeviceArgs} + * injects it when the run resolved a device (so a replayed teardown cannot reap + * devices another agent is mid-session on) and leaves it off when the run has + * none, rather than binding the empty string — `{ devices: [""] }` would be a + * teardown scoped to an id that owns nothing, reaping nothing while reporting + * pass, which is the failure {@link DEVICE_BIND_LIST_KEYS} exists to prevent. */ -const DEVICE_ARG_KEYS = [...DEVICE_BIND_KEYS, ...DEVICE_BIND_LIST_KEYS] as const; +const DEVICE_ARG_KEYS = DEVICE_BIND_KEYS; interface RawDevice { platform: FlowPlatform; @@ -227,16 +232,36 @@ export function flowRequiresDevice(registry: Registry, steps: FlowStep[]): boole return steps.some((step) => stepRequiresDevice(registry, step)); } +/** + * Whether any step would NARROW itself to the run device if one were resolved, + * without needing one to run — a `devices` scope, and only that today. + * + * Asked of a flow that {@link flowRequiresDevice} said no to, so the run has a + * choice: resolve a device opportunistically and scope the teardown to it + * (keeping the cross-agent protection the scope exists for), or, where no + * single device is resolvable, run the step's unscoped meaning rather than + * failing a flow whose whole purpose is to clear the machine. + */ +export function flowScopesDevice(registry: Registry, steps: FlowStep[]): boolean { + return steps.some( + (step) => step.kind === "tool" && declaresAny(registry, step.name, DEVICE_BIND_LIST_KEYS) + ); +} + function toolRequiresDevice(registry: Registry, toolName: string): boolean { - const toolDef = registry.getTool(toolName); // An unknown tool is assumed to need a device: the step is going to fail // either way, and it fails more usefully with one resolved. - if (!toolDef) return true; - const props = (toolDef.inputSchema as { properties?: Record } | undefined) + if (!registry.getTool(toolName)) return true; + return declaresAny(registry, toolName, DEVICE_ARG_KEYS); +} + +function declaresAny(registry: Registry, toolName: string, keys: readonly string[]): boolean { + const toolDef = registry.getTool(toolName); + const props = (toolDef?.inputSchema as { properties?: Record } | undefined) ?.properties; // A tool with no declared input takes no device. if (!props) return false; - return DEVICE_ARG_KEYS.some((k) => k in props); + return keys.some((k) => k in props); } export function bindDeviceArgs( @@ -251,7 +276,12 @@ export function bindDeviceArgs( const out = stripDeviceKeys(args); if (props) { for (const k of DEVICE_BIND_KEYS) if (k in props) out[k] = deviceId; - for (const k of DEVICE_BIND_LIST_KEYS) if (k in props) out[k] = [deviceId]; + // Scope keys only when there IS a device. A device-free run reaches here + // for a cleanup flow (see {@link DEVICE_ARG_KEYS}), and `[""]` there would + // scope the teardown to an id that owns nothing — it would reap nothing and + // still pass. Leaving the key off runs the sweep the YAML actually + // expresses. + if (deviceId) for (const k of DEVICE_BIND_LIST_KEYS) if (k in props) out[k] = [deviceId]; } return out; } diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 4e4c97e2e..cd241038f 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -45,6 +45,7 @@ import { resolveFlowDevice, bindDeviceArgs, flowRequiresDevice, + flowScopesDevice, stepRequiresDevice, type FlowPlatform, } from "./flow-device"; @@ -1149,16 +1150,32 @@ async function resolveRunDevice( // Checked after the chromium boot path, which only applies to a flow led by // a `launch` step — and a launch needs a device, so the two never compete. if (!flowRequiresDevice(registry, flow.steps)) { - return { device: null, booted: null }; + if (!flowScopesDevice(registry, flow.steps)) return { device: null, booted: null }; + // A flow that only SCOPES to a device (a cleanup flow) takes one when one + // is unambiguous, so the teardown stays narrowed to the run device and + // cannot reap what another agent is mid-session on. When resolution has + // no single answer — nothing booted, or several — that is not a question + // a sweep has an answer to, so run it unscoped rather than failing the + // flow. Swallowed only here: every other caller genuinely needs the + // device, and the diagnosis in the error is the useful thing there. + try { + return { + device: await resolveFlowDevice(registry, ctx, resolveOpts(params)), + booted: null, + }; + } catch { + return { device: null, booted: null }; + } } } - const device = await resolveFlowDevice(registry, ctx, { - device: params.device, - platform: params.platform as FlowPlatform | undefined, - }); + const device = await resolveFlowDevice(registry, ctx, resolveOpts(params)); return { device, booted: null }; } +function resolveOpts(params: Params): { device?: string; platform?: FlowPlatform } { + return { device: params.device, platform: params.platform as FlowPlatform | undefined }; +} + /** * The hoisted boot's failure, carrying the lock explanation when it is * lock-shaped. This is the likeliest way of all to meet the lock — the app is @@ -2115,10 +2132,12 @@ async function execLeafStep( case "tool": { // A device-less run reaches here only for a tool declaring none of - // `DEVICE_ARG_KEYS`, so binding injects nothing and merely strips any - // device key the recorded args carried. The `?? ""` is unreachable in - // that pairing and must stay unreachable: injecting the empty string - // would not fail the step, it would silently retarget it at no device. + // `DEVICE_ARG_KEYS` — a target key — so binding injects no target and + // merely strips any device key the recorded args carried. The `?? ""` is + // unreachable for those and must stay unreachable: injecting the empty + // string would not fail the step, it would silently retarget it at no + // device. A SCOPE key (`devices`) does reach here device-free, which is + // the cleanup-flow case `bindDeviceArgs` guards by leaving it unset. const args = bindDeviceArgs(registry, step.name, device?.id ?? "", step.args); const outputHint = registry.getTool(step.name)?.outputHint; if (step.delayMs && !(await sleepOrAbort(step.delayMs, signal))) { diff --git a/packages/tool-server/test/flows/flow-deviceless.test.ts b/packages/tool-server/test/flows/flow-deviceless.test.ts index 63cc33fee..069180481 100644 --- a/packages/tool-server/test/flows/flow-deviceless.test.ts +++ b/packages/tool-server/test/flows/flow-deviceless.test.ts @@ -291,11 +291,16 @@ describe("stepRequiresDevice", () => { expect(stepRequiresDevice(registry, toolStep("not-a-tool"))).toBe(true); }); - it("counts the REAL stop-all-simulator-servers schema as acting on a device", () => { + it("does NOT count the REAL stop-all-simulator-servers schema as needing a device", () => { // Against the derived JSON schema, not the mock above: the mock is only as // good as its agreement with the tool, and the failure this guards is - // exactly a drift between the two — the tool declaring a device key that - // `DEVICE_ARG_KEYS` does not list. Catches a rename of `devices` too. + // exactly a drift between the two. Catches a rename of `devices` too. + // + // `devices` is a SCOPE, not a target: the unscoped call is a complete, + // meaningful machine-wide sweep, so a flow whose only step is this one + // needs no device. Counting it made such a flow demand one — see the + // cleanup-flow cases below, which are the two situations it actually runs + // in. const schema = zodObjectToJsonSchema( createStopAllSimulatorServersTool({} as unknown as Registry).zodSchema! ); @@ -305,28 +310,30 @@ describe("stepRequiresDevice", () => { const registry = { getTool: () => ({ inputSchema: schema }) } as unknown as Registry; expect( stepRequiresDevice(registry, { kind: "tool", name: "stop-all-simulator-servers", args: {} }) - ).toBe(true); + ).toBe(false); }); - it("counts a device LIST argument as acting on a device", () => { - // `stop-all-simulator-servers` spells its scope `devices`, the only tool - // that does. Missing it here is not a missing injection but a wrong one: - // the run resolves no device, and the binding then rebinds the recorded - // scope to `[""]` — a teardown that reaps nothing and still reports pass. + it("counts a device TARGET argument, but not a device LIST scope", () => { + // The distinction is what a missing device does to the step: `screenshot` + // with no `udid` has nothing to point at, while the teardown with no + // `devices` is the sweep itself. const { registry } = mockRegistry(); expect( stepRequiresDevice(registry, { kind: "tool", name: "stop-all-simulator-servers", args: {} }) - ).toBe(true); + ).toBe(false); + expect(stepRequiresDevice(registry, { kind: "tool", name: "tap", args: {} })).toBe(true); }); }); -describe("a recorded teardown step", () => { - it("replays against the run device, not against an empty scope", async () => { - await writeFlow("teardownonly", [ - // What the recorder writes for a scoped `stop-all-simulator-servers`: - // the `devices` key is stripped at record time and re-injected here. - { kind: "tool", name: "stop-all-simulator-servers", args: {} }, - ]); +describe("a cleanup flow whose only step is stop-all-simulator-servers", () => { + const teardownOnly: FlowStep[] = [ + // What the recorder writes for a `stop-all-simulator-servers`: the + // `devices` key is stripped at record time and re-injected at replay. + { kind: "tool", name: "stop-all-simulator-servers", args: {} }, + ]; + + it("replays against the run device when exactly one is booted", async () => { + await writeFlow("teardownonly", teardownOnly); const { registry, invokeTool } = mockRegistry({ booted: [DEVICE] }); const run = asRun(await runAuto(registry, "teardownonly")); @@ -334,4 +341,78 @@ describe("a recorded teardown step", () => { expect(run.ok).toBe(true); expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { devices: [DEVICE] }); }); + + it("runs as the machine-wide sweep with NOTHING booted", async () => { + // One of the two situations a cleanup flow actually runs in. Requiring a + // device here failed it with "No booted device found" — on a flow whose + // entire purpose is to run when the machine needs clearing. + await writeFlow("teardownonly", teardownOnly); + const { registry, invokeTool } = mockRegistry({ booted: [] }); + const run = asRun(await runAuto(registry, "teardownonly")); + + expect(run.ok).toBe(true); + expect(run.passed).toBe(1); + // No scope, and emphatically not `[""]` — an id that owns nothing would + // reap nothing and still pass. + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", {}); + }); + + it("runs as the machine-wide sweep with SEVERAL booted, without disambiguation", async () => { + // The other one. Requiring a device here failed with "2 booted devices + // matched — pass --device or --platform", which is not a question a sweep + // has an answer to. + await writeFlow("teardownonly", teardownOnly); + const other = "11111111-1111-1111-1111-111111111111"; + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE, other] }); + const run = asRun(await runAuto(registry, "teardownonly")); + + expect(run.ok).toBe(true); + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", {}); + }); + + it("scopes to an explicitly passed device", async () => { + // The narrowing is deliberate where the run has an answer: a replayed + // teardown must not reap devices another agent is mid-session on. + await writeFlow("teardownonly", teardownOnly); + const other = "11111111-1111-1111-1111-111111111111"; + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE, other] }); + const runFlow = createRunFlowTool(registry); + const run = asRun( + await runFlow.execute({}, { name: "teardownonly", project_root: tmpDir, device: DEVICE }) + ); + + expect(run.ok).toBe(true); + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { devices: [DEVICE] }); + }); + + it("falls back to the sweep when a passed platform still matches several", async () => { + // A platform that does not narrow to one device is not an answer either, + // and the flow must still run rather than demanding --device. + await writeFlow("teardownonly", teardownOnly); + const other = "11111111-1111-1111-1111-111111111111"; + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE, other] }); + const runFlow = createRunFlowTool(registry); + const run = asRun( + await runFlow.execute({}, { name: "teardownonly", project_root: tmpDir, platform: "ios" }) + ); + + expect(run.ok).toBe(true); + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", {}); + }); + + it("still scopes the teardown when the flow ALSO has a device step", async () => { + // A flow with a real device step resolves one as it always did, and the + // teardown is scoped to it — the cross-agent protection the scope exists + // for is unaffected by any of the above. + await writeFlow("teardownmixed", [ + { kind: "tool", name: "tap", args: { x: 1, y: 2 } }, + ...teardownOnly, + ]); + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE] }); + const run = asRun(await runAuto(registry, "teardownmixed")); + + expect(run.device).toBe(DEVICE); + expect(run.ok).toBe(true); + expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { devices: [DEVICE] }); + }); }); From 95ead1d48055af3158bb68aafb6d93123f4f7796 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:32:00 +0200 Subject: [PATCH 32/98] fix(flow): keep a recorded teardown's device scope in the YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stripDeviceKeys deleted `devices` along with the device-target keys, and an empty `args` is omitted from the YAML, so a correctly scoped `stop-all-simulator-servers {"devices":[""]}` recorded as a bare `- tool: stop-all-simulator-servers` — which is not a narrower version of that call, it is the machine-wide sweep. Replay was safe (the runner rebinds to the run device), but the YAML is the artifact that gets committed and read, and the create-flow skill's "Strategy 2 — Manual execution" tells agents to hand-execute remaining steps from the file. Run verbatim, that step reaps every device on the machine — the cross-agent teardown the scope exists to prevent. The contrast is the point: a recorded `screenshot` loses its `udid` too, but hand-running it fails loudly because `udid` is required. Losing `devices` fails OPEN. A target is stripped so the flow points at no device; a scope is not, because dropping it changes what the step means. The recorded ids are host-specific, which costs a no-op plus an `unmatched` report on another machine — the safe direction, and a legible one. bindDeviceArgs keeps the portability the strip was there for: it still overrides the recorded scope with the run device whenever one resolved, still refuses to forward a scope to a tool that does not declare it (a .strict() schema would reject the call), and now leaves a recorded scope alone when the run has no device at all — there is no run target to override, and dropping it would widen a teardown the recording had scoped. Verified over HTTP against a tool-server built from this branch: the recorded YAML carries `devices: [emulator-5556]`; replaying with --device emulator-5558 rebinds to 5558; replaying device-free with six booted keeps 5556 rather than sweeping all six. --- .../skills/skills/argent-create-flow/SKILL.md | 2 + .../src/tools/flows/flow-device.ts | 38 +++++++++++++----- .../test/flows/flow-composition.test.ts | 39 +++++++++++++++++-- .../tool-server/test/flows/flow-tools.test.ts | 19 ++++++--- 4 files changed, 79 insertions(+), 19 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index ebf1eb49f..5972045ad 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -9,6 +9,8 @@ A flow is a sequence of steps saved to a `.yaml` file in the `.argent/flows/` di Flows store **no device id**: the runner binds a device (the single booted one, or pass `device`/`platform`). A recorded coordinate `gesture-tap` is captured as a portable `tap: { selector }` step whenever the tapped element has stable text/identifier. +The one exception is a device _scope_ rather than a target: `stop-all-simulator-servers`' `devices` list is kept in the YAML, because without it the step means the machine-wide sweep and would tear down devices other agents are mid-session on. Replay still rebinds it to the run device, so the recorded ids only matter if you hand-run the step (see _Strategy 2 — Manual execution_), where they keep the teardown scoped instead of reaping everything. A cleanup flow whose only step is that teardown needs no device and runs whether none or several are booted. + **Two flow types** - **e2e** — begins with a `launch:` step, which starts that app from scratch (terminate + relaunch), so the flow controls its own start state. No `executionPrerequisite`. May `run:` other flows, and may itself be a `run:` target — when nested, its `launch` runs inline, restarting the app for that sub-scenario. **On Chromium a launch is a process, not a relaunch:** the "device" is the booted app (its id is the CDP port). The runner needs a device before step 1, so it boots for the launch the run _begins_ with, following a leading `run:` — a fragment whose first step composes a chromium e2e flow boots that flow's app (pass `--platform chromium` when the launch names several platforms, or the target is ambiguous and auto-detection is used instead). That first launch then just settles the instance it was booted for; every _later_ launch — a nested e2e flow's own, or a mid-flow `launch:` of the same app — boots its own instance and the run moves onto it for the remaining steps, replacing the one the runner already owns for that app. Every instance the runner boots is torn down at run end; one you pinned with `--device` is attached to, never killed — so relaunching _that_ app mid-flow fails if it holds a single-instance lock. A launch that names no id for the run's platform is an error — a `chromium:` entry does not make a flow runnable on iOS, and the run never switches platforms mid-flight. Record one by adding a `restart-app` of the app under test as the **first** step — it is captured as the `launch` step. Not on Chromium, though: `restart-app` has no chromium support and only successful calls are recorded, so a recorded chromium flow is always a fragment — write the `launch: { chromium: }` line into the YAML yourself afterward, and delete any `executionPrerequisite` the recording declared: with its own launch the flow controls its start state, and a launch-first flow must not carry one. diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index 5542fc352..2ab5b65f7 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -156,16 +156,29 @@ export async function resolveFlowDevice( } /** - * Strip the device-id keys from a set of args (so a flow stores none). + * Strip the device-TARGET keys from a set of args (so a recorded flow stores no + * device to point at). Scope keys are deliberately kept — see below. * * Schema-blind on purpose: `bindDeviceArgs` strips unconditionally and re-injects * only what the target tool declares, so a stale id is never forwarded to a tool * that does not want it. + * + * A SCOPE survives into the YAML because dropping it changes what the recorded + * step MEANS. `stop-all-simulator-servers` with no `devices` is the machine-wide + * sweep, so a correctly scoped teardown would record as a bare + * `- tool: stop-all-simulator-servers` — and the YAML is the artifact that gets + * committed, read, and (per the create-flow skill's manual-execution strategy) + * hand-run a step at a time. Replay rebinds it either way, but hand-running that + * bare step reaps every device on the machine, which is the cross-agent teardown + * the scope exists to prevent. The contrast is the point: a recorded `screenshot` + * loses its `udid` too, and hand-running it fails loudly because `udid` is + * required. Losing `devices` fails OPEN. The recorded ids are host-specific, but + * that costs only a no-op plus an `unmatched` report on another machine — the + * safe direction, and a legible one. */ export function stripDeviceKeys(args: Record): Record { const out = { ...args }; for (const k of DEVICE_BIND_KEYS) delete out[k]; - for (const k of DEVICE_BIND_LIST_KEYS) delete out[k]; return out; } @@ -274,14 +287,19 @@ export function bindDeviceArgs( const props = (toolDef?.inputSchema as { properties?: Record } | undefined) ?.properties; const out = stripDeviceKeys(args); - if (props) { - for (const k of DEVICE_BIND_KEYS) if (k in props) out[k] = deviceId; - // Scope keys only when there IS a device. A device-free run reaches here - // for a cleanup flow (see {@link DEVICE_ARG_KEYS}), and `[""]` there would - // scope the teardown to an id that owns nothing — it would reap nothing and - // still pass. Leaving the key off runs the sweep the YAML actually - // expresses. - if (deviceId) for (const k of DEVICE_BIND_LIST_KEYS) if (k in props) out[k] = [deviceId]; + for (const k of DEVICE_BIND_LIST_KEYS) { + // Never forward a scope to a tool that does not declare it — a `.strict()` + // schema would reject the whole call. + if (!props || !(k in props)) delete out[k]; + // The run device wins over anything recorded, so a flow recorded against + // one device stays portable. With NO run device (a cleanup flow, see + // {@link DEVICE_ARG_KEYS}) the recorded scope is kept rather than dropped: + // there is no run target for it to override, and dropping it would widen a + // teardown the recording scoped — the one direction that costs another + // agent their devices. `[""]` is never bound: an id that owns nothing reaps + // nothing and still reports pass. + else if (deviceId) out[k] = [deviceId]; } + if (props) for (const k of DEVICE_BIND_KEYS) if (k in props) out[k] = deviceId; return out; } diff --git a/packages/tool-server/test/flows/flow-composition.test.ts b/packages/tool-server/test/flows/flow-composition.test.ts index 3f4d5f4ba..788064e5b 100644 --- a/packages/tool-server/test/flows/flow-composition.test.ts +++ b/packages/tool-server/test/flows/flow-composition.test.ts @@ -2375,10 +2375,43 @@ describe("device binding (portability)", () => { expect(out).toEqual({ foo: 1 }); }); - it("stripDeviceKeys removes udid / device_id / device / devices, leaving other args untouched", () => { + it("stripDeviceKeys removes udid / device_id / device, leaving other args untouched", () => { + expect(stripDeviceKeys({ udid: "A", device_id: "B", device: "C", x: 1 })).toEqual({ x: 1 }); + }); + + it("stripDeviceKeys KEEPS a `devices` scope, because dropping it changes the step's meaning", () => { + // A target is stripped so the flow points at no device. A scope is not: + // `stop-all-simulator-servers` with no `devices` is the machine-wide sweep, + // so stripping it would record a correctly scoped teardown as a bare step + // that reaps every device on the machine when hand-run from the YAML — the + // manual-execution strategy the create-flow skill documents. Replay rebinds + // it either way (see bindDeviceArgs below). + expect(stripDeviceKeys({ udid: "A", devices: ["D", "E"], x: 1 })).toEqual({ + devices: ["D", "E"], + x: 1, + }); + }); + + it("bindDeviceArgs keeps a recorded scope when the run resolved NO device", () => { + // A cleanup flow resolves no device when none is unambiguous. Dropping the + // recorded scope there would widen the teardown from the devices the + // recording named to every device on the machine — the one direction that + // costs another agent their session. There is no run target to override. expect( - stripDeviceKeys({ udid: "A", device_id: "B", device: "C", devices: ["D", "E"], x: 1 }) - ).toEqual({ x: 1 }); + bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "", { + devices: ["RECORDED"], + }) + ).toEqual({ devices: ["RECORDED"] }); + }); + + it("bindDeviceArgs never forwards a scope to a tool that does not declare it", () => { + // The schema-blind strip's job: a `.strict()` schema would reject the call. + expect( + bindDeviceArgs(reg({ port: {} }), "stop-metro", "RESOLVED", { + devices: ["RECORDED"], + port: 8081, + }) + ).toEqual({ port: 8081 }); }); it("rebinds a nested flow-execute onto the run device (issue #607)", () => { diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index 259a81c15..c7b343ef7 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -1363,10 +1363,13 @@ describe("flow-add-step", () => { expect(flow.steps).toEqual([]); }); - it("strips the devices list when recording a scoped teardown (device ids stay off disk)", async () => { - // stop-all-simulator-servers' `devices` names the recording host's device - // ids the same way a udid does; a recorded scoped teardown must not bake - // that host's ids into the flow, or replay on another host stops nothing. + it("keeps the devices list when recording a scoped teardown, so the YAML stays scoped", async () => { + // `devices` is a scope, not a target: with it stripped, a correctly scoped + // teardown recorded as a bare `- tool: stop-all-simulator-servers`, which + // IS the machine-wide sweep — so hand-running the step from the YAML (the + // create-flow skill's manual-execution strategy) reaped every device on the + // machine. Replay rebinds the scope to the run device regardless, so + // keeping it costs portability nothing. const registry = createMockRegistry({ "stop-all-simulator-servers": { result: { stopped: 1 } }, }); @@ -1387,9 +1390,13 @@ describe("flow-add-step", () => { expect(registry.invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", { devices: ["00000000-HOST-DEVICE-ID"], }); - // …but the recorded step carries no device id, keeping the flow portable. + // …and the recorded step still reads as the scoped teardown it was. expect(parseFlow(result.flowFile).steps).toEqual([ - { kind: "tool", name: "stop-all-simulator-servers", args: {} }, + { + kind: "tool", + name: "stop-all-simulator-servers", + args: { devices: ["00000000-HOST-DEVICE-ID"] }, + }, ]); }); From 548edf0ad8ec4ed9dd6db1d74b7021be85bdb40d Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:38:35 +0200 Subject: [PATCH 33/98] fix(debugger): keep the log-registry's promise on Chromium too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool description was strengthened to "Absent that note, empty really does mean the app has logged nothing", and the tool documents itself as working against Hermes (iOS / Android / Vega) and V8 (Chromium). Only the Hermes blueprint recorded the breadcrumb; chromium-js-runtime-debugger's dispose deletes its log file and recorded nothing — and this PR added ChromiumJsRuntimeDebugger to DEVICE_OWNED_NAMESPACES, so the teardown now reaches it by name. Destroyed console history therefore read as "the app logged nothing" on V8, which is the wrong conclusion to hand an agent debugging a silent app. Record the same breadcrumb on that side rather than weakening the sentence. One id there, not two: a chromium device's logicalDeviceId IS its device id. Also fixes the consume: the disposer writes ONE event under two keys so either spelling can read it back, but `take(canonical) ?? take(raw)` short-circuited and spent only the key that matched. The survivor then attached a stale explanation to a later, unrelated empty read, against the report-once invariant the breadcrumb store states. All of the device's ids are now spent on that one read — including the logical id, which after `forgetDeviceAlias` only the freshly resolved api still knows. teardown-log-history only ever connected with the logical id, so `api.logicalDeviceId === deviceId` and the two-key write never fired — the Chromium/Vega shape, not the iOS/Android one. It now covers the differing-id case from both spellings. Verified over HTTP against a tool-server built from this branch, driving a throwaway Electron app: 18 entries captured, torn down via stop-all-simulator-servers, reconnect reports totalEntries 0 WITH the note and the second read is silent; same via stop-simulator-server with 29 entries. --- .../chromium-js-runtime-debugger.ts | 23 ++++++++ .../tools/debugger/debugger-log-registry.ts | 25 +++++++- .../test/chromium-js-runtime-debugger.test.ts | 45 ++++++++++++++ .../test/metro/teardown-log-history.test.ts | 59 +++++++++++++++++++ 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/packages/tool-server/src/blueprints/chromium-js-runtime-debugger.ts b/packages/tool-server/src/blueprints/chromium-js-runtime-debugger.ts index c2e5af1a1..c14e440f7 100644 --- a/packages/tool-server/src/blueprints/chromium-js-runtime-debugger.ts +++ b/packages/tool-server/src/blueprints/chromium-js-runtime-debugger.ts @@ -12,6 +12,7 @@ import { SourceMapsRegistry } from "../utils/debugger/source-maps"; import type { SourceResolver } from "../utils/debugger/source-resolver"; import { LogFileWriter } from "../utils/debugger/log-file-writer"; import { consoleTimestampToIso } from "../utils/debugger/console-timestamp"; +import { recordReapedSession } from "../utils/reaped-sessions"; import { type ConsoleLogEntry, type ConsoleLogEvents, @@ -246,6 +247,28 @@ export const chromiumJsRuntimeDebuggerBlueprint: ServiceBlueprint 0) { + recordReapedSession( + "js-runtime-debugger", + device.id, + `The ${captured} captured console ${captured === 1 ? "entry" : "entries"} went with ` + + `it — the log file is deleted on teardown, so this registry starts empty rather ` + + `than the app having logged nothing.` + ); + } logWriter.close(); // Do NOT disconnect the cdp — it belongs to the ChromiumCdp service. // Disposing this blueprint must leave the underlying CDP session alive diff --git a/packages/tool-server/src/tools/debugger/debugger-log-registry.ts b/packages/tool-server/src/tools/debugger/debugger-log-registry.ts index b974fdb53..e987ab918 100644 --- a/packages/tool-server/src/tools/debugger/debugger-log-registry.ts +++ b/packages/tool-server/src/tools/debugger/debugger-log-registry.ts @@ -68,9 +68,28 @@ Use when investigating warnings, errors, or unexpected output — call this firs // reporting this session's own capture, and consuming a breadcrumb there // would attach a stale explanation to a healthy result. if (stats.totalEntries === 0) { - const reaped = - takeReapedSession("js-runtime-debugger", canonicalDeviceId(params.device_id)!) ?? - takeReapedSession("js-runtime-debugger", params.device_id); + // Every id this device answers to, and all of them unconditionally — NOT + // `a ?? b`. The disposer writes ONE event under two keys (the id the + // caller connected with and the `logicalDeviceId` Metro echoed) so either + // spelling can read it back. Short-circuiting consumed only the key that + // matched and left the other behind, where it would attach a stale + // explanation to a later, unrelated empty read — against the report-once + // invariant the breadcrumb store states. `forgetDeviceAlias` runs in that + // same dispose, so by the time this read happens the alias no longer + // joins the two: the logical id has to come from the freshly resolved + // api, which is the only thing that still knows it. + const aliases = [ + canonicalDeviceId(params.device_id), + params.device_id, + api.logicalDeviceId, + ].filter((id): id is string => id !== undefined); + let reaped: ReturnType; + for (const id of new Set(aliases)) { + // Take FIRST, keep second: `reaped ??= take(...)` would short-circuit + // once one matched and leave the rest behind — the very bug above. + const entry = takeReapedSession("js-runtime-debugger", id); + reaped ??= entry; + } if (reaped) response.note = describeReapedSession(reaped, "JS-runtime debugger session"); } return response; diff --git a/packages/tool-server/test/chromium-js-runtime-debugger.test.ts b/packages/tool-server/test/chromium-js-runtime-debugger.test.ts index 71aae0e4d..0ef641f55 100644 --- a/packages/tool-server/test/chromium-js-runtime-debugger.test.ts +++ b/packages/tool-server/test/chromium-js-runtime-debugger.test.ts @@ -8,6 +8,7 @@ import { import { resolveDevice } from "../src/utils/device-info"; import type { ChromiumCdpApi } from "../src/blueprints/chromium-cdp"; import type { CDPClientEvents } from "../src/utils/debugger/cdp-client"; +import { takeReapedSession, __resetReapedSessionsForTesting } from "../src/utils/reaped-sessions"; function makeFakeChromiumCdpApi(): { api: ChromiumCdpApi; @@ -131,6 +132,50 @@ describe("ChromiumJsRuntimeDebugger blueprint", () => { expect(received).toHaveLength(0); }); + it("dispose leaves a reaped-session breadcrumb when it deletes captured history", async () => { + // `debugger-log-registry` documents itself as working against Hermes AND + // V8, and promises that an empty registry with no `note` means the app + // logged nothing. `logWriter.close()` here unlinks the log file, and since + // ChromiumJsRuntimeDebugger joined DEVICE_OWNED_NAMESPACES a + // stop-all-simulator-servers (or a stop-simulator-server cascading through + // ChromiumCdp) routinely triggers this dispose. Without the breadcrumb the + // promise is false on V8: destroyed history reads as a silent app. + __resetReapedSessionsForTesting(); + const fake = makeFakeChromiumCdpApi(); + const instance = await chromiumJsRuntimeDebuggerBlueprint.factory( + { chromium: fake.api }, + "chromium-cdp-19222", + { device: chromiumDevice } + ); + for (let i = 0; i < 18; i++) { + instance.api.logWriter.write({ + id: i, + timestamp: new Date(1710000000000 + i * 1000).toISOString(), + level: "log", + message: `captured ${i}`, + }); + } + await instance.dispose(); + + const reaped = takeReapedSession("js-runtime-debugger", "chromium-cdp-19222"); + expect(reaped).toBeDefined(); + expect(reaped!.salvage).toContain("18 captured console entries"); + }); + + it("dispose leaves NO breadcrumb when there was no history to lose", async () => { + // A dispose of a session that captured nothing destroyed nothing, and + // claiming otherwise would make every empty registry look like a lost one. + __resetReapedSessionsForTesting(); + const fake = makeFakeChromiumCdpApi(); + const instance = await chromiumJsRuntimeDebuggerBlueprint.factory( + { chromium: fake.api }, + "chromium-cdp-19222", + { device: chromiumDevice } + ); + await instance.dispose(); + expect(takeReapedSession("js-runtime-debugger", "chromium-cdp-19222")).toBeUndefined(); + }); + it("dispose does NOT disconnect the underlying CDP — that belongs to ChromiumCdp", async () => { const fake = makeFakeChromiumCdpApi(); // Track whether anything calls disconnect on the cdp. diff --git a/packages/tool-server/test/metro/teardown-log-history.test.ts b/packages/tool-server/test/metro/teardown-log-history.test.ts index f9ed4ab3b..cfa670d90 100644 --- a/packages/tool-server/test/metro/teardown-log-history.test.ts +++ b/packages/tool-server/test/metro/teardown-log-history.test.ts @@ -178,4 +178,63 @@ describe("a debugger session reaped by stop-all-simulator-servers", () => { expect(first.note).toBeDefined(); expect(second.note).toBeUndefined(); }); + + describe("when the connect id and the logicalDeviceId differ", () => { + // Every case above connects with LOGICAL_ID, so `api.logicalDeviceId === + // deviceId` and the disposer's SECOND recordReapedSession never fires — + // that is the Chromium/Vega shape. On iOS/Android the caller connects with + // a udid/serial and Metro echoes its own logical id, so one teardown writes + // two breadcrumbs. They describe one event and must be spent as one. + const CONNECT_ID = "00000000-0000-0000-0000-0000000000ab"; + + beforeEach(async () => { + await registry.disposeService(`JsRuntimeDebugger:${mockPort}:${CONNECT_ID}`).catch(() => {}); + __resetReapedSessionsForTesting(); + }); + + it("explains the loss whichever of the two ids the read uses", async () => { + const urn = await connectAndCapture(CONNECT_ID, 29); + const api = await registry.resolveService(urn); + // The premise: this really is the two-id shape, so both keys get written. + expect(api.logicalDeviceId).toBe(LOGICAL_ID); + expect(api.logicalDeviceId).not.toBe(CONNECT_ID); + await registry.disposeService(urn); + + const viaConnectId = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: CONNECT_ID, + })) as { totalEntries: number; note?: string }; + + expect(viaConnectId.totalEntries).toBe(0); + expect(viaConnectId.note).toContain("29 captured console entries"); + }); + + it("spends BOTH breadcrumbs on that one read, so no copy outlives the event", async () => { + // The read consumed one key and left the other, so a later unrelated + // empty read — a fresh session that genuinely logged nothing — collected + // the leftover and blamed a teardown that had already been explained. + const urn = await connectAndCapture(CONNECT_ID, 7); + await registry.disposeService(urn); + + const first = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: CONNECT_ID, + })) as { note?: string }; + expect(first.note).toBeDefined(); + + // The other spelling of the same device, and the same spelling again: + // neither may still be holding a copy of that one teardown. + const viaLogicalId = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { note?: string }; + const again = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: CONNECT_ID, + })) as { note?: string }; + + expect(viaLogicalId.note).toBeUndefined(); + expect(again.note).toBeUndefined(); + }); + }); }); From a7de7d9620913019b6e6bc20b4c78bd5ae2f8378 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:42:07 +0200 Subject: [PATCH 34/98] fix(flow): classify a failed flows-dir creation as a flow failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `writeNewFlowFile`'s `mkdir` sat outside the FailureError wrapping, so only half of what the tool description promises — "fails if the `.argent/flows/` directory cannot be created OR the flow file cannot be written" — was actually kept. A `project_root` naming an existing file, or an unwritable one, returned REGISTRY_TOOL_EXECUTION_FAILED with a bare `ENOTDIR`/`EACCES` and no remediation hint, while the same permission problem one line later returned FLOW_FILE_WRITE_FAILED with one. Telemetry attributed the first to the registry rather than to flows. Its hint is its own rather than the swap's: `mkdir -p`'s surprising failure is a path COMPONENT that is not a directory, which for a caller-supplied project_root almost always means it named a file — so that case says so, instead of explaining that a rename needs directory permission. --- .../tool-server/src/tools/flows/flow-utils.ts | 55 ++++++++++++++++++- .../flows/flow-concurrent-recording.test.ts | 36 ++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 279433f82..ac8ced4d3 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2624,12 +2624,65 @@ async function writeFlowFile(filePath: string, content: string): Promise { } } +/** + * Why the flows directory could not be created, per errno. Separate from + * {@link writeFailureHint} because the surprising cause differs: the swap's + * hazard is needing permission on the directory, while `mkdir -p`'s is a path + * COMPONENT that is not a directory — which for a caller-supplied + * `project_root` almost always means it named a file. + */ +function mkdirFailureHint(code: string | undefined, dir: string): string { + switch (code) { + case "ENOTDIR": + return ( + `a component of ${dir} exists and is not a directory — check that project_root ` + + `names a directory rather than a file.` + ); + case "EACCES": + case "EPERM": + case "EROFS": + return `the nearest existing parent of ${dir} is not writable.`; + case "ENOSPC": + case "EDQUOT": + return `the filesystem holding ${dir} is out of space (or over quota).`; + case "ENAMETOOLONG": + return `${dir} is longer than this filesystem allows.`; + default: + return `${dir} could not be created.`; + } +} + /** * Create or reset a flow file with `content`, making the parent directory if * needed. Atomic (see {@link writeFlowFile}). + * + * Both halves are classified. The tool description promises this "fails if the + * `.argent/flows/` directory cannot be created OR the flow file cannot be + * written", and leaving the mkdir outside the wrapping made only the second + * half keep that promise: a `project_root` naming an existing file, or an + * unwritable one, surfaced as a bare `ENOTDIR`/`EACCES` under + * REGISTRY_TOOL_EXECUTION_FAILED — no remediation hint, and telemetry + * attributing a flow failure to the registry — while the same permission + * problem one line later returned FLOW_FILE_WRITE_FAILED with a hint. */ export async function writeNewFlowFile(filePath: string, content: string): Promise { - await fs.mkdir(path.dirname(filePath), { recursive: true }); + const dir = path.dirname(filePath); + try { + await fs.mkdir(dir, { recursive: true }); + } catch (err) { + const code = err instanceof Error ? (err as NodeJS.ErrnoException).code : undefined; + throw new FailureError( + `Failed to create the flows directory ${dir}${typeof code === "string" ? ` (${code})` : ""} — ` + + mkdirFailureHint(typeof code === "string" ? code : undefined, dir), + { + error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, + failure_stage: "flow_dir_create", + failure_area: "tool_server", + error_kind: "unknown", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); + } await writeFlowFile(filePath, content); } diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 3151fc368..11bd93e1b 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -837,6 +837,42 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(listActiveRecordings()).toEqual([]); }); + it("classifies a project_root that is a FILE as a flow write failure, not a registry one", async () => { + // The tool description promises it "fails if the .argent/flows/ directory + // cannot be created OR the flow file cannot be written". With the mkdir + // outside the wrapping only the second half kept that promise: this + // surfaced as a bare ENOTDIR under REGISTRY_TOOL_EXECUTION_FAILED, with no + // remediation hint, and telemetry blaming the registry for a flow failure. + const root = await makeRoot("root-is-a-file"); + const notADir = path.join(root, "notadir"); + await fs.writeFile(notADir, "x", "utf8"); + + const err = await start(notADir, "alpha").catch((e: unknown) => e); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_dir_create"); + const message = formatErrorForAgent(err); + expect(message).toContain(path.join(notADir, ".argent", "flows")); + expect(message).toContain("project_root"); + // The kernel's own errno is worth keeping. + expect(message).toContain("ENOTDIR"); + }); + + it("classifies an unwritable project_root the same way", async () => { + const root = await makeRoot("root-unwritable"); + const proj = path.join(root, "proj"); + await fs.mkdir(proj); + await fs.chmod(proj, 0o555); + try { + const err = await start(proj, "alpha").catch((e: unknown) => e); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_dir_create"); + expect(formatErrorForAgent(err)).toMatch(/not writable/); + } finally { + await fs.chmod(proj, 0o755); + } + }); + it("names only the flow file when a read-only flows dir fails an append", async () => { // The review's own repro: chmod 500 the flows dir, then append to a live // recording. This fails at the temp OPEN — earlier than either case above — From 02bd72d6b4d9b21a093ecfad10076c8da535dd00 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:45:05 +0200 Subject: [PATCH 35/98] fix(flow): point a symlinked flow's write failure at the directory that failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `writeFailureHint` used `path.dirname(filePath)` while the temp file and the rename use `path.dirname(realpath(filePath))`. For a flow file that is a symlink into a shared vault those are different directories, and only the second can be the cause — so a 0755 flows dir holding a link into a 0555 vault produced "…so /.argent/flows must be writable", naming a directory that already is, while the vault went unmentioned. Take the resolved target the swap actually uses, and say outright why it is not the directory the reader expected when the two differ: "your flows dir is fine, the link target is not" is the whole diagnosis there. --- .../tool-server/src/tools/flows/flow-utils.ts | 27 ++++++++++++----- .../flows/flow-concurrent-recording.test.ts | 29 +++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index ac8ced4d3..1911430ce 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2525,25 +2525,36 @@ let flowWriteSeq = 0; * (`ENAMETOOLONG` out of `rename`) into a report of a directory-permissions * problem the user would then go and not find. */ -function writeFailureHint(code: string | undefined, filePath: string): string { - const dir = path.dirname(filePath); +function writeFailureHint(code: string | undefined, filePath: string, target: string): string { + // The directory the swap actually uses — `dirname(realpath(filePath))`, not + // `dirname(filePath)`. For a flow file that is a symlink into a shared vault + // those are different directories, and only the first one can be the cause: + // naming the second sent the reader to a `.argent/flows` that is already + // writable while the vault, the only unwritable thing in the picture, went + // unmentioned. Say so when they differ, since "your flows dir is fine, the + // link target is not" is the whole diagnosis there. + const dir = path.dirname(target); + const via = + dir === path.dirname(filePath) + ? "" + : ` (${path.basename(filePath)} is a symlink, so the write lands in ${dir}, not in ${path.dirname(filePath)})`; switch (code) { case "EACCES": case "EPERM": case "EROFS": return ( `an append replaces the file via a sibling temp file and rename, so ${dir} must be ` + - `writable — permission on the flow file itself is not enough.` + `writable — permission on the flow file itself is not enough${via}.` ); case "ENOSPC": case "EDQUOT": - return `the filesystem holding ${dir} is out of space (or over quota).`; + return `the filesystem holding ${dir} is out of space (or over quota)${via}.`; case "ENAMETOOLONG": - return `the flow name makes ${path.basename(filePath)} longer than this filesystem allows — use a shorter name.`; + return `the flow name makes ${path.basename(target)} longer than this filesystem allows — use a shorter name.`; case "ENOENT": - return `${dir} does not exist.`; + return `${dir} does not exist${via}.`; default: - return `an append replaces the file via a sibling temp file and rename in ${dir}.`; + return `an append replaces the file via a sibling temp file and rename in ${dir}${via}.`; } } @@ -2612,7 +2623,7 @@ async function writeFlowFile(filePath: string, content: string): Promise { const errno = err instanceof Error ? (err as NodeJS.ErrnoException) : undefined; const code = typeof errno?.code === "string" ? errno.code : undefined; throw new FailureError( - `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — ${writeFailureHint(code, filePath)}`, + `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — ${writeFailureHint(code, filePath, target)}`, { error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, failure_stage: "flow_file_write", diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 11bd93e1b..3c5806a2c 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -837,6 +837,35 @@ describe("flow-file writes as seen by a concurrent reader", () => { expect(listActiveRecordings()).toEqual([]); }); + it("blames the LINK TARGET's directory when a symlinked flow cannot be written", async () => { + // A 0755 flows dir holding a link into a 0555 vault. The hint used + // `dirname(filePath)` while the temp file and rename use + // `dirname(realpath(filePath))`, so it named a directory that already IS + // writable and never mentioned the vault — the only unwritable thing. + const vault = await makeRoot("hint-vault"); + const root = await makeRoot("hint-proj"); + const shared = path.join(vault, "f.yaml"); + const link = flowPath(root, "f"); + await fs.writeFile(shared, "steps: []\n", "utf8"); + await fs.mkdir(path.dirname(link), { recursive: true }); + await fs.symlink(shared, link); + await start(root, "f"); + await fs.chmod(vault, 0o555); + try { + const err = await addEcho(root, "f", "note").catch((e: unknown) => e); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + const message = formatErrorForAgent(err); + // The real directory is named… + expect(message).toContain(await fs.realpath(vault)); + // …and the reader is told why it is not the one they expected. + expect(message).toContain("is a symlink"); + expect(message).toMatch(/must be writable/); + } finally { + await fs.chmod(vault, 0o755); + } + }); + it("classifies a project_root that is a FILE as a flow write failure, not a registry one", async () => { // The tool description promises it "fails if the .argent/flows/ directory // cannot be created OR the flow file cannot be written". With the mkdir From 65a0d5c14edabdfb24955b4f53ac5f3fd58c6cd5 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:49:05 +0200 Subject: [PATCH 36/98] fix(flow): validate project_root on the flow_path branch again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setActiveProjectRoot` ran unconditionally at the top of `resolveFlowSource`, and its body is exactly today's `assertValidProjectRoot`. Deleting it left the only surviving check inside `getFlowPath`, which the `name` branch alone reaches, so relative and ".."-bearing roots now resolved on the `flow_path` branch — and the JSDoc's "Name and project_root are validated in every branch" stopped being true. No exploit today: project_root is unused on that branch, and `flow_file` is `skipWhenSet: flow_path`. That is why the guardrail is pinned by a test rather than left to be re-derived by whoever next reads project_root there. Also fixes flow-add-step's citation of `setActiveProjectRoot` — a function this PR deleted, and the last reference to it in the repo — to name the check that actually backs the claim. --- .../src/tools/flows/flow-add-step.ts | 3 +- .../tool-server/src/tools/flows/flow-run.ts | 15 ++++++++-- .../test/http-flow-path-boundary.test.ts | 29 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-add-step.ts b/packages/tool-server/src/tools/flows/flow-add-step.ts index 7dac842f4..b83e0db05 100644 --- a/packages/tool-server/src/tools/flows/flow-add-step.ts +++ b/packages/tool-server/src/tools/flows/flow-add-step.ts @@ -216,7 +216,8 @@ async function rewriteSiblingFlowPath( // anchors a relative root at the tool SERVER's cwd, which bears no relation // to the calling agent's, so a relative root would pass or fail by accident // of where the server was started. flow-execute itself demands an absolute - // root (setActiveProjectRoot), so this refuses nothing that could have run. + // root (`assertValidProjectRoot`, called by `resolveFlowSource` before either + // of its branches), so this refuses nothing that could have run. const projectRoot = args.project_root; if (typeof projectRoot !== "string" || !path.isAbsolute(projectRoot)) { throw invalid( diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index cd241038f..96adaa3df 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -22,6 +22,7 @@ import type { import { appIdForPlatform, assertSafeFlowName, + assertValidProjectRoot, chromiumLaunchSpec, classifyOnDiskSpelling, describeSelector, @@ -2215,8 +2216,9 @@ function errMsg(err: unknown): string { * name must then appear in that flow's own directory listing byte-for-byte — a * case-insensitive filesystem opens files under spellings no directory entry * carries, and the name is what keys the report and `__baselines__/` (see - * {@link classifyOnDiskSpelling}). Name and project_root are validated in every - * branch. + * {@link classifyOnDiskSpelling}). Name is validated on the branch that has one; + * project_root is validated up front, before either branch, since only the + * `name` branch would otherwise reach a check. * * Resolution is pure: it reads and mutates no shared state, so replaying a flow * in one project can never rebind the paths of a recording in progress in @@ -2244,6 +2246,15 @@ export async function resolveFlowSource( }); } + // Before either branch, so both are covered. `getFlowPath` validates the root + // on the `name` branch only, and deleting `setActiveProjectRoot` — which ran + // here, unconditionally, and whose body is today's assertValidProjectRoot — + // removed the check on the `flow_path` branch entirely, letting relative and + // ".."-bearing roots through. Nothing reads project_root on that branch + // today, so this restores a guardrail rather than fixing a live exploit; it + // is here so a future reader of it does not have to establish that. + assertValidProjectRoot(params.project_root); + if (params.flow_path !== undefined) { if (flowPathInput?.viaUpload) { throw new FailureError( diff --git a/packages/tool-server/test/http-flow-path-boundary.test.ts b/packages/tool-server/test/http-flow-path-boundary.test.ts index ff18ce3b0..24471803d 100644 --- a/packages/tool-server/test/http-flow-path-boundary.test.ts +++ b/packages/tool-server/test/http-flow-path-boundary.test.ts @@ -184,6 +184,35 @@ describe("flow-execute flow_path over HTTP", () => { } }); + it("still validates project_root on the flow_path branch", async () => { + // `getFlowPath` validates the root, but only the `name` branch reaches it. + // Deleting `setActiveProjectRoot` — which ran unconditionally, ahead of + // both branches — left this branch with no check at all, so a relative or + // ".."-bearing root sailed through. Nothing reads project_root here today, + // which is exactly why the guardrail has to be pinned rather than assumed. + const st = await fs.stat(flowPath); + const wrapper = { + __argentFileInput: true, + path: flowPath, + size: st.size, + mtimeMs: st.mtimeMs, + }; + + for (const [root, expected] of [ + ["relative/root", /project_root must be an absolute path/], + [`${projectRoot}/../elsewhere`, /must not contain "\.\." segments/], + ] as const) { + const res = await supertest(handle.app) + .post("/tools/flow-execute") + .send({ project_root: root, device: DEVICE, flow_path: wrapper }); + + expect(res.status).toBe(500); + expect(res.body.error).toMatch(expected); + expect(res.body.error_code).toBe("FLOW_PROJECT_ROOT_INVALID"); + expect(steps.invokeTool).not.toHaveBeenCalled(); + } + }); + it('rejects a ".." flow_path whose kernel and lexical resolutions disagree', async () => { // /link -> /deep/inner, so the kernel reads /deep/flow.yaml // while path.dirname keeps "/link/.." and path.join collapses it to From 61f27327234c3f69a52293ea9a7b2ee663612bb7 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 14:56:06 +0200 Subject: [PATCH 37/98] docs: correct what the telemetry, swap and start-recording comments claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `http.ts`'s "Latent today: BOTH consumers of this function are gated on the tool declaring a capability" was wrong on both counts. `extractDeviceArg` has three call sites and only the capability gate is gated: `emitHttpFailure` classifies a rejected call straight from `req.body`, and `platformFromArgs` via `deriveChildInvocationMeta` attributes a sub-tool from its own args. The `devices` branch is therefore live today, which is what `registry/src/types.ts` — added by this same PR — already said. The claim was repeated at the capability gate and in http-tools-meta, and both now say what is actually true of each. `deriveChildInvocationMeta`'s own doc still named `udid` as "the only correct source" while its parallel comment in registry/src/types.ts had been updated to include `devices`. flow-utils' atomic-swap rationale was orphaned: commit c95ffe22 inserted `writeFailureHint`'s JSDoc immediately after it, so the swap rationale sat on an errno-string builder while `writeFlowFile` had none and two `{@link writeFlowFile}` references pointed at an undocumented symbol. Moved back onto the function it describes, with the "sibling temp file" wording corrected — the sibling is the resolved TARGET's, which for a symlinked flow is the vault, and that pairing is what makes rename(2) atomic. Its "the one thing that enumerates this directory" also undercounted; the conclusion holds (every site filters on .yaml + FLOW_NAME_PATTERN) but the count did not. `flow-start-recording`'s description asserted unconditionally that it creates the .yaml and fails if the directory cannot be created, which client persist mode does not do — the tool's own code says so at :181-185. Two tests that pinned nothing now pin something: http-tools-meta's "ignores a devices list that holds no usable id" never sent a non-string element, so deleting the `typeof devices[0] === "string"` guard left the suite green (verified: it now fails), and the ungated failure-classification path that makes the branch live had no coverage at all. --- packages/tool-server/src/http.ts | 32 ++++---- .../src/tools/flows/flow-start-recording.ts | 4 +- .../tool-server/src/tools/flows/flow-utils.ts | 74 ++++++++++--------- .../tool-server/test/http-tools-meta.test.ts | 58 +++++++++++++-- 4 files changed, 111 insertions(+), 57 deletions(-) diff --git a/packages/tool-server/src/http.ts b/packages/tool-server/src/http.ts index 35b3f8937..0d926cf6c 100644 --- a/packages/tool-server/src/http.ts +++ b/packages/tool-server/src/http.ts @@ -148,12 +148,14 @@ function extractDeviceArg(data: unknown): string | null { // devices of different platforms; the first is enough for the coarse // telemetry platform. // - // Latent today: BOTH consumers of this function are gated on the tool - // declaring a capability — the gate directly, and `extractInvocationMeta` - // through its `hasCapability` argument — and the one tool that spells the - // parameter this way declares none. Kept so the reading of `devices` is - // defined in one place if a capability-bearing tool ever takes a device list, - // rather than being rediscovered then. + // Live today, not latent. Of this function's three consumers only the + // capability gate is gated — and `stop-all-simulator-servers` declares no + // capability, so `devices` never reaches that one. The other two are ungated + // and do read it: `emitHttpFailure` classifies a rejected call straight from + // `req.body` (a `.strict()` rejection of the `udids` slip is a real 400 on a + // body that carries `devices`), and `platformFromArgs` via + // `deriveChildInvocationMeta` attributes a sub-tool from its own args — which + // for a replayed teardown step is exactly `devices`. if (Array.isArray(record.devices) && typeof record.devices[0] === "string") { return record.devices[0]; } @@ -234,9 +236,12 @@ function platformFromArgs(data: unknown): TelemetryPlatform | null { /** * Attribution for a sub-tool an orchestrator dispatches: the outer request's AI * client is inherited unchanged, but the platform is re-derived from the child's - * OWN device arg. Orchestrators like flow-execute carry no platform (and a flow - * can span several devices), so the child's `udid` is the only correct source; - * the parent's platform is the fallback when the child has no device arg. + * OWN device arg — `udid` / `device_id` / `devices` / `avdName`, whichever it + * spells. Orchestrators like flow-execute carry no platform (and a flow can span + * several devices), so the child's device arg is the only correct source; the + * parent's platform is the fallback when the child has none. A replayed + * `stop-all-simulator-servers` step is the `devices` case, and it resolves here + * rather than falling back. */ function deriveChildInvocationMeta(parentMeta: InvocationMeta, childArgs: unknown): InvocationMeta { const childPlatform = platformFromArgs(childArgs); @@ -758,10 +763,11 @@ export function createHttpApp(registry: Registry, options?: HttpAppOptions): Htt // `extractDeviceArg` honours all three so an Android serial reaching an // iOS-only device_id-tool is rejected at the gate instead of falling // through to the deeper blueprint error (which surfaces as a generic 500). - // Only the first two ever reach this gate — the `devices` tool declares no - // capability, and neither does telemetry read it for the same reason (see - // `extractDeviceArg`). The third is defined here so it behaves like the - // others the day a capability-bearing tool takes a device list. + // Only the first two ever reach THIS gate — the `devices` tool declares + // no capability. That is a fact about the gate alone: telemetry reads + // `devices` today through two ungated consumers (see `extractDeviceArg`). + // The third spelling is honoured here so it behaves like the others the + // day a capability-bearing tool takes a device list. const deviceArg = extractDeviceArg(parsedData); if (def.capability && deviceArg) { try { diff --git a/packages/tool-server/src/tools/flows/flow-start-recording.ts b/packages/tool-server/src/tools/flows/flow-start-recording.ts index 1d18eb57f..beaa8dc8e 100644 --- a/packages/tool-server/src/tools/flows/flow-start-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-start-recording.ts @@ -80,10 +80,10 @@ export const flowStartRecordingTool: ToolDefinition< failedMsg: ({ params, failureSignal }) => `Failed to start recording of flow ${params.name}: ${failureSignal.error_code}`, }, - description: `Start recording a new flow. Creates a .yaml file in the .argent/flows/ directory, replacing any existing one. + description: `Start recording a new flow, resetting .argent/flows/.yaml to an empty flow and replacing any existing one. Use when you want to capture a reusable sequence of device interactions for later replay. Returns { message, flowFile, savedTo } and optionally { restarted, discardedSteps } if a live recording of the same flow was discarded. -Fails if the .argent/flows/ directory cannot be created or the flow file cannot be written. +Whether this server writes that file depends on where your project is: co-located, it creates it and fails if the .argent/flows/ directory cannot be created or the file cannot be written; against a remote tool-server it writes nothing and \`savedTo\` is a directive your client applies (a null \`savedTo\` back means it did not). Several flows can be recorded at once — each keyed by the \`name\` + \`project_root\` that every subsequent recording tool repeats — and one recording's steps never diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 1911430ce..5076904d5 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2484,40 +2484,6 @@ export function parseFlow(content: string): FlowFile { */ let flowWriteSeq = 0; -/** - * Replace a flow file's contents so no reader can ever observe it half-written. - * - * {@link withFlowFileLock} serializes WRITERS, but every reader of a flow YAML - * stays outside it — `flow-execute`'s own load, its `run:` fragment load, - * `flow-read-prerequisite`, `flow-add-step`'s sibling-fragment check — and the - * `argent` CLI reads these files from another process entirely, where an - * in-process lock cannot reach. A plain `fs.writeFile` opens with O_TRUNC, so - * such a reader could land in the window between the truncate and the write and - * parse a truncated file, or an empty one — and `parseFlow("")` yields - * `{ steps: [] }` with no error, which replays as a top-level PASS over zero - * steps. - * - * Writing to a sibling temp file and renaming makes the swap atomic: a reader - * sees either the whole previous file or the whole new one. The temp name is - * dotted and `.tmp`-suffixed so a half-written scratch file can never be - * mistaken for a flow: `getFlowPath` only ever produces `.yaml`, and the - * one thing that enumerates this directory — `argent flow list` — filters on - * that extension. Keep both halves of that agreement if either side changes. - * - * It deliberately does NOT embed the flow name. A flow name has no length cap - * (`FLOW_NAME_PATTERN` constrains the character set only), so `.yaml` can - * legitimately run to NAME_MAX — and prefixing that with a discriminator would - * push the scratch name past the limit, turning an append that used to work - * into ENAMETOOLONG. pid + counter is unique on its own: the counter separates - * writers inside this process, the pid separates this process from a second - * tool-server (a different install bundle) that could be writing the same - * directory. - * - * The swap costs two things a write-through would have kept, both accepted for - * the atomicity: it needs write permission on the DIRECTORY rather than on the - * file, and it replaces the inode, so a chmod on the flow file or a hardlink to - * it does not survive an append. - */ /** * What actually went wrong, per errno. The swap needs write permission on the * DIRECTORY, which is the surprising part and worth stating — but only when @@ -2594,6 +2560,46 @@ async function canonicalFlowPath(filePath: string): Promise { return await fs.realpath(filePath).catch(() => path.join(dir, path.basename(filePath))); } +/** + * Replace a flow file's contents so no reader can ever observe it half-written. + * + * {@link withFlowFileLock} serializes WRITERS, but every reader of a flow YAML + * stays outside it — `flow-execute`'s own load, its `run:` fragment load, + * `flow-read-prerequisite`, `flow-add-step`'s sibling-fragment check — and the + * `argent` CLI reads these files from another process entirely, where an + * in-process lock cannot reach. A plain `fs.writeFile` opens with O_TRUNC, so + * such a reader could land in the window between the truncate and the write and + * parse a truncated file, or an empty one — and `parseFlow("")` yields + * `{ steps: [] }` with no error, which replays as a top-level PASS over zero + * steps. + * + * Writing to a temp file beside the target and renaming makes the swap atomic: + * a reader sees either the whole previous file or the whole new one. Beside the + * TARGET, note — `canonicalFlowPath`'s result — which for a symlinked flow is + * the vault the link points into, not `path.dirname(filePath)`; rename(2) is + * atomic only within one filesystem, and that is the pairing that guarantees it. + * + * The temp name is dotted and `.tmp`-suffixed so a half-written scratch file can + * never be mistaken for a flow: `getFlowPath` only ever produces `.yaml`, + * and every site that enumerates a flows directory — `argent flow list`, + * {@link classifyOnDiskSpelling}, the CLI's recursive suite walk — filters on + * `.yaml` plus `FLOW_NAME_PATTERN`, so none of them can see one. Keep both + * halves of that agreement if either side changes. + * + * It deliberately does NOT embed the flow name. A flow name has no length cap + * (`FLOW_NAME_PATTERN` constrains the character set only), so `.yaml` can + * legitimately run to NAME_MAX — and prefixing that with a discriminator would + * push the scratch name past the limit, turning an append that used to work + * into ENAMETOOLONG. pid + counter is unique on its own: the counter separates + * writers inside this process, the pid separates this process from a second + * tool-server (a different install bundle) that could be writing the same + * directory. + * + * The swap costs two things a write-through would have kept, both accepted for + * the atomicity: it needs write permission on the DIRECTORY rather than on the + * file, and it replaces the inode, so a chmod on the flow file or a hardlink to + * it does not survive an append. + */ async function writeFlowFile(filePath: string, content: string): Promise { const target = await canonicalFlowPath(filePath); const tmpPath = path.join( diff --git a/packages/tool-server/test/http-tools-meta.test.ts b/packages/tool-server/test/http-tools-meta.test.ts index c61be5f04..5b99bb3fa 100644 --- a/packages/tool-server/test/http-tools-meta.test.ts +++ b/packages/tool-server/test/http-tools-meta.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import supertest from "supertest"; +import { z } from "zod"; import { createHttpApp, type HttpAppHandle } from "../src/http"; import type { Registry } from "@argent/registry"; @@ -196,10 +197,12 @@ describe("GET /tools progressive-loading metadata", () => { // pinned above; deleting the `devices` branch of `extractDeviceArg` left // the whole suite green, so a scoped teardown silently lost its platform. // Driven through `device-tool` because `extractInvocationMeta` derives a - // platform only for a tool that declares a capability. `stop-all-simulator- - // servers`, the sole tool spelling `devices` today, declares none — so this - // branch is latent in production and only a capability-bearing tool can - // exercise it. + // platform only for a tool that declares a capability, and + // `stop-all-simulator-servers` — the sole tool spelling `devices` today — + // declares none. That is a fact about THIS consumer: `extractDeviceArg`'s + // other two are ungated and read `devices` in production (a failure + // classified from `req.body`, and a replayed teardown step attributed + // through `deriveChildInvocationMeta`). let seenMeta: Record | undefined; const recordInvocation = vi.fn((_id: string, meta: Record) => { seenMeta = meta; @@ -219,15 +222,54 @@ describe("GET /tools progressive-loading metadata", () => { }); it("ignores a devices list that holds no usable id", async () => { + // `devices: []` and `devices: [123]` must both yield no device arg. The + // empty case alone was a tautology — deleting the + // `typeof record.devices[0] === "string"` guard left the whole suite green, + // because a non-string element was never sent. The schema rejects such a + // call in production, so this is the guard's only exercise. const recordInvocation = vi.fn(() => vi.fn()); handle.dispose(); handle = createHttpApp(stubRegistry(), { recordInvocation }); - await request(handle.app).post("/tools/device-tool").send({ devices: [] }).expect(200); + for (const devices of [[], [123], [null]]) { + recordInvocation.mockClear(); + await request(handle.app).post("/tools/device-tool").send({ devices }).expect(200); + // No device arg, so no platform — and with nothing else to record, no + // invocation metadata at all. + expect(recordInvocation, `devices: ${JSON.stringify(devices)}`).not.toHaveBeenCalled(); + } + }); - // An empty scope yields no device arg, so no platform — and with nothing - // else to record, no invocation metadata at all. - expect(recordInvocation).not.toHaveBeenCalled(); + it("classifies a FAILED call from its devices scope, with no capability in play", async () => { + // `emitHttpFailure` is one of `extractDeviceArg`'s two UNGATED consumers, + // and the one that makes the `devices` branch live in production: a + // rejected `stop-all-simulator-servers` call is classified straight from + // `req.body`, which carries the scope. Driven through a tool that declares + // no capability, exactly like the real one. + const recordFailure = vi.fn(); + const registry = stubRegistry(); + // The real shape: `stop-all-simulator-servers` is `.strict()` precisely so + // the `udids` slip cannot be stripped down to a machine-wide sweep, and + // that rejection is a 400 classified from `req.body` — which carries + // `devices`. No capability anywhere in the path. + (registry.getTool as unknown as ReturnType).mockReturnValue({ + id: "strict-teardown", + description: "Scoped teardown", + inputSchema: { type: "object", properties: { devices: {} } }, + zodSchema: z.object({ devices: z.array(z.string()).optional() }).strict(), + services: () => ({}), + execute: async () => ({}), + }); + handle.dispose(); + handle = createHttpApp(registry, { recordFailure }); + + await request(handle.app) + .post("/tools/strict-teardown") + .send({ devices: ["emulator-5554"], udids: ["oops"] }) + .expect(400); + + expect(recordFailure).toHaveBeenCalled(); + expect(recordFailure.mock.calls[0][1]).toMatchObject({ platform: "android" }); }); it("refines an iOS device to `tvos` when its cached runtime kind is tv", async () => { From 9ee66741e90457332c115012bcc7e874339a6d66 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:17:55 +0200 Subject: [PATCH 38/98] test: pin the changed behaviour the suite was leaving to inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every case below was verified to FAIL against the code it describes before being kept, by mutating that code and re-running — the same check that showed the two it replaces were pinning nothing. - `hadUnretrievedCapture`'s other two arms. Only `recordingActive` was covered; `pendingRetrieval` (the likeliest real sequence — the cap fires, the video is finalized and waiting, the teardown lands in that window) and `startPending` (a start still mid-readiness) both owe the caller a video and now say so. - `wasLive` for STARTING and TERMINATING in the stop-all sweep. Only RUNNING and ERROR were exercised, and these are the two arms that decide whether a caller is told their device was reaped. - The `unmatched` de-duplication across CASE variants. It lowercases to match the lookup, but every existing case repeated an id in one spelling, so mutating `seen` to identity kept all 61 stop-tool tests green. - What the mock registry's cascade recursion is actually for. The case that claimed to pin a cascaded dependent matched the device directly, so removing the recursion changed nothing; it is reframed as the insertion-order case it really is, and a dependent this tool does NOT match by device is added — it dies with its dependency but must not be reported in `stopped`. - `reaped-sessions`' key semantics, which no test reached: kind-scoping (one teardown reaps all three of a device's capture services, and each owner reads back separately) and case folding (matching every device-id lookup in the stop tools), plus that the message names the disposer's spelling, not the reader's. - The `takeReapedSession` clear at profiler start, driven through the real `startNativeProfilerAndroid` rather than called directly, including that it leaves another device's breadcrumb alone. - Both react-profiler message rewrites. Reverting either to main's wording left `test/react-profiler/**` entirely green; each now fails. - http-tools-meta's `devices` guard, which never saw a non-string element, and the ungated failure-classification path that makes the branch live. --- ...e-profiler-start-clears-breadcrumb.test.ts | 97 +++++++++++++++++++ .../test/react-profiler/session-owner.test.ts | 40 +++++++- .../react-profiler/status-ownership.test.ts | 25 +++++ .../tool-server/test/reaped-sessions.test.ts | 75 ++++++++++++++ .../tool-server/test/screen-recording.test.ts | 58 +++++++++++ packages/tool-server/test/stop-tools.test.ts | 86 +++++++++++++--- 6 files changed, 369 insertions(+), 12 deletions(-) create mode 100644 packages/tool-server/test/native-profiler-start-clears-breadcrumb.test.ts create mode 100644 packages/tool-server/test/reaped-sessions.test.ts diff --git a/packages/tool-server/test/native-profiler-start-clears-breadcrumb.test.ts b/packages/tool-server/test/native-profiler-start-clears-breadcrumb.test.ts new file mode 100644 index 000000000..843dd79e2 --- /dev/null +++ b/packages/tool-server/test/native-profiler-start-clears-breadcrumb.test.ts @@ -0,0 +1,97 @@ +/** + * A teardown breadcrumb explains ONE confusing answer: the "no active session" + * a reaped capture's own stop would otherwise get. A start that succeeds after + * the teardown means that stop will succeed instead, so the breadcrumb is never + * consumed by the read it was left for — and would sit in the process-global + * map until some genuinely unrelated "no active session", possibly much later, + * collected it and blamed a teardown that had nothing to do with it. + * + * Both platform starts clear it for that reason. Only the stop-side consume was + * covered; this drives the real `startNativeProfilerAndroid` (perfetto, adb and + * the debug dir stubbed at their module boundaries) so the clear is exercised + * where it actually lives rather than called directly. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { EventEmitter } from "events"; +import type { ChildProcess } from "child_process"; +import type { DeviceInfo } from "@argent/registry"; +import * as os from "node:os"; + +vi.mock("../src/utils/adb", () => ({ adbShell: vi.fn(async () => "") })); +vi.mock("@argent/native-devtools-android", () => ({ + disposeWarmEngine: vi.fn(async () => {}), + TraceProcessorUnavailableError: class extends Error {}, +})); +vi.mock("../src/utils/android-profiler/detect-app", () => ({ + detectAndroidRunningApp: vi.fn(async () => "com.example.app"), + validateAndroidAppProcess: vi.fn(async () => {}), +})); +vi.mock("../src/utils/react-profiler/debug/dump", () => ({ + getDebugDir: vi.fn(async () => os.tmpdir()), +})); +vi.mock("../src/utils/android-profiler/capture", () => ({ + startPerfetto: vi.fn(async () => ({ + pid: 4242, + onDeviceTracePath: "/data/misc/perfetto-traces/fake.pftrace", + child: new EventEmitter() as unknown as ChildProcess, + })), + stopPerfetto: vi.fn(async () => {}), +})); + +import { + nativeProfilerSessionBlueprint, + type NativeProfilerSessionApi, +} from "../src/blueprints/native-profiler-session"; +import { + startNativeProfilerAndroid, + stopNativeProfilerAndroid, +} from "../src/tools/profiler/native-profiler/platforms/android"; +import { + recordReapedSession, + takeReapedSession, + __resetReapedSessionsForTesting, +} from "../src/utils/reaped-sessions"; + +const androidDevice = { id: "emulator-5554", platform: "android" } as DeviceInfo; + +async function session(): Promise { + const instance = await nativeProfilerSessionBlueprint.factory({}, androidDevice, { + device: androidDevice, + } as never); + return instance.api as NativeProfilerSessionApi; +} + +beforeEach(() => { + __resetReapedSessionsForTesting(); +}); + +describe("native-profiler-start after a teardown", () => { + it("clears the breadcrumb, so a later unrelated absence is not blamed on it", async () => { + recordReapedSession("native-profiler", androidDevice.id, "salvage note"); + + const api = await session(); + await startNativeProfilerAndroid(api, { device_id: androidDevice.id }); + expect(api.profilingActive).toBe(true); + + // Nothing is left for a later read to pick up… + expect(takeReapedSession("native-profiler", androidDevice.id)).toBeUndefined(); + + // …so a genuine "no active session" much later stays a plain absence. + const fresh = await session(); + const err = await stopNativeProfilerAndroid(fresh).catch((e: unknown) => e); + expect((err as Error).message).toBe( + "No active native profiling session found. Call native-profiler-start first." + ); + }); + + it("leaves another device's breadcrumb alone", async () => { + // The clear is scoped to the device the start ran on. Clearing broadly + // would silently disarm the explanation another agent's reaped capture is + // still owed. + recordReapedSession("native-profiler", "emulator-5556", "other device"); + + await startNativeProfilerAndroid(await session(), { device_id: androidDevice.id }); + + expect(takeReapedSession("native-profiler", "emulator-5556")).toBeDefined(); + }); +}); diff --git a/packages/tool-server/test/react-profiler/session-owner.test.ts b/packages/tool-server/test/react-profiler/session-owner.test.ts index 25362ab53..4b6da459f 100644 --- a/packages/tool-server/test/react-profiler/session-owner.test.ts +++ b/packages/tool-server/test/react-profiler/session-owner.test.ts @@ -4,7 +4,11 @@ import { DEFAULT_STALE_THRESHOLD_MS, type ProfilerSessionOwner, } from "../../src/utils/react-profiler/session-ownership"; -import { flattenProfilingData } from "../../src/tools/profiler/react/react-profiler-stop"; +import { + flattenProfilingData, + createReactProfilerStopTool, +} from "../../src/tools/profiler/react/react-profiler-stop"; +import { FAILURE_CODES, getFailureSignal, type Registry } from "@argent/registry"; import { buildHotCommitSummaries } from "../../src/utils/react-profiler/pipeline/00-hot-commits"; import type { DevToolsFiberCommit, @@ -359,3 +363,37 @@ describe("buildHotCommitSummaries (unattributed threading)", () => { expect(summaries[0]!.unattributedFiberCount).toBeUndefined(); }); }); + +// ── The absent-session message ──────────────────────────────────────── + +/** + * A react-profiler session rides on the device's JS-runtime debugger, which + * `stop-all-simulator-servers` reaps — so a teardown (commonly another agent's, + * since one tool-server serves every agent using an install) is a live cause of + * "no active profiling session", alongside the Metro reload that used to be the + * only one named. Nothing pinned the wording, so reverting it left the whole + * react-profiler suite green. + */ +describe("react-profiler-stop with no live session", () => { + it("names the teardown as a cause, not just a Metro reload", async () => { + const registry = { + getSnapshot: () => ({ services: new Map(), namespaces: [], tools: [] }), + resolveService: async () => { + throw new Error("must not resolve"); + }, + } as unknown as Registry; + + const err = await createReactProfilerStopTool(registry).execute!( + {}, + { port: 8081, device_id: "emulator-5554" } + ).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.REACT_PROFILER_NO_ACTIVE_SESSION); + expect(message).toContain("stop-all-simulator-servers"); + expect(message).toContain("JS-runtime debugger"); + // The pre-existing cause and the recovery both survive. + expect(message).toContain("Metro reload"); + expect(message).toContain("Call react-profiler-start"); + }); +}); diff --git a/packages/tool-server/test/react-profiler/status-ownership.test.ts b/packages/tool-server/test/react-profiler/status-ownership.test.ts index db92d1b45..bb324366e 100644 --- a/packages/tool-server/test/react-profiler/status-ownership.test.ts +++ b/packages/tool-server/test/react-profiler/status-ownership.test.ts @@ -121,6 +121,31 @@ describe("react-profiler-status: server-side ownership", () => { expect(res.current_session_id).toBe("uuid-stranger"); }); + it("names a teardown among the causes of a taken-over session", async () => { + // A react-profiler session rides on the device's JS-runtime debugger, which + // `stop-all-simulator-servers` reaps — and one tool-server serves every + // agent using this install, so the takeover is commonly another agent's + // teardown rather than a second tool-server or a restart. The note is the + // only place an agent learns that; reverting it to the previous wording + // left every react-profiler test green. + const api = buildApi({ + sessionId: "uuid-mine", + state: { + hookExists: true, + rendererInterfaceFound: true, + isRunning: true, + owner: buildOwner("uuid-stranger"), + }, + }); + const res = await runStatus(api); + expect(res.session_status).toBe("taken_over"); + expect(res.note).toContain("stop-all-simulator-servers"); + expect(res.note).toContain("JS-runtime debugger"); + // The pre-existing causes are still offered, and so is the way out. + expect(res.note).toContain("another tool-server instance took over"); + expect(res.note).toContain("force: true"); + }); + it("returns 'stopped' when no session is running, regardless of api.sessionId", async () => { const api = buildApi({ sessionId: "uuid-mine", diff --git a/packages/tool-server/test/reaped-sessions.test.ts b/packages/tool-server/test/reaped-sessions.test.ts new file mode 100644 index 000000000..81aec4add --- /dev/null +++ b/packages/tool-server/test/reaped-sessions.test.ts @@ -0,0 +1,75 @@ +/** + * The breadcrumb store's key semantics. Three tools read it — screen-recording + * stop, native-profiler stop, debugger-log-registry — and each was tested only + * against its own kind and its own single spelling, so nothing pinned what the + * key itself does: scope by kind, and fold case the way every device-id lookup + * in the stop tools does. + */ +import { describe, it, expect, beforeEach } from "vitest"; +import { + recordReapedSession, + takeReapedSession, + describeReapedSession, + __resetReapedSessionsForTesting, +} from "../src/utils/reaped-sessions"; + +const UDID = "6DBF83B4-0000-0000-0000-000000000000"; + +beforeEach(() => { + __resetReapedSessionsForTesting(); +}); + +describe("the reaped-session key", () => { + it("scopes by kind, so one device's three captures do not collide", () => { + // A teardown reaps all three of a device's capture services at once, and + // each owner reads back separately. An unscoped key would let the + // screen-recording read consume the profiler's explanation. + recordReapedSession("screen-recording", UDID, "the video"); + recordReapedSession("native-profiler", UDID, "the trace"); + recordReapedSession("js-runtime-debugger", UDID, "the console log"); + + expect(takeReapedSession("screen-recording", UDID)?.salvage).toBe("the video"); + // …and taking one leaves the other two intact. + expect(takeReapedSession("native-profiler", UDID)?.salvage).toBe("the trace"); + expect(takeReapedSession("js-runtime-debugger", UDID)?.salvage).toBe("the console log"); + }); + + it("folds case, so a device read back in another spelling still finds it", () => { + // Device ids reach the two sides from different places — an iOS UDID comes + // back uppercase from simctl and lowercase from some tool args — and every + // id lookup in the stop tools already compares case-insensitively. A + // case-sensitive key here would silently strand the explanation. + recordReapedSession("native-profiler", UDID.toUpperCase(), "the trace"); + + expect(takeReapedSession("native-profiler", UDID.toLowerCase())).toBeDefined(); + // Consumed once, whichever spelling asked. + expect(takeReapedSession("native-profiler", UDID.toUpperCase())).toBeUndefined(); + }); + + it("reports the device id in the spelling the DISPOSER used, not the reader's", () => { + // The message names the device; it must name the one the teardown actually + // reaped rather than echoing back whatever the reader happened to type. + recordReapedSession("screen-recording", UDID.toUpperCase()); + + const entry = takeReapedSession("screen-recording", UDID.toLowerCase())!; + expect(describeReapedSession(entry, "screen recording")).toContain(UDID.toUpperCase()); + }); + + it("keeps the newest record when one kind+device is reaped twice", () => { + recordReapedSession("screen-recording", UDID, "first"); + recordReapedSession("screen-recording", UDID, "second"); + + expect(takeReapedSession("screen-recording", UDID)?.salvage).toBe("second"); + expect(takeReapedSession("screen-recording", UDID)).toBeUndefined(); + }); + + it("omits the salvage clause entirely when nothing survived", () => { + recordReapedSession("native-profiler", UDID); + + const entry = takeReapedSession("native-profiler", UDID)!; + expect(entry.salvage).toBeUndefined(); + const message = describeReapedSession(entry, "native profiling session"); + expect(message).toContain("It was not a session that never started."); + expect(message).toMatch(/never started\.$/); + }); +}); diff --git a/packages/tool-server/test/screen-recording.test.ts b/packages/tool-server/test/screen-recording.test.ts index a7932f2b6..17e1fb095 100644 --- a/packages/tool-server/test/screen-recording.test.ts +++ b/packages/tool-server/test/screen-recording.test.ts @@ -321,6 +321,64 @@ describe("screen-recording session blueprint", () => { ); }); + it("tells the owner when the teardown hit a CAPPED capture awaiting retrieval", async () => { + // `hadUnretrievedCapture` has three arms and only `recordingActive` was + // covered. This is the likeliest real sequence of the three: the time + // limit fires, the video is finalized and waiting to be handed over, and + // the teardown lands in that window. The caller is owed a video just as + // much as in the mid-capture case. + const instance = await screenRecordingSessionBlueprint.factory({}, iosDevice, { + device: iosDevice, + } as never); + fakeStream(); + fakeChild().exitOnStdinEnd(); + await startAndSettle(instance.api, { timeLimitSeconds: 5 }); + await vi.advanceTimersByTimeAsync(5_000); + expect(instance.api.recordingActive).toBe(false); + expect(instance.api.pendingRetrieval).toBe(true); + const output = instance.api.outputFile!; + + await instance.dispose(); + const err = await stopCapture(await makeSession(iosDevice)).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).not.toMatch(/Call `screen-recording-start` first/); + expect(message).toContain("torn down"); + expect(message).toContain(output); + expect(getFailureSignal(err)?.error_code).toBe( + FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN + ); + }); + + it("tells the owner when the teardown hit a start still mid-readiness", async () => { + // The third arm. `startPending` is set synchronously before start's first + // await, so a teardown here destroys a capture whose child may already be + // spawned — reported as a teardown, not as "you never started one". + const instance = await screenRecordingSessionBlueprint.factory({}, iosDevice, { + device: iosDevice, + } as never); + fakeStream(); + fakeChild(); + const pending = startCapture(instance.api, { + streamUrl: STREAM_URL, + timeLimitSeconds: 180, + watermark: false, + trimStatic: false, + }); + pending.catch(() => {}); + expect(instance.api.startPending).toBe(true); + + await instance.dispose(); + await pending.catch(() => {}); + const err = await stopCapture(await makeSession(iosDevice)).catch((e: unknown) => e); + + expect((err as Error).message).not.toMatch(/Call `screen-recording-start` first/); + expect((err as Error).message).toContain("torn down"); + expect(getFailureSignal(err)?.error_code).toBe( + FAILURE_CODES.SCREEN_RECORDING_SERVER_SHUTTING_DOWN + ); + }); + it("still reports a plain absence when no capture was reaped", async () => { // The breadcrumb must not turn every "you never started one" into an // accusation: disposing an idle session leaves nothing behind. diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 1d2fe8a97..467b824f7 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -316,22 +316,49 @@ describe("stop-all-simulator-servers", () => { it.each([ ["debugger-connect first (dependent inserted first)", [CHROMIUM_DEBUGGER, CDP]], ["boot/describe first (dependency inserted first)", [CDP, CHROMIUM_DEBUGGER]], - ])("names a cascading debugger in `stopped` — %s", async (_label, order) => { - const services = new Map( - order.map((urn) => [urn, urn === CDP ? cdpWithDependent() : live()] as const) - ); + ])( + "names a chromium debugger in `stopped` whichever order it was inserted — %s", + async (_label, order) => { + // Both URNs carry the device id, so each is matched DIRECTLY; the + // cascade is incidental here and this case is about insertion order not + // changing membership. What the cascade alone decides is pinned below. + const services = new Map( + order.map((urn) => [urn, urn === CDP ? cdpWithDependent() : live()] as const) + ); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: ["chromium-cdp-9222"] }); + + // Order follows the snapshot; membership must not. + expect((result as { stopped: string[] }).stopped.slice().sort()).toEqual( + [CDP, CHROMIUM_DEBUGGER].sort() + ); + expect(result).not.toHaveProperty("unmatched"); + expect(services.get(CHROMIUM_DEBUGGER)?.state).toBe(ServiceState.IDLE); + expect(services.get(CDP)?.state).toBe(ServiceState.IDLE); + } + ); + + it("takes a non-device dependent down with its dependency without claiming to have reaped it", async () => { + // The distinction the mock's recursion exists for, and the one the case + // above cannot make: a dependent this tool does NOT match by device. It + // still dies — the registry cascades — but it is somebody else's + // dependent, not something the teardown reaped by name, so it must not + // appear in `stopped`. Reporting it there would tell an agent a + // device-scoped teardown deliberately killed its Metro session. + const METRO = "Metro:8081"; + const services = new Map([ + [CDP, { state: ServiceState.RUNNING, dependents: [METRO] }], + [METRO, { state: ServiceState.RUNNING, dependents: [] as string[] }], + ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); const result = await tool.execute!({}, { devices: ["chromium-cdp-9222"] }); - // Order follows the snapshot; membership must not. - expect((result as { stopped: string[] }).stopped.slice().sort()).toEqual( - [CDP, CHROMIUM_DEBUGGER].sort() - ); - expect(result).not.toHaveProperty("unmatched"); - expect(services.get(CHROMIUM_DEBUGGER)?.state).toBe(ServiceState.IDLE); - expect(services.get(CDP)?.state).toBe(ServiceState.IDLE); + expect(result).toEqual({ stopped: [CDP] }); + expect(services.get(METRO)?.state).toBe(ServiceState.IDLE); }); it("returns empty list when no simulators are running", async () => { @@ -376,6 +403,27 @@ describe("stop-all-simulator-servers", () => { expect(registry.disposeService).toHaveBeenCalledTimes(2); }); + it("reports a STARTING node as stopped and a TERMINATING one as not, in the sweep", async () => { + // `wasLive` is `isLiveServiceState` — RUNNING or STARTING. The sweep's use + // of it was only ever exercised for RUNNING and ERROR; STARTING (a server + // mid-boot, which really is being killed) and TERMINATING (already on its + // way down, so nothing here stopped it) are the two arms that decide + // whether a caller is told their device was reaped. + const services = new Map([ + ["SimulatorServer:STARTING-ONE", { state: ServiceState.STARTING, dependents: [] }], + ["SimulatorServer:TERMINATING-ONE", { state: ServiceState.TERMINATING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, {}); + + expect(result).toEqual({ stopped: ["SimulatorServer:STARTING-ONE"] }); + // Both are disposed — the point is what gets REPORTED, not what gets cleaned. + expect(registry.disposeService).toHaveBeenCalledWith("SimulatorServer:TERMINATING-ONE"); + expect(registry.disposeService).toHaveBeenCalledTimes(2); + }); + it("stops the focus-driven TV control services (Apple TV + Android TV)", async () => { // The TvControl daemon owns the spawned tvos-ax / tvos-hid processes, so a // session-end stop must dispose it — not just the simulator-server/CDP nodes. @@ -984,6 +1032,22 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); }); + it("names a repeated missing id only once across CASE variants too", async () => { + // The de-duplication lowercases, matching the lookup — but every case above + // repeats an id in one spelling, so mutating `seen` to identity kept the + // whole stop-tool suite green. Two spellings of one wrong id are one + // mistake, and it is reported in the caller's FIRST spelling. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: ["GHOST-9999", "ghost-9999"] }); + + expect(result).toEqual({ stopped: [], unmatched: ["GHOST-9999"] }); + }); + it("reports neither spelling when one device is named twice in different cases", async () => { const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], From a91aa82f65ef37d70fe4122703d6adb679288a9e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:17:55 +0200 Subject: [PATCH 39/98] fix(flow): reject rather than throw synchronously from resolveFlowKey It returns a Promise, so `getFlowPath`'s validation throws belong in the rejection like every other failure on this path. Every call site today is an async function that converts them anyway; this removes the footgun for the next one. --- packages/tool-server/src/tools/flows/flow-utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 5076904d5..2f3e29ae6 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -155,7 +155,9 @@ export function getFlowPath(projectRoot: string, name: string): string { * when the file is on another machine. Two clients that share a flow file * across that boundary are beyond this process's reach, as they were before. */ -export function resolveFlowKey(projectRoot: string, name: string): Promise { +// `async`, so `getFlowPath`'s validation throws land as a rejection like every +// other failure here rather than synchronously out of a promise-returning call. +export async function resolveFlowKey(projectRoot: string, name: string): Promise { const spelled = getFlowPath(projectRoot, name); const inFlight = keyResolutions.get(spelled); if (inFlight) return inFlight; From 0c5ab0bb04dded32025a245b6c0f88b76af35459 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 16:45:09 +0200 Subject: [PATCH 40/98] fix(stop): name the debugger sessions a device scope cannot reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With two or more devices on one Metro, `debugger-connect` refuses a udid or serial and instructs the caller to re-target with the `logicalDeviceId` Metro echoed. That id then keys the JsRuntimeDebugger URN, so a teardown scoped to `list-devices` ids can never match it — and because the caller's serial still matches that device's other services, it is not reported `unmatched` either. A teardown leaving a CDP socket, a bound loopback console server and a log file handle behind read as a clean machine. Record the case where the connect id IS the logicalDeviceId (the one place both are compared) and report those live sessions as `left_running`. A session another agent opened with its own serial is deliberately not named: that id is one a scope could have supplied, and reporting it would invite the cross-agent teardown the `devices` scope exists to prevent. The tool description and the metro-debugger skill now say to pass the logicalDeviceId alongside the device id, which does reap the session. --- .../skills/argent-metro-debugger/SKILL.md | 2 + .../src/blueprints/js-runtime-debugger.ts | 15 +- .../src/tools/simulator/device-services.ts | 43 +++- .../simulator/stop-all-simulator-servers.ts | 58 +++++- .../src/utils/debugger/device-alias.ts | 41 ++++ packages/tool-server/test/stop-tools.test.ts | 193 +++++++++++++++++- 6 files changed, 342 insertions(+), 10 deletions(-) diff --git a/packages/skills/skills/argent-metro-debugger/SKILL.md b/packages/skills/skills/argent-metro-debugger/SKILL.md index a4a04196d..67cb9dd39 100644 --- a/packages/skills/skills/argent-metro-debugger/SKILL.md +++ b/packages/skills/skills/argent-metro-debugger/SKILL.md @@ -27,6 +27,8 @@ All tools accept `port` (default 8081) AND `device_id` (the iOS Simulator UDID, One Metro port can serve multiple connected devices (e.g. two simulators on `localhost:8081`, or an iOS simulator alongside an Android emulator with `adb reverse` set up). `device_id` pins every debugger/network/profiler call to a specific device so sessions do not collide. +With two or more devices on one Metro, `debugger-connect` refuses a udid/serial and hands back the `logicalDeviceId` to re-target with. That id then keys the session — including for teardown. **Pass it in `stop-all-simulator-servers`' `devices` alongside the device id**, or the session survives your session end holding its CDP socket, console server and log file. The teardown reports what it could not reach in `left_running`; re-call with the id it names. + ### Connect & diagnostics | Tool | Purpose | diff --git a/packages/tool-server/src/blueprints/js-runtime-debugger.ts b/packages/tool-server/src/blueprints/js-runtime-debugger.ts index 4d8d72db9..2e99bb6e4 100644 --- a/packages/tool-server/src/blueprints/js-runtime-debugger.ts +++ b/packages/tool-server/src/blueprints/js-runtime-debugger.ts @@ -9,7 +9,12 @@ import { discoverMetro } from "../utils/debugger/discovery"; import { classifyDevice } from "../utils/device-info"; import { proxyStart } from "../utils/sim-remote"; import { selectTarget } from "../utils/debugger/target-selection"; -import { rememberDeviceAlias, forgetDeviceAlias } from "../utils/debugger/device-alias"; +import { + rememberDeviceAlias, + forgetDeviceAlias, + rememberLogicalKeyedDevice, + forgetLogicalKeyedDevice, +} from "../utils/debugger/device-alias"; import { recordReapedSession } from "../utils/reaped-sessions"; import { CDPClient, type ConsoleAPICalledParams } from "../utils/debugger/cdp-client"; import { createSourceResolver, type SourceResolver } from "../utils/debugger/source-resolver"; @@ -266,6 +271,13 @@ export const jsRuntimeDebuggerBlueprint: ServiceBlueprint(); @@ -310,6 +322,7 @@ export const jsRuntimeDebuggerBlueprint: ServiceBlueprint JsRuntimeDebugger:`, so neither // can be in a snapshot without it and neither adds any ownership the debugger @@ -225,3 +226,43 @@ export function deviceIdOwningUrn( export function isDeviceServiceUrn(urn: string, namespaces: readonly string[]): boolean { return namespaces.some((ns) => urn.startsWith(`${ns}:`)); } + +/** + * The device-id portion of `urn` under whichever of `namespaces` owns it, in + * that namespace's own URN shape — the same reading {@link deviceIdOwningUrn} + * matches against, minus the caller's id list. Undefined when no namespace in + * the set prefixes it. + */ +export function deviceIdOfUrn(urn: string, namespaces: readonly string[]): string | undefined { + for (const namespace of namespaces) { + const portion = deviceIdPortion(urn, namespace); + if (portion !== undefined) return portion; + } + return undefined; +} + +/** + * Of `urns`, the port-keyed sessions no device-scoped teardown could ever name, + * whatever ids it was given. + * + * `JsRuntimeDebugger`'s URN embeds the id the caller CONNECTED with, and on a + * Metro serving two or more devices that cannot be a UDID or serial: + * `selectTarget` refuses to guess which target a device id means and instructs + * the caller to re-target with the `logicalDeviceId` Metro echoed — an opaque + * per-connection handle `list-devices` never mints, and the only id that then + * resolves the session. A teardown scoped to real device ids therefore leaves + * that session holding its CDP socket to Metro, a bound loopback console + * server and a log file handle; and because the caller's serial still matches + * that device's OTHER services, the serial is not reported `unmatched` either, + * so the whole thing reads as a clean machine. + * + * Which ids those are is not inferred from the URN — it is recorded by the + * connect that minted it, the one place both ids are known at once (see + * {@link isLogicalKeyedDevice}). A session another agent opened with its own + * serial is therefore NOT reported: that id is one `list-devices` hands out, so + * a scope could have named it, and a session left on someone else's device is + * that agent's business rather than a scope that cannot express itself. + */ +export function unnameableSessionUrns(urns: readonly string[]): string[] { + return urns.filter((urn) => isLogicalKeyedDevice(deviceIdOfUrn(urn, PORT_KEYED_NAMESPACES))); +} diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index f522471b3..73a221c26 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -1,7 +1,13 @@ import { z } from "zod"; import { ServiceState, isLiveServiceState } from "@argent/registry"; import type { Registry, ToolDefinition } from "@argent/registry"; -import { DEVICE_OWNED_NAMESPACES, deviceIdOwningUrn, isDeviceServiceUrn } from "./device-services"; +import { + DEVICE_OWNED_NAMESPACES, + PORT_KEYED_NAMESPACES, + deviceIdOwningUrn, + isDeviceServiceUrn, + unnameableSessionUrns, +} from "./device-services"; const zodSchema = z .object({ @@ -27,7 +33,10 @@ const zodSchema = z export function createStopAllSimulatorServersTool( registry: Registry -): ToolDefinition, { stopped: string[]; unmatched?: string[] }> { +): ToolDefinition< + z.infer, + { stopped: string[]; unmatched?: string[]; left_running?: string[] } +> { return { id: "stop-all-simulator-servers", interaction: { @@ -48,16 +57,30 @@ export function createStopAllSimulatorServersTool( // that — "Stopped 0 simulator servers" for a teardown that reaped // nothing because every id was wrong. const unmatched = result.unmatched; - return unmatched?.length - ? `${base} (${unmatched.length} supplied ${unmatched.length === 1 ? "id" : "ids"} matched no service)` - : base; + const notes: string[] = []; + if (unmatched?.length) { + notes.push( + `${unmatched.length} supplied ${unmatched.length === 1 ? "id" : "ids"} matched no service` + ); + } + // Same reason as `unmatched`: a debugger session keyed by an id no + // device scope can name is still a session left holding a CDP socket + // and a bound port, and silence about it reads as a clean machine. + const left = result.left_running; + if (left?.length) { + notes.push( + `${left.length} debugger ${left.length === 1 ? "session" : "sessions"} left running` + ); + } + return notes.length ? `${base} (${notes.join("; ")})` : base; }, failedMsg: ({ failureSignal }) => `Failed to stop simulator servers: ${failureSignal.error_code}`, }, description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions, and JS-runtime debugger sessions along with the network inspectors and React profiler sessions that ride on them - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — one tool-server serves every agent, subagent and CLI call using this argent install, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. -Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Past the schema - which rejects an unknown key outright, so the \`udids\` slip is an error rather than a silent machine-wide sweep - the call always succeeds; reaping nothing is a result, not a failure.`, +A JS-runtime debugger session is keyed by the id you called \`debugger-connect\` with. On a Metro serving two or more devices that id is not a udid or serial - connect refuses those and tells you to re-target with the \`logicalDeviceId\` it returns - so a scope built from \`list-devices\` ids cannot reach that session. Pass any such \`logicalDeviceId\` in \`devices\` ALONGSIDE the device id; { left_running } names the ones you missed. +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. { left_running } lists live debugger sessions (and the network inspectors / React profiler sessions riding on them) whose id no device scope can name - re-call with that id to reap them. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Past the schema - which rejects an unknown key outright, so the \`udids\` slip is an error rather than a silent machine-wide sweep - the call always succeeds; reaping nothing is a result, not a failure.`, zodSchema, services: () => ({}), async execute(_services, params) { @@ -69,6 +92,10 @@ Returns { stopped } - the URNs of the services that were actually live and got s const snapshot = registry.getSnapshot(); const stopped: string[] = []; const matchedIds = new Set(); + // Live device-owned services this scope did NOT claim. Only the port-keyed + // ones are ever reported (see `unnameableSessionUrns`) — the rest are + // other agents' devices, which a scoped stop leaves alone by design. + const survivors: string[] = []; for (const [urn, entry] of snapshot.services) { const matchedId = scoped ? deviceIdOwningUrn(urn, DEVICE_OWNED_NAMESPACES, devices) @@ -90,6 +117,12 @@ Returns { stopped } - the URNs of the services that were actually live and got s const wasLive = isLiveServiceState(entry.state); await registry.disposeService(urn); if (wasLive) stopped.push(urn); + } else if ( + scoped && + isLiveServiceState(entry.state) && + isDeviceServiceUrn(urn, PORT_KEYED_NAMESPACES) + ) { + survivors.push(urn); } } if (!scoped) return { stopped }; @@ -107,7 +140,18 @@ Returns { stopped } - the URNs of the services that were actually live and got s seen.add(key); return true; }); - return unmatched.length > 0 ? { stopped, unmatched } : { stopped }; + // A debugger session opened against a multi-device Metro is keyed by the + // `logicalDeviceId` Metro echoed, which no `list-devices` id equals — so + // no `devices` scope can reap it, and the ids that DID match keep it out + // of `unmatched`. Name it, so the caller can pass that id (which does + // reap it) instead of reading silence as a clean machine. + const leftRunning = unnameableSessionUrns(survivors); + const result: { stopped: string[]; unmatched?: string[]; left_running?: string[] } = { + stopped, + }; + if (unmatched.length > 0) result.unmatched = unmatched; + if (leftRunning.length > 0) result.left_running = leftRunning; + return result; }, }; } diff --git a/packages/tool-server/src/utils/debugger/device-alias.ts b/packages/tool-server/src/utils/debugger/device-alias.ts index 8e6fe0a76..e70fe21d8 100644 --- a/packages/tool-server/src/utils/debugger/device-alias.ts +++ b/packages/tool-server/src/utils/debugger/device-alias.ts @@ -58,7 +58,48 @@ export function forgetDeviceAlias(logicalDeviceId: string | undefined): void { if (logicalDeviceId) logicalIdToConnectId.delete(logicalDeviceId); } +/** + * Connect ids that ARE a Metro `logicalDeviceId` — the case the alias above has + * nothing to record, because the two ids are the same string. + * + * It happens whenever two or more devices share one Metro: `selectTarget` + * refuses to guess which target a udid or serial means and tells the caller to + * re-target with the logicalDeviceId, so that is what the debugger service ends + * up keyed by. Nothing joins such an id back to a device — Metro never sees the + * udid — so a teardown scoped to `list-devices` ids cannot reach the session, + * and its serial still matches that device's other services, so the miss is + * invisible. `stop-all-simulator-servers` reads this to say so. + * + * Recorded at connect, which is the only place the two ids are compared, and + * dropped on dispose alongside the alias. + */ +const logicalKeyedConnectIds = new Set(); + +/** + * Note that `connectDeviceId` is itself the `logicalDeviceId` Metro echoed, so + * no device-scoped teardown can name the session it keys. No-op otherwise. + */ +export function rememberLogicalKeyedDevice( + logicalDeviceId: string | undefined, + connectDeviceId: string +): void { + if (logicalDeviceId && logicalDeviceId === connectDeviceId) { + logicalKeyedConnectIds.add(connectDeviceId.toLowerCase()); + } +} + +/** Whether `deviceId` keys a session only its logicalDeviceId can address. */ +export function isLogicalKeyedDevice(deviceId: string | undefined): boolean { + return deviceId !== undefined && logicalKeyedConnectIds.has(deviceId.toLowerCase()); +} + +/** Drop the logical-keyed marker when its debugger connection is disposed. */ +export function forgetLogicalKeyedDevice(connectDeviceId: string): void { + logicalKeyedConnectIds.delete(connectDeviceId.toLowerCase()); +} + /** Test-only: clear all learned aliases. */ export function resetDeviceAliases(): void { logicalIdToConnectId.clear(); + logicalKeyedConnectIds.clear(); } diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 467b824f7..72ea3fd7c 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -1,4 +1,9 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + forgetLogicalKeyedDevice, + rememberLogicalKeyedDevice, + resetDeviceAliases, +} from "../src/utils/debugger/device-alias"; import type { z } from "zod"; import { Registry, ServiceState, zodObjectToJsonSchema } from "@argent/registry"; import { createStopSimulatorServerTool } from "../src/tools/simulator/stop-simulator-server"; @@ -1103,6 +1108,158 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); }); +describe("stop-all-simulator-servers left_running", () => { + // With two or more devices on one Metro, `debugger-connect` refuses a udid / + // serial and instructs the caller to re-target with the `logicalDeviceId` + // Metro echoed. That id keys the session's URN, and no `list-devices` id + // equals it — so no `devices` scope can reap the CDP socket, the bound + // loopback console server or the log file handle it holds. Worse, the + // caller's real serial DOES match that device's other services, so it never + // lands in `unmatched` and the teardown reads as a clean machine. + const LOGICAL = "b5f2c1e0-7a44-4d8e-9c31-metro-logical"; + + // What the JsRuntimeDebugger factory records when the id it was resolved with + // IS the logicalDeviceId Metro echoed — the one place both ids are compared. + beforeEach(() => { + resetDeviceAliases(); + rememberLogicalKeyedDevice(LOGICAL, LOGICAL); + }); + afterEach(() => resetDeviceAliases()); + + it("names a logicalDeviceId-keyed debugger session the scope could not reach", async () => { + const services = new Map([ + [`AndroidDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ + stopped: [`AndroidDevtools:${MINE}`], + left_running: [`JsRuntimeDebugger:8081:${LOGICAL}`], + }); + // The serial matched a service, so it is not a typo — the point is that + // `unmatched` cannot be the thing that reports this. + expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).not.toHaveBeenCalledWith(`JsRuntimeDebugger:8081:${LOGICAL}`); + }); + + it("names the network inspector and React profiler riding on that session too", async () => { + const services = new Map([ + [`AndroidDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [ + `JsRuntimeDebugger:8081:${LOGICAL}`, + { + state: ServiceState.RUNNING, + dependents: [`NetworkInspector:8081:${LOGICAL}`, `ReactProfilerSession:8081:${LOGICAL}`], + }, + ], + [`NetworkInspector:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ReactProfilerSession:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result.left_running).toEqual([ + `JsRuntimeDebugger:8081:${LOGICAL}`, + `NetworkInspector:8081:${LOGICAL}`, + `ReactProfilerSession:8081:${LOGICAL}`, + ]); + }); + + it("reaps rather than reports the session once the logicalDeviceId is supplied", async () => { + // The documented recovery, and the proof the id is the whole gap: pass it + // alongside the serial and the session is stopped like anything else. + const services = new Map([ + [`AndroidDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE, LOGICAL] }); + + expect(result).toEqual({ + stopped: [`AndroidDevtools:${MINE}`, `JsRuntimeDebugger:8081:${LOGICAL}`], + }); + expect(result).not.toHaveProperty("left_running"); + }); + + it("stays silent about another agent's serial-keyed session", async () => { + // `THEIRS` connected by serial (one device on that Metro), so it is an id + // `list-devices` hands out and a scope COULD have named it. A session left + // on it is that agent's business, not a scope that cannot express itself — + // reporting it would invite exactly the cross-agent teardown the `devices` + // scope exists to prevent. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`SimulatorServer:${MINE}`] }); + }); + + it("stops reporting the session once its debugger connection is disposed", async () => { + // The marker is dropped in the blueprint's dispose alongside the alias, so a + // stale one cannot make a later teardown accuse a session that is gone. + forgetLogicalKeyedDevice(LOGICAL); + const services = new Map([ + [`AndroidDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + expect(await tool.execute!({}, { devices: [MINE] })).toEqual({ + stopped: [`AndroidDevtools:${MINE}`], + }); + }); + + it("reports nothing on an unscoped sweep, which reaps every namespace anyway", async () => { + const services = new Map([ + [`JsRuntimeDebugger:8081:${LOGICAL}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + const result = await tool.execute!({}, {}); + + expect(result).toEqual({ stopped: [`JsRuntimeDebugger:8081:${LOGICAL}`] }); + }); + + it("ignores an IDLE session, which holds nothing left to leave running", async () => { + const services = new Map([ + [`AndroidDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`JsRuntimeDebugger:8081:${LOGICAL}`, { state: ServiceState.IDLE, dependents: [] }], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [`AndroidDevtools:${MINE}`] }); + }); + + it("matches the marker case-insensitively, as every other id comparison here does", async () => { + const services = new Map([ + [ + `JsRuntimeDebugger:8081:${LOGICAL.toUpperCase()}`, + { state: ServiceState.RUNNING, dependents: [] }, + ], + ]); + const tool = createStopAllSimulatorServersTool(createMockRegistry(services)); + + expect(await tool.execute!({}, { devices: [MINE] })).toEqual({ + stopped: [], + unmatched: [MINE], + left_running: [`JsRuntimeDebugger:8081:${LOGICAL.toUpperCase()}`], + }); + }); +}); + describe("stop-all-simulator-servers interaction messages", () => { // Both formatters previously had no coverage at all — flattening either to // an unconditional string left the whole suite green. Pin the exact wording @@ -1169,6 +1326,40 @@ describe("stop-all-simulator-servers interaction messages", () => { }) ).toBe("Stopped 0 simulator servers (2 supplied ids matched no service)"); }); + + it("completedMsg appends the left_running clause, singular and plural", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect( + completedMsg({ + params: { devices: [MINE] }, + result: { + stopped: [`SimulatorServer:${MINE}`], + left_running: ["JsRuntimeDebugger:8081:L"], + }, + }) + ).toBe("Stopped 1 simulator server (1 debugger session left running)"); + expect( + completedMsg({ + params: { devices: [MINE] }, + result: { + stopped: [], + left_running: ["JsRuntimeDebugger:8081:L", "NetworkInspector:8081:L"], + }, + }) + ).toBe("Stopped 0 simulator servers (2 debugger sessions left running)"); + }); + + it("completedMsg reports both clauses when a call hits both", () => { + const completedMsg = tool().interaction!.completedMsg!; + expect( + completedMsg({ + params: { devices: ["GHOST-1"] }, + result: { stopped: [], unmatched: ["GHOST-1"], left_running: ["JsRuntimeDebugger:8081:L"] }, + }) + ).toBe( + "Stopped 0 simulator servers (1 supplied id matched no service; 1 debugger session left running)" + ); + }); }); describe("stop-metro", () => { From 2c175cccd1065b4c1e28bd4e5c61decc9d9ca921 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 16:47:12 +0200 Subject: [PATCH 41/98] fix(react-profiler): stop the in-app profiler when a teardown disposes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReactProfilerSession.dispose() sent only Profiler.disable, on the assumption that react-profiler-stop had already ended the run. That held while the stop tool was the only route to a dispose; it stopped holding when this session joined stop-all-simulator-servers' namespace set, which disposes it mid-run. The React DevTools backend inside the app then keeps recording every commit into a buffer only an app or bundle reload frees, while the patched commit hook keeps re-serializing that whole accumulated buffer on React's commit path — outliving the argent session, with the teardown reporting the session as stopped. Stop the renderers (STOP_FOR_TAKEOVER_SCRIPT) and the Hermes sampler when the run is still active. Registry._teardown disposes dependents before their dependency, so the JsRuntimeDebugger's CDP session is still up here. --- .../src/blueprints/react-profiler-session.ts | 35 ++++- .../react-profiler/session-dispose.test.ts | 124 ++++++++++++++++++ 2 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 packages/tool-server/test/react-profiler/session-dispose.test.ts diff --git a/packages/tool-server/src/blueprints/react-profiler-session.ts b/packages/tool-server/src/blueprints/react-profiler-session.ts index 56c9cc748..494470b2b 100644 --- a/packages/tool-server/src/blueprints/react-profiler-session.ts +++ b/packages/tool-server/src/blueprints/react-profiler-session.ts @@ -7,7 +7,10 @@ import { } from "@argent/registry"; import type { CDPClient } from "../utils/debugger/cdp-client"; import type { JsRuntimeDebuggerApi } from "./js-runtime-debugger"; -import { FIBER_ROOT_TRACKER_SCRIPT } from "../utils/react-profiler/scripts"; +import { + FIBER_ROOT_TRACKER_SCRIPT, + STOP_FOR_TAKEOVER_SCRIPT, +} from "../utils/react-profiler/scripts"; export const REACT_PROFILER_SESSION_NAMESPACE = "ReactProfilerSession"; @@ -195,7 +198,35 @@ export const reactProfilerSessionBlueprint: ServiceBlueprint { - // Profiler.stop is called explicitly in react-profiler-stop before disposal. + // A dispose reached from `react-profiler-stop` arrives with the run + // already ended — that tool clears `profilingActive`, calls + // `Profiler.stop` and runs STOP_AND_READ_SCRIPT (which stops every + // renderer) before disposing. A dispose reached from + // `stop-all-simulator-servers` does not: this session is in that + // teardown's namespace set, so it arrives mid-run, with nothing having + // stopped the in-app backend. + // + // Left alone, the React DevTools backend keeps recording every commit + // into a buffer only an app or bundle reload frees, and argent's + // patched commit hook keeps re-serializing that whole accumulated + // buffer synchronously on React's commit path — inside the user's app, + // outliving the argent session that started it. Stop it here while the + // CDP session is still up: `Registry._teardown` disposes dependents + // before their dependency, so the JsRuntimeDebugger this rides on has + // not disconnected yet, and this is the last moment anything can reach + // the app. + if (state.profilingActive) { + state.profilingActive = false; + await cdp + .send("Runtime.evaluate", { + expression: STOP_FOR_TAKEOVER_SCRIPT, + returnByValue: true, + }) + .catch(warnOnError("STOP_FOR_TAKEOVER_SCRIPT")); + // And the Hermes CPU sampler `react-profiler-start` enabled, which + // `Profiler.disable` alone is not documented to end. + await cdp.send("Profiler.stop").catch(ignore); + } await cdp.send("Profiler.disable").catch(ignore); }, events, diff --git a/packages/tool-server/test/react-profiler/session-dispose.test.ts b/packages/tool-server/test/react-profiler/session-dispose.test.ts new file mode 100644 index 000000000..2f266392d --- /dev/null +++ b/packages/tool-server/test/react-profiler/session-dispose.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from "vitest"; +import { TypedEventEmitter } from "@argent/registry"; +import { reactProfilerSessionBlueprint } from "../../src/blueprints/react-profiler-session"; +import type { JsRuntimeDebuggerApi } from "../../src/blueprints/js-runtime-debugger"; +import { STOP_FOR_TAKEOVER_SCRIPT } from "../../src/utils/react-profiler/scripts"; + +/** + * What `ReactProfilerSession.dispose()` leaves behind IN THE APP. + * + * `react-profiler-stop` was once the only route to a dispose, so dispose could + * assume the run had already been stopped. Since `ReactProfilerSession` joined + * `stop-all-simulator-servers`' namespace set that is no longer true: a + * teardown disposes it mid-run, and an in-app React DevTools backend nobody + * stopped keeps recording every commit into a buffer only an app or bundle + * reload frees — outliving the argent session, inside the user's app, while + * the teardown reports the session as stopped. + */ + +interface SentCall { + method: string; + params?: Record; +} + +function fakeDebuggerApi(sent: SentCall[]): JsRuntimeDebuggerApi { + const events = new TypedEventEmitter void>>(); + const cdp = { + events, + send: async (method: string, params?: Record) => { + sent.push({ method, params }); + return {}; + }, + // The factory's own probes: architecture flags, then the Hermes version. + evaluate: async (expression: string) => { + if (expression.includes("RN$Bridgeless")) { + return JSON.stringify({ bridgeless: true, turboModules: true, fabric: true }); + } + if (expression.includes("HermesInternal")) { + return JSON.stringify({ "OSS Release Version": "0.12.0" }); + } + return undefined; + }, + isConnected: () => true, + }; + return { + port: 8081, + projectRoot: "/tmp/app", + deviceName: "iPhone 16 Pro", + appName: "Bluesky", + logicalDeviceId: undefined, + isNewDebugger: true, + cdp, + } as unknown as JsRuntimeDebuggerApi; +} + +async function makeSession(sent: SentCall[]) { + return reactProfilerSessionBlueprint.factory( + { debugger: fakeDebuggerApi(sent) }, + "8081:AAAA-1111", + undefined + ); +} + +describe("ReactProfilerSession dispose", () => { + it("stops the in-app backend and the Hermes sampler when a run is still active", async () => { + const sent: SentCall[] = []; + const instance = await makeSession(sent); + instance.api.profilingActive = true; + sent.length = 0; + + await instance.dispose(); + + const takeover = sent.find( + (c) => c.method === "Runtime.evaluate" && c.params?.expression === STOP_FOR_TAKEOVER_SCRIPT + ); + expect(takeover, "the renderers must be told to stop profiling").toBeDefined(); + expect(sent.map((c) => c.method)).toEqual([ + "Runtime.evaluate", + "Profiler.stop", + "Profiler.disable", + ]); + // Nothing can reach the session after this, so the flag must not outlive it + // and read as a run still in progress. + expect(instance.api.profilingActive).toBe(false); + }); + + it("only disables the domain when the run already ended", async () => { + // The `react-profiler-stop` path: that tool clears `profilingActive`, sends + // `Profiler.stop` and runs the stop-and-read script itself. Re-stopping + // here would send a second `Profiler.stop` against an un-started sampler. + const sent: SentCall[] = []; + const instance = await makeSession(sent); + expect(instance.api.profilingActive).toBe(false); + sent.length = 0; + + await instance.dispose(); + + expect(sent.map((c) => c.method)).toEqual(["Profiler.disable"]); + }); + + it("still disables the domain when the in-app stop throws", async () => { + const sent: SentCall[] = []; + const api = fakeDebuggerApi(sent); + const cdp = api.cdp as unknown as { send: (m: string, p?: unknown) => Promise }; + const instance = await reactProfilerSessionBlueprint.factory( + { debugger: api }, + "8081:AAAA-1111", + undefined + ); + instance.api.profilingActive = true; + sent.length = 0; + cdp.send = async (method: string) => { + sent.push({ method }); + if (method !== "Profiler.disable") throw new Error("CDP went away mid-teardown"); + return {}; + }; + + await expect(instance.dispose()).resolves.toBeUndefined(); + expect(sent.map((c) => c.method)).toEqual([ + "Runtime.evaluate", + "Profiler.stop", + "Profiler.disable", + ]); + }); +}); From eb7d10740040cd0c7b3d1bc88a530391d2d00d20 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 16:52:50 +0200 Subject: [PATCH 42/98] fix(flow): stop a replayed cleanup flow retargeting a device it never named MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recorded `stop-all-simulator-servers` scope was rebound to the run device unconditionally. When that device was auto-detected — a cleanup flow resolves one opportunistically whenever exactly one is booted — the flow named device A and the replay tore down device B's services, which is precisely the cross-agent teardown the `devices` scope was added to prevent. Rebind a recorded scope only when the caller named the run device explicitly; an auto-detected one names nobody's intent, so the recorded ids stand. A step that recorded no scope is still narrowed onto the run device, since binding can only make the machine-wide sweep smaller. Corrects the comments on both sides of the binding, and the create-flow skill's prose form of the same claim (including its stale cross-reference to "Strategy 2 - Manual execution"). --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-device.ts | 70 +++++++++++-------- .../tool-server/src/tools/flows/flow-run.ts | 19 ++++- .../test/flows/flow-composition.test.ts | 55 ++++++++++++--- 4 files changed, 107 insertions(+), 39 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 5972045ad..00876ee6c 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -9,7 +9,7 @@ A flow is a sequence of steps saved to a `.yaml` file in the `.argent/flows/` di Flows store **no device id**: the runner binds a device (the single booted one, or pass `device`/`platform`). A recorded coordinate `gesture-tap` is captured as a portable `tap: { selector }` step whenever the tapped element has stable text/identifier. -The one exception is a device _scope_ rather than a target: `stop-all-simulator-servers`' `devices` list is kept in the YAML, because without it the step means the machine-wide sweep and would tear down devices other agents are mid-session on. Replay still rebinds it to the run device, so the recorded ids only matter if you hand-run the step (see _Strategy 2 — Manual execution_), where they keep the teardown scoped instead of reaping everything. A cleanup flow whose only step is that teardown needs no device and runs whether none or several are booted. +The one exception is a device _scope_ rather than a target: `stop-all-simulator-servers`' `devices` list is kept in the YAML, because without it the step means the machine-wide sweep and would tear down devices other agents are mid-session on. Replay rebinds a recorded scope only when you pass `device` explicitly — an auto-detected device would retarget the teardown at a device the flow never named, which is the cross-agent teardown the scope exists to prevent. So the recorded ids are what run when you replay without `device`, and when you hand-run the step (see _Strategy 2 — Manual recovery + continue_); on another host they reap nothing and come back in `unmatched`. Re-record the cleanup flow there, or pass `device`. A step that recorded NO scope is still narrowed onto whatever device the run resolved, since binding can only make the machine-wide sweep smaller. A cleanup flow whose only step is that teardown needs no device and runs whether none or several are booted. **Two flow types** diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index 2ab5b65f7..22a2d1da4 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -41,18 +41,13 @@ const DEVICE_BIND_KEYS = ["udid", "device_id", "device"] as const; * run and a flow that named several would be naming the recording host's. * * `stop-all-simulator-servers`' `devices` is the only such key. It is a scope - * rather than a target, but the failure is the same one: kept verbatim, a - * recorded teardown names the machine it was recorded on, so on any other host - * it reaps nothing and passes — a stale baked-in id overriding the run target, - * which is exactly what this binding exists to prevent. The scoped form is what - * the tool description, the MCP instructions and the skills all now tell agents - * to call, so it is the form that gets recorded. - * - * Binding is unconditional once the tool declares the key, so a recording of - * the UNSCOPED sweep replays as a stop of the run device. That direction is - * deliberate: the replayed artifact must not tear down devices another agent is - * mid-session on, which is the hazard the `devices` scope was added for, and a - * flow has exactly one resolved device to be talking about. + * rather than a target, and that difference decides when it is rebound. A + * recording of the UNSCOPED sweep always replays as a stop of the run device: + * the replayed artifact must not tear down devices another agent is mid-session + * on, which is the hazard the `devices` scope was added for, and binding can + * only narrow there. A recorded scope, on the other hand, is the flow's own + * statement of what to reap, and is overridden only by an explicit `device` — + * see {@link bindDeviceArgs}, which is where the two cases part. */ const DEVICE_BIND_LIST_KEYS = ["devices"] as const; @@ -184,12 +179,15 @@ export function stripDeviceKeys(args: Record): Record + args: Record, + deviceIsExplicit = false ): Record { const toolDef = registry.getTool(toolName); const props = (toolDef?.inputSchema as { properties?: Record } | undefined) @@ -290,15 +289,30 @@ export function bindDeviceArgs( for (const k of DEVICE_BIND_LIST_KEYS) { // Never forward a scope to a tool that does not declare it — a `.strict()` // schema would reject the whole call. - if (!props || !(k in props)) delete out[k]; - // The run device wins over anything recorded, so a flow recorded against - // one device stays portable. With NO run device (a cleanup flow, see - // {@link DEVICE_ARG_KEYS}) the recorded scope is kept rather than dropped: - // there is no run target for it to override, and dropping it would widen a - // teardown the recording scoped — the one direction that costs another - // agent their devices. `[""]` is never bound: an id that owns nothing reaps - // nothing and still reports pass. - else if (deviceId) out[k] = [deviceId]; + if (!props || !(k in props)) { + delete out[k]; + continue; + } + // `[""]` is never bound: an id that owns nothing reaps nothing and still + // reports pass. + if (!deviceId) continue; + // With NO recorded scope the run device NARROWS what the step would + // otherwise do — an unscoped `stop-all-simulator-servers` is the + // machine-wide sweep — so bind it whatever resolved it. Strictly the safe + // direction, and the reason a device is resolved for a cleanup flow at all. + if (out[k] === undefined) { + out[k] = [deviceId]; + continue; + } + // With one recorded, OVERRIDING it is destructive rather than portable: a + // flow that named device A would tear down whichever device happened to + // resolve, which is precisely the cross-agent teardown the `devices` scope + // was added to prevent. Only an explicit `device` overrides — there the + // caller named the run target itself, so retargeting the teardown at it is + // what they asked for. An auto-resolved device names nobody's intent, so + // the recorded ids stand: on another host they reap nothing and come back + // in `unmatched`, which is the safe direction and a legible one. + if (deviceIsExplicit) out[k] = [deviceId]; } if (props) for (const k of DEVICE_BIND_KEYS) if (k in props) out[k] = deviceId; return out; diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 96adaa3df..ae774818d 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -732,6 +732,12 @@ function noChromiumAppReason(device: DeviceInfo): string { // (via `deviceEnv`) and the compiler can find the ones that don't. interface ExecState extends Omit { device: DeviceInfo | null; + /** + * Whether {@link device} is the one the CALLER named, rather than one + * auto-detected from what happens to be booted. Only a named device may + * override a scope a recording already carries — see {@link bindDeviceArgs}. + */ + deviceIsExplicit: boolean; /** * The ROOT flow file's canonical (realpath'd) directory — the anchor for * snapshot baselines and a chromium launch's relative app path, so a @@ -1057,6 +1063,7 @@ returns a notice with the prerequisite instead of running.`, registry, ctx, device, + deviceIsExplicit: Boolean(params.device), signal, flowsDir, viaUpload, @@ -2138,8 +2145,16 @@ async function execLeafStep( // unreachable for those and must stay unreachable: injecting the empty // string would not fail the step, it would silently retarget it at no // device. A SCOPE key (`devices`) does reach here device-free, which is - // the cleanup-flow case `bindDeviceArgs` guards by leaving it unset. - const args = bindDeviceArgs(registry, step.name, device?.id ?? "", step.args); + // the cleanup-flow case `bindDeviceArgs` guards by keeping whatever the + // recording scoped — and, when the run device was only auto-detected, it + // keeps that even with a device resolved. + const args = bindDeviceArgs( + registry, + step.name, + device?.id ?? "", + step.args, + state.deviceIsExplicit + ); const outputHint = registry.getTool(step.name)?.outputHint; if (step.delayMs && !(await sleepOrAbort(step.delayMs, signal))) { return { ...base, status: "skip", tool: step.name, reason: "run aborted during delay" }; diff --git a/packages/tool-server/test/flows/flow-composition.test.ts b/packages/tool-server/test/flows/flow-composition.test.ts index 788064e5b..9bbe98387 100644 --- a/packages/tool-server/test/flows/flow-composition.test.ts +++ b/packages/tool-server/test/flows/flow-composition.test.ts @@ -2450,21 +2450,60 @@ describe("device binding (portability)", () => { expect(out).not.toHaveProperty("devices"); }); - it("replaces a stale recorded devices list rather than merging or appending to it", () => { - // The runner is authoritative on device — a flow recorded on one host must - // not carry that host's ids forward when replayed on another. - const out = bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "RESOLVED", { - devices: ["OLD-HOST-ID", "OTHER"], - }); + it("replaces a stale recorded devices list when the caller NAMED the run device", () => { + // An explicit `device` is the caller saying which device this run is about, + // so retargeting the teardown at it is what they asked for — and a flow + // recorded on one host must not carry that host's ids forward. + const out = bindDeviceArgs( + reg({ devices: {} }), + "stop-all-simulator-servers", + "RESOLVED", + { devices: ["OLD-HOST-ID", "OTHER"] }, + true + ); expect(out).toEqual({ devices: ["RESOLVED"] }); }); + it("keeps a recorded scope when the run device was only auto-detected", () => { + // The destructive direction: the flow named one device, exactly one other + // happens to be booted, and replay would reap THAT one — a device nobody in + // this run ever named, quite possibly another agent's. This is the + // cross-agent teardown the `devices` scope exists to prevent, so the + // recorded ids stand; on another host they reap nothing and come back in + // `unmatched`, which is the safe direction and a legible one. + const out = bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "AUTO", { + devices: ["RECORDED-HOST"], + }); + expect(out).toEqual({ devices: ["RECORDED-HOST"] }); + }); + + it("still narrows an UNSCOPED recorded sweep onto an auto-detected device", () => { + // Nothing recorded means the step is the machine-wide sweep, so binding can + // only narrow it. That is why a cleanup flow resolves a device at all. + const out = bindDeviceArgs(reg({ devices: {} }), "stop-all-simulator-servers", "AUTO", {}); + expect(out).toEqual({ devices: ["AUTO"] }); + }); + it("binds a scalar and a list device key together when a tool declares both", () => { - const out = bindDeviceArgs(reg({ udid: {}, devices: {} }), "hypothetical-tool", "RESOLVED", { + const out = bindDeviceArgs( + reg({ udid: {}, devices: {} }), + "hypothetical-tool", + "RESOLVED", + { udid: "STALE", devices: ["OLD"] }, + true + ); + expect(out).toEqual({ udid: "RESOLVED", devices: ["RESOLVED"] }); + }); + + it("rebinds the TARGET but not the recorded SCOPE on an auto-detected device", () => { + // The two keys part company here: a stale `udid` must never survive (the + // step would drive the wrong device), while a stale `devices` must never be + // retargeted (the step would destroy the wrong device). + const out = bindDeviceArgs(reg({ udid: {}, devices: {} }), "hypothetical-tool", "AUTO", { udid: "STALE", devices: ["OLD"], }); - expect(out).toEqual({ udid: "RESOLVED", devices: ["RESOLVED"] }); + expect(out).toEqual({ udid: "AUTO", devices: ["OLD"] }); }); }); From ed39aa835abe7f92863ebaddbe89c231b8ec4e21 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 16:55:15 +0200 Subject: [PATCH 43/98] fix(flow): stop the alias guard asserting a truncation that may not have happened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two spellings resolve to one flow file the guard cannot tell "another caller restarted this key" from "the same caller respelled its own root or flow name" — and it asserted the former as fact: "Starting that recording truncated this one ... restarting here would destroy their take in turn." In the second case nothing was truncated, there is no other caller, and the take is live and intact; the message sent the agent to abandon a healthy in-progress recording and re-walk the whole flow on the device. On macOS the respelling needs no mistake at all: /tmp is a symlink, so any path that realpaths a root produces it. Report the fact instead — the key is held by a take registered under another spelling — and give the advice that recovers both readings: re-address it exactly as flow-start-recording was given it. --- .../tool-server/src/tools/flows/flow-utils.ts | 40 ++++++++++++------- .../flows/flow-concurrent-recording.test.ts | 33 ++++++++++++++- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 2f3e29ae6..1b1287152 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -494,24 +494,36 @@ export async function requireRecordingSession( ); } // The key is the file's identity, so a session found under it may have been - // registered by a caller who spells that one file differently — a symlink - // into a shared vault, a symlinked `.argent/flows`, a name cased two ways on - // APFS. Handing it over would silently enrol this caller in the OTHER take: - // its steps would land in a file it never addressed, under a prerequisite it - // never declared, and its finish would report the other agent's steps as its - // own. That collision is what the restart already destroyed this caller's - // take for, so report it as the loss it is rather than papering over it. A - // root spelled with a trailing slash is not one of these — `getFlowPath` - // normalizes both sides before they are compared. + // registered under a DIFFERENT spelling of that one file — a symlink into a + // shared vault, a symlinked `.argent/flows`, a root spelled `/tmp` vs + // `/private/tmp`, a name cased two ways on APFS. Handing it over would risk + // silently enrolling this caller in someone else's take: its steps would land + // in a file it never addressed, under a prerequisite it never declared, and + // its finish would report the other agent's steps as its own. A root spelled + // with a trailing slash is not one of these — `getFlowPath` normalizes both + // sides before they are compared. + // + // Which of two situations this is cannot be told apart from here, so the + // message must assert neither. It is EITHER the same caller respelling its + // own root or name — nothing was truncated, the take is live and intact, and + // re-addressing it under the registered spelling resumes it — OR another + // caller's restart, which did truncate. Naming the second as fact sent a + // caller in the first situation to abandon a healthy recording and re-walk + // the whole flow on the device. The advice that recovers both is the same: + // use the spelling the session is registered under, which is the one + // `flow-start-recording` was given. const asked = getFlowPath(projectRoot, name); const held = getFlowPath(session.projectRoot, session.name); if (asked !== held) { throw new FailureError( - `Recording of "${name}" in ${projectRoot} is no longer active — ${held} and ${asked} ` + - `are the same file on this filesystem (a symlink, or a case-insensitive volume), and ` + - `"${session.name}" in ${session.projectRoot} is the take that now holds it. Starting ` + - `that recording truncated this one. Record under a name that resolves to its own file, ` + - `or coordinate with the other caller — restarting here would destroy their take in turn.`, + `Recording of "${name}" in ${projectRoot} is not registered under that spelling — ${held} ` + + `and ${asked} are the same file on this filesystem (a symlink, or a case-insensitive ` + + `volume), and the live take on it is registered as "${session.name}" in ` + + `${session.projectRoot}. If that is your own recording spelled another way, re-address ` + + `it exactly as you passed it to flow-start-recording — the take is intact and still ` + + `recording. If it is another caller's, their flow-start-recording truncated yours; ` + + `record under a name that resolves to its own file rather than restarting here, which ` + + `would destroy their take in turn.`, { error_code: FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING, failure_stage: "flow_recording_key_aliased", diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 3c5806a2c..c5b24bd20 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -355,7 +355,11 @@ describe("two recording keys that resolve to one file", () => { // B's take. const err = await captureFailure(addEcho(rootA, "checkout", "h1-d")); expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); - expect(formatErrorForAgent(err)).toContain("no longer active"); + // The guard cannot tell this from the same caller respelling its own root, + // so it names the take that holds the key and offers both readings rather + // than asserting the destructive one. Here the destructive one is true. + expect(formatErrorForAgent(err)).toContain("not registered under that spelling"); + expect(formatErrorForAgent(err)).toContain("truncated yours"); await addEcho(rootB, "checkout", "h2-a"); // A's finish reports the same loss, rather than handing back B's take as @@ -409,6 +413,33 @@ describe("two recording keys that resolve to one file", () => { expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); }); + it("does not accuse a caller that respelled its own root of destroying a take", async () => { + // The other half of the guard's ambiguity, and the common one on macOS: + // `/tmp` is a symlink, so any code path that realpaths a root produces the + // second spelling. Nothing was truncated, there is no other caller, and the + // take is live and intact — so the message must say how to resume it rather + // than sending the agent to re-walk the whole flow on the device. + const root = await makeRoot("respelled-root"); + const realRoot = await fs.realpath(root); + if (realRoot === root) return; // no symlinked ancestor on this host + + await start(root, "checkout"); + await addEcho(root, "checkout", "c1"); + + const err = await captureFailure(addEcho(realRoot, "checkout", "c2")); + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + const message = formatErrorForAgent(err); + expect(message).toContain("re-address it exactly as you passed it to flow-start-recording"); + expect(message).toContain("the take is intact and still recording"); + // The claim that made this a false alarm. + expect(message).not.toMatch(/truncated this one/); + expect(message).toContain(root); + + // And the take really is resumable under its registered spelling. + await addEcho(root, "checkout", "c3"); + expect(await readMarkers(root, "checkout")).toEqual(["echo:c1", "echo:c3"]); + }); + it("keeps two genuinely distinct flows independent", async () => { // The control: no symlink, no case variance, so nothing is canonicalized // together and the isolation guarantee holds exactly as stated. From 73fdd9f5d5771c67f2ae06680621518c21f16605 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 16:58:38 +0200 Subject: [PATCH 44/98] fix(flow): write through a dangling flow-file symlink instead of replacing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit realpath fails on the whole path when a symlink's target is missing, so canonicalFlowPath fell back to the link's own spelling and the atomic swap renamed onto it — replacing the symlink with a regular file. That is the shared-vault workflow's normal starting state: the link is created before the first recording, or the vault copy goes with a branch switch or a git clean. The vault target was never created, the project was permanently detached from the vault, any sibling project on the same target was left dangling, and flow-start-recording reported success. Resolve such a link by hand, one hop at a time, canonicalizing each target's directory so the result agrees with what a later append computes via plain realpath. Since resolveFlowKey shares this resolution, two projects linking one not-yet-created vault file now key as the one file the write produces. --- .../tool-server/src/tools/flows/flow-utils.ts | 62 +++++++++++++++++-- .../flows/flow-concurrent-recording.test.ts | 42 +++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 1b1287152..2b78596f3 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -135,10 +135,14 @@ export function getFlowPath(projectRoot: string, name: string): string { * minting a second session that silently truncates the first. * * This is the same resolution {@link writeFlowFile} performs before its swap, - * deliberately: the key and the write agree by construction, so wherever the - * filesystem declines to answer (a dangling symlink, a flows dir that does not - * exist yet) both fall back to the same pure-path spelling and the two remain - * two — which is correct, because two files is what the write then produces. + * deliberately: the key and the write agree by construction. Where the + * filesystem declines to answer at all — a flows dir that does not exist yet — + * both fall back to the same pure-path spelling and two spellings remain two, + * which is correct, because two files is what the write then produces. Where it + * declines only because the target is missing — a dangling vault symlink — both + * follow the link by hand ({@link followDanglingLink}), so the two spellings + * become one key, which is equally correct: one file is what the write produces + * there. * * It costs one `realpath` pair per recording tool call, on a path already doing * file I/O. @@ -2566,12 +2570,60 @@ function scrubTempPath(err: unknown, tmpPath: string, filePath: string): Error { * first swap and the rest would disagree wherever an ancestor is itself a * symlink, which is the default for the temp dir on macOS. * + * A DANGLING link is the case `realpath` cannot express — it fails on the whole + * path rather than answering with the target — and that failure would put the + * link's own spelling back in front of `rename`, i.e. exactly the swap this + * exists to prevent. {@link followDanglingLink} resolves it by hand. + * * Shared with {@link resolveFlowKey}, so the identity a recording is keyed by * and the file its steps land in can never disagree. */ async function canonicalFlowPath(filePath: string): Promise { const dir = await fs.realpath(path.dirname(filePath)).catch(() => path.dirname(filePath)); - return await fs.realpath(filePath).catch(() => path.join(dir, path.basename(filePath))); + const real = await fs.realpath(filePath).catch(() => null); + if (real !== null) return real; + return followDanglingLink(path.join(dir, path.basename(filePath))); +} + +/** + * How deep a chain of not-yet-existing symlinks {@link followDanglingLink} + * walks. A backstop against a link cycle, which `readlink` alone cannot detect; + * far past any real vault layout, which is one hop. + */ +const MAX_DANGLING_LINK_HOPS = 32; + +/** + * Where a link whose TARGET does not exist actually points. + * + * `realpath` fails outright on a dangling symlink, so the fallback above would + * hand back the link's own path — and `rename(2)` replaces the path it is + * given, so the first write of a recording would swap the symlink for a regular + * file. That is the shared-vault setup's normal starting state: the link is + * created before the first recording, or its target is removed by a branch + * switch or a `git clean`. The vault copy is then never created, the project is + * permanently detached from the vault, and any sibling project linked to the + * same target is left dangling — with the tool reporting success. + * + * So resolve the link by hand, one hop at a time, canonicalizing each target's + * DIRECTORY the way {@link canonicalFlowPath} does so the result agrees with + * what a later append (by then a plain `realpath`) will compute. A path that is + * not a link — the ordinary "flow file does not exist yet" case — comes back + * unchanged on the first probe. + */ +async function followDanglingLink(linkPath: string): Promise { + let current = linkPath; + for (let hop = 0; hop < MAX_DANGLING_LINK_HOPS; hop++) { + const target = await fs.readlink(current).catch(() => null); + if (target === null) return current; + const resolved = path.resolve(path.dirname(current), target); + // The rest of the chain may well exist — only the last hop has to dangle + // for `realpath` to have refused the whole path. + const real = await fs.realpath(resolved).catch(() => null); + if (real !== null) return real; + const targetDir = await fs.realpath(path.dirname(resolved)).catch(() => path.dirname(resolved)); + current = path.join(targetDir, path.basename(resolved)); + } + return current; } /** diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index c5b24bd20..a6884c06b 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -440,6 +440,48 @@ describe("two recording keys that resolve to one file", () => { expect(await readMarkers(root, "checkout")).toEqual(["echo:c1", "echo:c3"]); }); + it("writes THROUGH a dangling vault symlink instead of replacing it", async () => { + // The shared-vault workflow's normal starting state: the link is created + // before the first recording, or the vault copy is removed by a branch + // switch or a `git clean`. `realpath` fails on the whole path there, so the + // swap used to rename onto the link's own spelling — replacing the symlink + // with a regular file, never creating the vault target, and permanently + // detaching the project from the vault while reporting success. + const vault = await makeRoot("dangling-vault"); + const root = await makeRoot("dangling-proj"); + const target = path.join(vault, "shared.yaml"); + await fs.mkdir(path.dirname(flowPath(root, "shared")), { recursive: true }); + await fs.symlink(target, flowPath(root, "shared")); + + await start(root, "shared"); + await addEcho(root, "shared", "s1"); + + expect((await fs.lstat(flowPath(root, "shared"))).isSymbolicLink()).toBe(true); + expect(markers(parseFlow(await fs.readFile(target, "utf8")).steps)).toEqual(["echo:s1"]); + }); + + it("keys two projects onto one dangling vault target, as one file", async () => { + // The key follows the same resolution as the write, so two projects linking + // the same not-yet-created vault file are one recording — matching what the + // write then produces, rather than two sessions racing onto one output. + const vault = await makeRoot("dangling-shared-vault"); + const rootA = await makeRoot("dangling-a"); + const rootB = await makeRoot("dangling-b"); + const target = path.join(vault, "checkout.yaml"); + for (const root of [rootA, rootB]) { + await fs.mkdir(path.dirname(flowPath(root, "checkout")), { recursive: true }); + await fs.symlink(target, flowPath(root, "checkout")); + } + + await start(rootA, "checkout"); + await addEcho(rootA, "checkout", "a1"); + + const restarted = await start(rootB, "checkout"); + expect(restarted.restarted).toBe(true); + expect(restarted.discardedSteps).toBe(1); + expect((await fs.lstat(flowPath(rootA, "checkout"))).isSymbolicLink()).toBe(true); + }); + it("keeps two genuinely distinct flows independent", async () => { // The control: no symlink, no case variance, so nothing is canonicalized // together and the isolation guarantee holds exactly as stated. From 9070744df0af0b12a156eb5c820355a17e69b43a Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:05:19 +0200 Subject: [PATCH 45/98] fix(native-profiler): fail a start whose session a teardown destroyed mid-handshake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit native-profiler-start spawns its capture child and only then awaits a readiness handshake. A stop-all-simulator-servers arriving in that window saw profilingActive still false, so it disposed the session without killing the child and reported it stopped; the start then resumed and returned status: "recording" against a session the registry had already destroyed. The owner's native-profiler-stop answered "call native-profiler-start first" and the trace file was left with nothing able to reach it. Kill the capture child on dispose whenever one exists — the flag says the run has been declared active, not that a process was spawned — and mark the session disposed so a resuming start reaps what it spawned and fails with NATIVE_PROFILER_SESSION_TORN_DOWN instead of reporting a recording. --- packages/registry/src/failure-codes.ts | 1 + .../src/blueprints/native-profiler-session.ts | 27 +++- .../native-profiler/platforms/android.ts | 24 +++ .../profiler/native-profiler/platforms/ios.ts | 29 ++++ .../native-profiler-teardown-race.test.ts | 140 ++++++++++++++++++ 5 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 packages/tool-server/test/native-profiler-teardown-race.test.ts diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index ce5d3d572..62057d719 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -193,6 +193,7 @@ export const FAILURE_CODES = { NATIVE_PROFILER_XCTRACE_READY_TIMEOUT: "NATIVE_PROFILER_XCTRACE_READY_TIMEOUT", NATIVE_PROFILER_TRACE_TEMPLATE_MISSING: "NATIVE_PROFILER_TRACE_TEMPLATE_MISSING", NATIVE_PROFILER_NO_ACTIVE_SESSION: "NATIVE_PROFILER_NO_ACTIVE_SESSION", + NATIVE_PROFILER_SESSION_TORN_DOWN: "NATIVE_PROFILER_SESSION_TORN_DOWN", NATIVE_PROFILER_APP_PROCESS_NOT_FOUND: "NATIVE_PROFILER_APP_PROCESS_NOT_FOUND", NATIVE_PROFILER_NO_EXPORTED_TRACE: "NATIVE_PROFILER_NO_EXPORTED_TRACE", // Android perfetto start-failure modes — mirror the iOS xctrace set so a diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index 5618b6950..deb1e67aa 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -87,6 +87,19 @@ export interface NativeProfilerSessionApi { * Null when unknown — before any stop, on Android, or after a load. */ mallocStackLogging: boolean | null; + /** + * Whether this session has been torn down. Set by `dispose()` and never + * cleared: `Registry._teardown` nulls the node's instance, so the next + * resolve builds a fresh api rather than reviving this one. + * + * Read by `native-profiler-start`, which spawns its capture child and then + * awaits a readiness handshake. A teardown arriving inside that window + * destroys the session the start is about to report success for — leaving a + * `status: "recording"` against a session the registry no longer has, whose + * owner's `native-profiler-stop` then answers "call native-profiler-start + * first". Start checks this before returning and fails instead. + */ + disposed: boolean; recordingTimeout: NodeJS.Timeout | null; recordingTimedOut: boolean; recordingExitedUnexpectedly: boolean; @@ -164,6 +177,7 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< cpuFilterPid: null, recordingMallocStackLogging: null, mallocStackLogging: null, + disposed: false, recordingTimeout: null, recordingTimedOut: false, recordingExitedUnexpectedly: false, @@ -176,6 +190,11 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< return { api: state, dispose: async () => { + // Before anything else, and read by a start still inside its readiness + // handshake: from here on this session no longer exists, so a start + // that resumes must fail rather than report a recording nothing can + // reach. See {@link NativeProfilerSessionApi.disposed}. + state.disposed = true; if (state.recordingTimeout) { clearTimeout(state.recordingTimeout); state.recordingTimeout = null; @@ -191,7 +210,13 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< if (state.platform === "ios") { const child = state.captureProcess; try { - if (state.profilingActive && child) { + // Whether or not the run has been declared active: `attemptStart` + // hands the child over BEFORE awaiting xctrace's readiness + // handshake, so a teardown inside that window sees `profilingActive` + // still false while a spawned xctrace is very much running. Gating + // the kill on the flag left it behind, recording into a trace + // nobody would ever stop. + if (child) { try { child.kill("SIGKILL"); } catch { diff --git a/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts b/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts index b0d460975..5bff80712 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/platforms/android.ts @@ -71,6 +71,30 @@ export async function startNativeProfilerAndroid( timestamp, }); + // See the iOS twin: a `stop-all-simulator-servers` that landed while + // `startPerfetto` was in flight has already destroyed this session, and + // stamping state onto a dead api would report a recording whose owner's stop + // answers "call native-profiler-start first". The daemon is this attempt's to + // reap — the teardown never saw it, since `capturePid` is only handed over + // below. + if (api.disposed) { + const { adbShell } = await import("../../../../utils/adb"); + await adbShell(params.device_id, `kill -KILL ${pid}`).catch(() => {}); + await adbShell(params.device_id, `rm -f ${onDeviceTracePath}`).catch(() => {}); + throw new FailureError( + `The native profiling session for ${api.deviceId} was torn down by a ` + + `stop-all-simulator-servers while perfetto was starting, so nothing was recorded — ` + + `one tool-server serves every agent using this argent install, so this may have been ` + + `another agent ending its session. Call native-profiler-start again.`, + { + error_code: FAILURE_CODES.NATIVE_PROFILER_SESSION_TORN_DOWN, + failure_stage: "android_native_profiler_start", + failure_area: "tool_server", + error_kind: "not_found", + } + ); + } + // Perfetto is up — this capture now owns the session; stamp its descriptors // and clear any prior capture's recovery flags (superseded on success only). api.recordingTimedOut = false; diff --git a/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts b/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts index 58232c584..943bf3b14 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/platforms/ios.ts @@ -756,6 +756,35 @@ export async function startNativeProfilerIos( } const { child: xctraceProcess, pid: xctracePid } = started; + // A `stop-all-simulator-servers` that landed inside the readiness handshake + // above has already destroyed this session — `Registry._teardown` nulled the + // node's instance, so nothing can resolve `api` again and the owner's + // `native-profiler-stop` would answer "call native-profiler-start first". + // Reporting `status: "recording"` here would hand back a session that does + // not exist, with a trace file on disk and no way to reach it. Reap what this + // attempt spawned and say what happened instead. + if (api.disposed) { + try { + xctraceProcess.kill("SIGKILL"); + } catch { + // already dead + } + resetStartState(api); + throw new FailureError( + `The native profiling session for ${api.deviceId} was torn down by a ` + + `stop-all-simulator-servers while this start was waiting for xctrace to become ` + + `ready, so nothing was recorded — one tool-server serves every agent using this ` + + `argent install, so this may have been another agent ending its session. Call ` + + `native-profiler-start again.`, + { + error_code: FAILURE_CODES.NATIVE_PROFILER_SESSION_TORN_DOWN, + failure_stage: "native_profiler_xctrace_start", + failure_area: "tool_server", + error_kind: "not_found", + } + ); + } + // Stamp the per-capture descriptors only now, on SUCCESS: a failed start // must leave the previous capture's still-loaded exports fully described // for analyze (trace name, all-processes filter PID, capture mode). The diff --git a/packages/tool-server/test/native-profiler-teardown-race.test.ts b/packages/tool-server/test/native-profiler-teardown-race.test.ts new file mode 100644 index 000000000..c23e60972 --- /dev/null +++ b/packages/tool-server/test/native-profiler-teardown-race.test.ts @@ -0,0 +1,140 @@ +/** + * A teardown that lands INSIDE `native-profiler-start`'s readiness window. + * + * Start spawns its capture child and only then awaits a readiness handshake — + * xctrace's `--notify-tracing-started`, or `startPerfetto`'s round trip. A + * `stop-all-simulator-servers` arriving in that window used to see + * `profilingActive` still false, so it disposed the session WITHOUT killing the + * child and reported the session as stopped. The start then resumed and + * returned `status: "recording"` against a session `Registry._teardown` had + * already destroyed: the owner's `native-profiler-stop` answered "No active + * native profiling session found. Call native-profiler-start first," and the + * trace file sat on disk with nothing able to reach it. + * + * This became reachable outside process shutdown only when + * `NativeProfilerSession` joined the teardown's namespace set. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { EventEmitter } from "events"; +import type { ChildProcess } from "child_process"; +import { FAILURE_CODES, getFailureSignal, type DeviceInfo } from "@argent/registry"; + +vi.mock("../src/utils/adb", () => ({ adbShell: vi.fn(async () => "") })); +vi.mock("@argent/native-devtools-android", () => ({ + disposeWarmEngine: vi.fn(async () => {}), + TraceProcessorUnavailableError: class extends Error {}, +})); +vi.mock("../src/utils/android-profiler/capture", () => ({ + startPerfetto: vi.fn(), + stopPerfetto: vi.fn(), +})); +vi.mock("../src/utils/android-profiler/detect-app", () => ({ + detectAndroidRunningApp: vi.fn(async () => "com.example.app"), + validateAndroidAppProcess: vi.fn(async () => {}), +})); + +import { adbShell } from "../src/utils/adb"; +import { startPerfetto } from "../src/utils/android-profiler/capture"; +import { + nativeProfilerSessionBlueprint, + type NativeProfilerSessionApi, +} from "../src/blueprints/native-profiler-session"; +import { startNativeProfilerAndroid } from "../src/tools/profiler/native-profiler/platforms/android"; + +const adbShellMock = vi.mocked(adbShell); +const startPerfettoMock = vi.mocked(startPerfetto); + +const iosDevice = { id: "6DBF83B4-0000-0000-0000-000000000000", platform: "ios" } as DeviceInfo; +const androidDevice = { id: "emulator-5554", platform: "android" } as DeviceInfo; + +class FakeChild extends EventEmitter { + kill = vi.fn((_signal?: NodeJS.Signals) => { + queueMicrotask(() => this.emit("exit", null, "SIGKILL")); + return true; + }); +} + +async function session(device: DeviceInfo) { + return nativeProfilerSessionBlueprint.factory({}, device, { device } as never); +} + +beforeEach(() => { + adbShellMock.mockClear(); + startPerfettoMock.mockReset(); +}); + +describe("a teardown inside the native-profiler start window", () => { + it("iOS: SIGKILLs a child the start handed over before declaring the run active", async () => { + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + // Exactly what `attemptStart` leaves behind while it awaits readiness. + const child = new FakeChild(); + api.captureProcess = child as unknown as ChildProcess; + api.capturePid = 4242; + expect(api.profilingActive).toBe(false); + + await instance.dispose(); + + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + expect(api.captureProcess).toBeNull(); + }); + + it("marks the session disposed, so a resuming start can see it is gone", async () => { + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + expect(api.disposed).toBe(false); + + await instance.dispose(); + + expect(api.disposed).toBe(true); + }); + + it("Android: fails the start instead of reporting a recording nothing can stop", async () => { + const instance = await session(androidDevice); + const api = instance.api as NativeProfilerSessionApi; + + // The teardown lands while perfetto is still coming up. + startPerfettoMock.mockImplementation(async () => { + await instance.dispose(); + return { + pid: 9001, + onDeviceTracePath: "/data/misc/perfetto-traces/fake.pftrace", + child: new FakeChild() as unknown as ChildProcess, + }; + }); + + const err = await startNativeProfilerAndroid(api, { device_id: androidDevice.id }).catch( + (e: unknown) => e + ); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.NATIVE_PROFILER_SESSION_TORN_DOWN); + expect((err as Error).message).toContain("torn down by a stop-all-simulator-servers"); + // The session state must stay clean — a `status: "recording"` was the bug. + expect(api.profilingActive).toBe(false); + expect(api.recordingTimeout).toBeNull(); + // And the daemon this attempt spawned is this attempt's to reap: the + // teardown never saw it, because `capturePid` is handed over after the await. + expect(adbShellMock).toHaveBeenCalledWith(androidDevice.id, "kill -KILL 9001"); + expect(adbShellMock).toHaveBeenCalledWith( + androidDevice.id, + "rm -f /data/misc/perfetto-traces/fake.pftrace" + ); + }); + + it("Android: an undisturbed start still reports the recording", async () => { + // The control — the guard must not fire on the ordinary path. + const instance = await session(androidDevice); + const api = instance.api as NativeProfilerSessionApi; + startPerfettoMock.mockResolvedValue({ + pid: 9002, + onDeviceTracePath: "/data/misc/perfetto-traces/real.pftrace", + child: new FakeChild() as unknown as ChildProcess, + }); + + const result = await startNativeProfilerAndroid(api, { device_id: androidDevice.id }); + + expect(result.status).toBe("recording"); + expect(api.profilingActive).toBe(true); + await instance.dispose(); + }); +}); From b57d543287dbdd6fcf23dd5162753f41b42a4cfa Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:08:52 +0200 Subject: [PATCH 46/98] fix(flow): stop the write hint calling an ordinary flow file a symlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit writeFailureHint compared the fully realpath-resolved swap directory against an unresolved one, so any symlinked ANCESTOR tripped the clause — which on macOS is every /tmp and /var/folders path. The message then claimed the flow file "is a symlink" and contrasted two spellings of one directory. Return the resolved flows directory alongside the target and compare against that, so the clause fires only when the flow file itself is a link. --- .../tool-server/src/tools/flows/flow-utils.ts | 37 +++++++--- .../tool-server/test/flows/flow-utils.test.ts | 70 +++++++++++++++++++ 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 2b78596f3..8961aebd8 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2509,7 +2509,12 @@ let flowWriteSeq = 0; * (`ENAMETOOLONG` out of `rename`) into a report of a directory-permissions * problem the user would then go and not find. */ -function writeFailureHint(code: string | undefined, filePath: string, target: string): string { +function writeFailureHint( + code: string | undefined, + filePath: string, + target: string, + resolvedDir: string +): string { // The directory the swap actually uses — `dirname(realpath(filePath))`, not // `dirname(filePath)`. For a flow file that is a symlink into a shared vault // those are different directories, and only the first one can be the cause: @@ -2517,11 +2522,17 @@ function writeFailureHint(code: string | undefined, filePath: string, target: st // writable while the vault, the only unwritable thing in the picture, went // unmentioned. Say so when they differ, since "your flows dir is fine, the // link target is not" is the whole diagnosis there. + // + // Compared against the RESOLVED flows dir, not the spelled one. Every + // symlinked ANCESTOR moves the target too — which on macOS is every `/tmp` + // and `/var/folders` path — so comparing against the spelling accused a flow + // file that is a perfectly ordinary regular file of being a symlink, and + // contrasted two names for one directory. const dir = path.dirname(target); const via = - dir === path.dirname(filePath) + dir === resolvedDir ? "" - : ` (${path.basename(filePath)} is a symlink, so the write lands in ${dir}, not in ${path.dirname(filePath)})`; + : ` (${path.basename(filePath)} is a symlink, so the write lands in ${dir}, not in ${resolvedDir})`; switch (code) { case "EACCES": case "EPERM": @@ -2577,12 +2588,22 @@ function scrubTempPath(err: unknown, tmpPath: string, filePath: string): Error { * * Shared with {@link resolveFlowKey}, so the identity a recording is keyed by * and the file its steps land in can never disagree. + * + * `dir` — the flows directory as the filesystem sees it — is returned alongside, + * because it is the only thing a caller can compare `target`'s directory against + * to tell "the flow FILE is a symlink" from "some ancestor of it is". Comparing + * against the spelled `path.dirname(filePath)` cannot: on macOS every `/tmp` and + * `/var/folders` path has a symlinked ancestor. */ -async function canonicalFlowPath(filePath: string): Promise { +async function canonicalFlowTarget(filePath: string): Promise<{ dir: string; target: string }> { const dir = await fs.realpath(path.dirname(filePath)).catch(() => path.dirname(filePath)); const real = await fs.realpath(filePath).catch(() => null); - if (real !== null) return real; - return followDanglingLink(path.join(dir, path.basename(filePath))); + if (real !== null) return { dir, target: real }; + return { dir, target: await followDanglingLink(path.join(dir, path.basename(filePath))) }; +} + +async function canonicalFlowPath(filePath: string): Promise { + return (await canonicalFlowTarget(filePath)).target; } /** @@ -2667,7 +2688,7 @@ async function followDanglingLink(linkPath: string): Promise { * it does not survive an append. */ async function writeFlowFile(filePath: string, content: string): Promise { - const target = await canonicalFlowPath(filePath); + const { dir: resolvedDir, target } = await canonicalFlowTarget(filePath); const tmpPath = path.join( path.dirname(target), `.argent-flow-${process.pid}-${++flowWriteSeq}.tmp` @@ -2695,7 +2716,7 @@ async function writeFlowFile(filePath: string, content: string): Promise { const errno = err instanceof Error ? (err as NodeJS.ErrnoException) : undefined; const code = typeof errno?.code === "string" ? errno.code : undefined; throw new FailureError( - `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — ${writeFailureHint(code, filePath, target)}`, + `Failed to write flow file ${filePath}${code ? ` (${code})` : ""} — ${writeFailureHint(code, filePath, target, resolvedDir)}`, { error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, failure_stage: "flow_file_write", diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index 6a6ddfa4a..bad8bfb83 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -19,6 +19,7 @@ import { getFlowPath, appIdForPlatform, chromiumLaunchSpec, + writeNewFlowFile, type FlowFile, } from "../../src/tools/flows/flow-utils"; @@ -1775,3 +1776,72 @@ describe("countStepsOnDisk", () => { expect(await countStepsOnDisk(asDir)).toBeUndefined(); }); }); + +describe("writeFlowFile failure hints", () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "flow-write-hint-")); + }); + + afterEach(async () => { + // Restore write permission first, or the recursive rm cannot descend. + for (const dir of [path.join(root, "vault"), path.join(root, ".argent", "flows")]) { + await fs.chmod(dir, 0o755).catch(() => {}); + } + await fs.rm(root, { recursive: true, force: true }); + }); + + /** Whether this process can be denied by mode bits at all (root cannot). */ + async function modeBitsBite(dir: string): Promise { + await fs.chmod(dir, 0o555); + const probe = path.join(dir, ".probe"); + const denied = await fs + .writeFile(probe, "x", "utf8") + .then(() => false) + .catch(() => true); + if (!denied) await fs.rm(probe, { force: true }); + return denied; + } + + it("does not call the flow file a symlink when only an ANCESTOR is one", async () => { + // On macOS the temp dir is reached through /var -> /private/var, so the + // resolved swap directory differs from the spelled one for a flow file that + // is a perfectly ordinary regular file. Comparing the two spellings made + // every such failure claim a symlink and then contrast one directory with + // itself. + const flowsDir = path.join(root, ".argent", "flows"); + await fs.mkdir(flowsDir, { recursive: true }); + if (!(await modeBitsBite(flowsDir))) return; + + const err = await writeNewFlowFile(path.join(flowsDir, "x.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + const message = (err as Error).message; + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + expect(message).toContain("must be writable"); + expect(message).not.toMatch(/is a symlink/); + }); + + it("still points at the vault when the flow file really is a symlink", async () => { + // The case the clause exists for: naming `.argent/flows` here would send the + // reader to a directory that is already writable while the vault, the only + // unwritable thing in the picture, went unmentioned. + const flowsDir = path.join(root, ".argent", "flows"); + const vault = path.join(root, "vault"); + await fs.mkdir(flowsDir, { recursive: true }); + await fs.mkdir(vault, { recursive: true }); + await fs.writeFile(path.join(vault, "shared.yaml"), "steps: []\n", "utf8"); + await fs.symlink(path.join(vault, "shared.yaml"), path.join(flowsDir, "shared.yaml")); + if (!(await modeBitsBite(vault))) return; + + const err = await writeNewFlowFile(path.join(flowsDir, "shared.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + const message = (err as Error).message; + expect(message).toContain("shared.yaml is a symlink, so the write lands in"); + expect(message).toContain(await fs.realpath(vault)); + }); +}); From 947c4b5e6b58b51262f10300488659b7dbc7bb7d Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:11:04 +0200 Subject: [PATCH 47/98] fix(debugger): clear the reaped-session breadcrumb on an explicit connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The breadcrumb's only consumer, debugger-log-registry, is gated on an EMPTY registry, so one left behind survives every read that finds entries — and then attaches "a teardown ate your logs" to a later, unrelated empty read, which the tool description tells the agent to trust. debugger-connect now drops it, the way the screen-recording and native-profiler starts drop theirs: from an explicit connect the capture is this session's own, so an empty registry honestly means nothing has been logged since. Not in the blueprint factory, which also runs for the implicit resolve debugger-log-registry itself performs — clearing there would consume the breadcrumb one line before the read that exists to report it. --- .../src/tools/debugger/debugger-connect.ts | 20 ++++++++++- .../test/metro/teardown-log-history.test.ts | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/src/tools/debugger/debugger-connect.ts b/packages/tool-server/src/tools/debugger/debugger-connect.ts index 19e281c5d..a7ab76811 100644 --- a/packages/tool-server/src/tools/debugger/debugger-connect.ts +++ b/packages/tool-server/src/tools/debugger/debugger-connect.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { ToolDefinition } from "@argent/registry"; import type { JsRuntimeDebuggerApi } from "../../blueprints/js-runtime-debugger"; import { DEBUGGER_TOOL_CAPABILITY, debuggerServiceRef } from "./debugger-service-ref"; +import { takeReapedSession } from "../../utils/reaped-sessions"; const zodSchema = z.object({ port: z.coerce @@ -44,8 +45,25 @@ Use when starting a debug session or before calling other debugger-* tools. Fail services: (params) => ({ debugger: debuggerServiceRef(params), }), - async execute(services) { + async execute(services, params) { const api = services.debugger as JsRuntimeDebuggerApi; + // Drop any teardown breadcrumb for this device, the way the screen-recording + // and native-profiler starts drop theirs. Its only consumer, + // `debugger-log-registry`, is gated on an EMPTY registry, so one left here + // survives every read that finds entries — and then attaches "a teardown ate + // your logs" to some later, unrelated empty read, which the tool description + // tells the agent to trust. An explicit connect makes it wrong anyway: from + // here the capture is this session's, so an empty registry honestly means + // this app has logged nothing since. + // + // Not in the blueprint's factory: that runs for an IMPLICIT resolve too — + // `debugger-log-registry` reconnects through it — and clearing there would + // consume the breadcrumb one line before the read that exists to report it. + for (const id of new Set( + [params.device_id, api.logicalDeviceId].filter((v): v is string => v !== undefined) + )) { + takeReapedSession("js-runtime-debugger", id); + } return { port: api.port, projectRoot: api.projectRoot, diff --git a/packages/tool-server/test/metro/teardown-log-history.test.ts b/packages/tool-server/test/metro/teardown-log-history.test.ts index cfa670d90..8b333d1e3 100644 --- a/packages/tool-server/test/metro/teardown-log-history.test.ts +++ b/packages/tool-server/test/metro/teardown-log-history.test.ts @@ -179,6 +179,27 @@ describe("a debugger session reaped by stop-all-simulator-servers", () => { expect(second.note).toBeUndefined(); }); + it("is dropped by an explicit debugger-connect, which starts a capture of its own", async () => { + // The consumer is gated on an EMPTY registry, so a breadcrumb survives every + // read that finds entries — and would then attach "a teardown ate your logs" + // to some later, unrelated empty read. An explicit connect makes it wrong + // anyway: from there the capture is this session's own, so empty honestly + // means nothing has been logged since. Same discipline as the + // screen-recording and native-profiler starts. + const urn = await connectAndCapture(LOGICAL_ID, 40); + await registry.disposeService(urn); + + await registry.invokeTool("debugger-connect", { port: mockPort, device_id: LOGICAL_ID }); + + const result = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id: LOGICAL_ID, + })) as { totalEntries: number; note?: string }; + + expect(result.totalEntries).toBe(0); + expect(result.note).toBeUndefined(); + }); + describe("when the connect id and the logicalDeviceId differ", () => { // Every case above connects with LOGICAL_ID, so `api.logicalDeviceId === // deviceId` and the disposer's SECOND recordReapedSession never fires — @@ -236,5 +257,20 @@ describe("a debugger session reaped by stop-all-simulator-servers", () => { expect(viaLogicalId.note).toBeUndefined(); expect(again.note).toBeUndefined(); }); + + it("drops BOTH breadcrumbs on an explicit connect, under either spelling", async () => { + const urn = await connectAndCapture(CONNECT_ID, 11); + await registry.disposeService(urn); + + await registry.invokeTool("debugger-connect", { port: mockPort, device_id: CONNECT_ID }); + + for (const device_id of [CONNECT_ID, LOGICAL_ID]) { + const result = (await registry.invokeTool("debugger-log-registry", { + port: mockPort, + device_id, + })) as { note?: string }; + expect(result.note).toBeUndefined(); + } + }); }); }); From c2474d6e79fb1a193afd4a67ae283d2a7d9db6bf Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:12:29 +0200 Subject: [PATCH 48/98] fix(reaped-sessions): stop the breadcrumb pinning the teardown on one caller The message always said "torn down by a stop-all-simulator-servers", but a blueprint's dispose() is called by Registry._teardown with no caller, so nothing that writes a breadcrumb knows which tool triggered it. Two other routes reach the same services: stop-simulator-server on Chromium cascades into the debugger through ChromiumCdp, and react-profiler-start { force: true } disposes it to reclaim the session. Name the family instead, keeping the common case first. --- .../tool-server/src/utils/reaped-sessions.ts | 25 +++++++++++++------ .../tool-server/test/reaped-sessions.test.ts | 20 +++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/tool-server/src/utils/reaped-sessions.ts b/packages/tool-server/src/utils/reaped-sessions.ts index 63edb8e72..cea84e482 100644 --- a/packages/tool-server/src/utils/reaped-sessions.ts +++ b/packages/tool-server/src/utils/reaped-sessions.ts @@ -76,17 +76,28 @@ export function takeReapedSession( } /** - * The sentence a tool shows in place of "no active session". Names the cause, - * says it is not necessarily this agent's own doing (one tool-server serves - * every agent), and points at whatever survived. + * The sentence a tool shows in place of "no active session". Names what + * happened, says it is not necessarily this agent's own doing (one tool-server + * serves every agent), and points at whatever survived. + * + * The disposer that leaves a breadcrumb cannot see who triggered it — a + * blueprint's `dispose()` is called by `Registry._teardown`, with no caller — so + * the message names the family rather than asserting one member. + * `stop-all-simulator-servers` is the common one and is named first, but it is + * not the only one: `stop-simulator-server` on Chromium cascades into the + * debugger through `ChromiumCdp` (its documented behaviour), and + * `react-profiler-start { force: true }` disposes the debugger and the profiler + * session to reclaim them. */ export function describeReapedSession(entry: ReapedSession, what: string): string { const secondsAgo = Math.max(0, Math.round((Date.now() - entry.atMs) / 1000)); return ( - `The ${what} for device ${entry.deviceId} was torn down ${secondsAgo}s ago by a ` + - `stop-all-simulator-servers, which reaps every service a device owns — one tool-server ` + - `serves every agent using this argent install, so this may have been another agent ending ` + - `its session. It was not a session that never started.` + + `The ${what} for device ${entry.deviceId} was torn down ${secondsAgo}s ago — by a ` + + `stop-all-simulator-servers, which reaps every service a device owns, or by another ` + + `teardown that reaches the same services (a stop-simulator-server on Chromium, or a ` + + `react-profiler-start reclaiming the session with force). One tool-server serves every ` + + `agent using this argent install, so this may have been another agent rather than your own ` + + `call. It was not a session that never started.` + (entry.salvage ? ` ${entry.salvage}` : "") ); } diff --git a/packages/tool-server/test/reaped-sessions.test.ts b/packages/tool-server/test/reaped-sessions.test.ts index 81aec4add..e3b0b96d9 100644 --- a/packages/tool-server/test/reaped-sessions.test.ts +++ b/packages/tool-server/test/reaped-sessions.test.ts @@ -63,6 +63,26 @@ describe("the reaped-session key", () => { expect(takeReapedSession("screen-recording", UDID)).toBeUndefined(); }); + it("does not pin the teardown on one caller the disposer cannot have seen", () => { + // A blueprint's dispose() is called by Registry._teardown with no caller, so + // nothing that writes a breadcrumb knows which tool triggered it. + // stop-all-simulator-servers is the common one, but stop-simulator-server on + // Chromium cascades into the debugger through ChromiumCdp, and + // react-profiler-start { force: true } disposes it to reclaim the session — + // so the message names the family rather than asserting one member. + recordReapedSession("js-runtime-debugger", UDID); + + const message = describeReapedSession( + takeReapedSession("js-runtime-debugger", UDID)!, + "JS-runtime debugger session" + ); + expect(message).toContain("stop-all-simulator-servers"); + expect(message).toContain("stop-simulator-server on Chromium"); + expect(message).toContain("react-profiler-start"); + // The claim that made it wrong two ways out of three. + expect(message).not.toMatch(/torn down \d+s ago by a stop-all-simulator-servers/); + }); + it("omits the salvage clause entirely when nothing survived", () => { recordReapedSession("native-profiler", UDID); From 8d5d4e3456c4a9d0a2aba1d5f2166ee41d4b51e4 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:14:25 +0200 Subject: [PATCH 49/98] fix(native-profiler): leave a breadcrumb for a capped or crashed capture too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit abandonedCapture read profilingActive alone, but the 10-minute cap and the unexpected-exit handler both clear that flag while leaving the trace recoverable — native-profiler-stop has a whole branch for exporting it. A teardown there destroyed the owner's only route to the trace and left no breadcrumb, so the stop tool reverted to "you never started one". That arm also needs its own salvage text: it already sent SIGINT (or the process exited on its own), so on iOS the bundle was finalized rather than half-written, and on Android the on-device .pftrace is still there because dispose's `rm -f` branch never runs. --- .../src/blueprints/native-profiler-session.ts | 65 ++++++++++++++++--- .../native-profiler-reaped-session.test.ts | 59 +++++++++++++++++ 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index deb1e67aa..ea351e951 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -117,6 +117,45 @@ export interface NativeProfilerSessionApi { const DISPOSE_REAP_MS = 1_000; const ANDROID_DISPOSE_ADB_TIMEOUT_MS = 5_000; +/** + * What survived an iOS teardown, per arm — see the two flags in `dispose()`. + * `midCapture` names the arm that was still recording; the other one had + * already stopped, by the 10-minute cap's SIGINT or by xctrace exiting on its + * own, so its bundle went through a finalize pass and calling it half-written + * would send the owner away from a trace they can still read. + */ +function iosSalvage(midCapture: boolean, traceFile: string | null): string | undefined { + if (!traceFile) return undefined; + return midCapture + ? `xctrace was killed without its finalize pass, so the partial bundle at ${traceFile} is ` + + `very likely unreadable — re-profile rather than trying to salvage it.` + : `The recording had already ended before this teardown (the 10-minute cap, or xctrace ` + + `exiting on its own), so the bundle at ${traceFile} was finalized and may well be ` + + `readable — but this session was the only thing that could export it, so re-profile ` + + `unless you can open that bundle yourself.`; +} + +/** The Android twin of {@link iosSalvage}. */ +function androidSalvage(midCapture: boolean, onDeviceTracePath: string | null): string { + if (midCapture) { + // The on-device .pftrace is removed by the kill branch, and nothing was + // pulled to the host yet, so there is genuinely nothing to point at. + return ( + "The perfetto daemon was killed and its on-device trace removed, so no trace " + + "survived — re-profile to capture again." + ); + } + // The cap arm sent SIGTERM and cleared `profilingActive`, so the kill branch + // does not run and the trace is still on the device — but only this session + // knew to pull it. + return ( + `The recording had already ended before this teardown (the 10-minute cap), so the ` + + `on-device trace was left in place${onDeviceTracePath ? ` at ${onDeviceTracePath}` : ""} — ` + + `but this session was the only thing that could pull it to the host. Re-profile, or ` + + `\`adb pull\` it yourself.` + ); +} + function clearLiveState(state: NativeProfilerSessionApi): void { state.profilingActive = false; state.capturePid = null; @@ -204,7 +243,20 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< // Android the on-device trace is removed outright — so the breadcrumb // exists purely so `native-profiler-stop` stops answering "call // native-profiler-start first" for a session that really did run. - const abandonedCapture = state.profilingActive; + const midCapture = state.profilingActive; + // …and a capture the 10-minute cap or an unexpected exit already ended + // is one that RAN too. Those arms clear `profilingActive` while leaving + // the trace recoverable — `native-profiler-stop` has a whole branch for + // exporting it — so a teardown here still destroys the owner's only way + // to reach it, and gating the breadcrumb on `profilingActive` alone sent + // that owner back to "you never started one". It is also destroyed + // DIFFERENTLY: that arm already sent SIGINT (or the process exited on + // its own), so the salvage text below must not call the bundle a + // half-written one. + const endedCapture = + (state.recordingTimedOut || state.recordingExitedUnexpectedly) && + state.traceFile !== null; + const abandonedCapture = midCapture || endedCapture; const abandonedTrace = state.traceFile; if (state.platform === "ios") { @@ -230,11 +282,7 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< recordReapedSession( "native-profiler", state.deviceId, - abandonedTrace - ? `xctrace was killed without its finalize pass, so the partial bundle at ` + - `${abandonedTrace} is very likely unreadable — re-profile rather than ` + - `trying to salvage it.` - : undefined + iosSalvage(midCapture, abandonedTrace) ); } } @@ -262,10 +310,7 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< recordReapedSession( "native-profiler", state.deviceId, - // The on-device .pftrace is removed above, and nothing was pulled - // to the host yet, so there is genuinely nothing to point at. - "The perfetto daemon was killed and its on-device trace removed, so no trace " + - "survived — re-profile to capture again." + androidSalvage(midCapture, onDeviceTracePath) ); } } diff --git a/packages/tool-server/test/native-profiler-reaped-session.test.ts b/packages/tool-server/test/native-profiler-reaped-session.test.ts index 010f1d60f..704798d78 100644 --- a/packages/tool-server/test/native-profiler-reaped-session.test.ts +++ b/packages/tool-server/test/native-profiler-reaped-session.test.ts @@ -86,6 +86,65 @@ describe("a native profiling session reaped by stop-all-simulator-servers", () = expect(message).toContain("no trace survived"); }); + it("iOS: still explains a capped capture, and does not call its bundle half-written", async () => { + // The 10-minute cap SIGINTs xctrace and clears `profilingActive` while + // leaving the trace recoverable — `native-profiler-stop` has a whole branch + // for exporting it. Gating the breadcrumb on `profilingActive` sent the + // owner of such a capture back to "you never started one", and the + // mid-capture salvage text would have been wrong there too: that arm's + // bundle went through a finalize pass. + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + api.profilingActive = false; + api.recordingTimedOut = true; + api.traceFile = "/tmp/argent-capped.trace"; + + await instance.dispose(); + + const fresh = (await session(iosDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerIos(fresh).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).not.toMatch(/Call native-profiler-start first/); + expect(message).toContain("/tmp/argent-capped.trace"); + expect(message).toContain("already ended before this teardown"); + expect(message).not.toMatch(/without its finalize pass/); + }); + + it("iOS: explains a capture that exited on its own the same way", async () => { + const instance = await session(iosDevice); + const api = instance.api as NativeProfilerSessionApi; + api.recordingExitedUnexpectedly = true; + api.traceFile = "/tmp/argent-crashed.trace"; + + await instance.dispose(); + + const fresh = (await session(iosDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerIos(fresh).catch((e: unknown) => e); + + expect((err as Error).message).toContain("already ended before this teardown"); + }); + + it("Android: says the capped trace is still on the device, not that none survived", async () => { + // The Android cap sends SIGTERM and clears `profilingActive`, so dispose's + // `rm -f` branch never runs — the on-device .pftrace really is still there. + const instance = await session(androidDevice); + const api = instance.api as NativeProfilerSessionApi; + api.recordingTimedOut = true; + api.traceFile = "/tmp/host.pftrace"; + api.androidOnDeviceTracePath = "/data/misc/perfetto-traces/capped.pftrace"; + + await instance.dispose(); + + const fresh = (await session(androidDevice)).api as NativeProfilerSessionApi; + const err = await stopNativeProfilerAndroid(fresh).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).toContain("/data/misc/perfetto-traces/capped.pftrace"); + expect(message).toContain("left in place"); + expect(message).not.toMatch(/no trace survived/); + }); + it("leaves a plain absence alone when the disposed session was idle", async () => { // Disposing a session nobody was profiling with is routine cleanup. If that // left a breadcrumb, the next honest "you never started one" would accuse a From ce56604cba541b897ddc98ab44554c50a03b12b4 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:16:07 +0200 Subject: [PATCH 50/98] fix(flow): stop the cleanup-flow device resolve swallowing genuine failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveRunDevice's bare `catch {}` was scoped by its comment to "nothing booted, or several", but resolveFlowDevice reaches list-devices through the registry, so an adb/simctl error, a dead sub-tool or an abort landed there too. The teardown step then ran UNSCOPED and reported pass — the machine-wide sweep this path exists to avoid, on a machine whose device list nobody could read. Swallow only FLOW_DEVICE_RESOLUTION; rethrow the rest. --- .../tool-server/src/tools/flows/flow-run.ts | 10 ++++++++- .../test/flows/flow-deviceless.test.ts | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index ae774818d..6e10c88f0 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -1166,12 +1166,20 @@ async function resolveRunDevice( // a sweep has an answer to, so run it unscoped rather than failing the // flow. Swallowed only here: every other caller genuinely needs the // device, and the diagnosis in the error is the useful thing there. + // + // And swallowed only for THAT answer. `resolveFlowDevice` also reaches + // `list-devices` through the registry, so a bare catch also absorbed an + // adb/simctl failure, a dead sub-tool, an abort — and the teardown step + // then ran unscoped and reported pass, which is the machine-wide sweep + // this whole path exists to avoid. Anything that is not the ambiguity + // rethrows and fails the run. try { return { device: await resolveFlowDevice(registry, ctx, resolveOpts(params)), booted: null, }; - } catch { + } catch (err) { + if (getFailureSignal(err)?.error_code !== FAILURE_CODES.FLOW_DEVICE_RESOLUTION) throw err; return { device: null, booted: null }; } } diff --git a/packages/tool-server/test/flows/flow-deviceless.test.ts b/packages/tool-server/test/flows/flow-deviceless.test.ts index 069180481..c97573188 100644 --- a/packages/tool-server/test/flows/flow-deviceless.test.ts +++ b/packages/tool-server/test/flows/flow-deviceless.test.ts @@ -400,6 +400,27 @@ describe("a cleanup flow whose only step is stop-all-simulator-servers", () => { expect(invokeTool).toHaveBeenCalledWith("stop-all-simulator-servers", {}); }); + it("fails the run when list-devices itself breaks, rather than sweeping the machine", async () => { + // The opportunistic resolve swallows one answer — "nothing booted, or + // several" — and used to swallow every other failure with it: an + // adb/simctl error, a dead sub-tool, an abort. The teardown then ran + // UNSCOPED and reported pass, which is the machine-wide sweep this path + // exists to avoid, on a machine whose device list nobody could even read. + await writeFlow("teardownonly", teardownOnly); + const { registry, invokeTool } = mockRegistry({ booted: [DEVICE] }); + vi.mocked(registry.invokeTool).mockImplementation(async (id: string) => { + if (id === "list-devices") throw new Error("adb: device offline"); + return { ok: true }; + }); + + await expect(runAuto(registry, "teardownonly")).rejects.toThrow(/adb: device offline/); + expect(invokeTool).not.toHaveBeenCalledWith( + "stop-all-simulator-servers", + expect.anything(), + expect.anything() + ); + }); + it("still scopes the teardown when the flow ALSO has a device step", async () => { // A flow with a real device step resolves one as it always did, and the // teardown is scoped to it — the cross-agent protection the scope exists From 28920bc7ba5f14fc81f545e521deccc01edb886a Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:21:39 +0200 Subject: [PATCH 51/98] fix(flow): clear a finished recording by the key the session holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clearRecordingSession re-resolved its spelling instead of using session.key — the opposite of the deliberate choice appendStepToFlow documents. Once the flow file's identity has moved under the session (a symlink repointed mid-recording, in the window between requireRecordingSession and the finish's own file read), the re-resolution looks up a key the map no longer holds: the delete missed silently and the finish reported success while the session stayed live, unfinishable, and holding the key against its own restart. Takes the session, so re-resolution is not expressible. --- .../src/tools/flows/flow-finish-recording.ts | 2 +- .../tool-server/src/tools/flows/flow-utils.ts | 14 +++++- .../tool-server/test/flows/flow-utils.test.ts | 44 +++++++++++++++++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-finish-recording.ts b/packages/tool-server/src/tools/flows/flow-finish-recording.ts index ad1bebfb5..520ae9fa5 100644 --- a/packages/tool-server/src/tools/flows/flow-finish-recording.ts +++ b/packages/tool-server/src/tools/flows/flow-finish-recording.ts @@ -119,7 +119,7 @@ You can still edit the .yaml file directly afterwards to remove or reorder steps // {@link renderToolArgs}; keeping the order is what makes the next one // recoverable rather than fatal. const summary = summarizeSteps(flow); - await clearRecordingSession(params.project_root, params.name); + clearRecordingSession(session); return { filePath, flowFile, savedTo, flow, summary }; } ); diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 8961aebd8..b7dc79dc1 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -540,8 +540,18 @@ export async function requireRecordingSession( return session; } -export async function clearRecordingSession(projectRoot: string, name: string): Promise { - recordings.delete(await resolveFlowKey(projectRoot, name)); +/** + * Retire a finished recording, by the key the session actually HOLDS rather + * than a fresh resolution of its spelling — the same choice + * {@link appendStepToFlow} makes, and for the same reason. A key that moved + * under the session (a symlinked flow file whose target went away + * mid-recording, or a link repointed) re-resolves to something this map does + * not hold, so the delete missed silently: the finish reported success while + * the session stayed live, unfinishable, and holding the key against its own + * restart. + */ +export function clearRecordingSession(session: RecordingSession): void { + recordings.delete(session.key); } export function __resetRecordingsForTesting(): void { diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index bad8bfb83..b23a44f11 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1153,7 +1153,7 @@ describe("recording sessions", () => { it("clearRecordingSession removes only that key", async () => { await start("/tmp/proj-a", "my-flow"); await start("/tmp/proj-a", "other-flow"); - await clearRecordingSession("/tmp/proj-a", "my-flow"); + clearRecordingSession(await requireRecordingSession("/tmp/proj-a", "my-flow")); expect(await getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); await expect(requireRecordingSession("/tmp/proj-a", "my-flow")).rejects.toThrow( /No active recording for flow "my-flow"/ @@ -1162,6 +1162,44 @@ describe("recording sessions", () => { expect((await requireRecordingSession("/tmp/proj-a", "other-flow")).name).toBe("other-flow"); }); + it("clearRecordingSession deletes by the key the session HOLDS, not a fresh resolution", async () => { + // The same choice appendStepToFlow documents. Re-resolving the spelling + // looks up a key the map may no longer hold once the flow file's identity + // has moved under the session — a symlink repointed mid-recording — so the + // delete missed silently and the finish reported success while the session + // stayed live, unfinishable, and holding the key against its own restart. + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "clear-moved-key-")); + try { + const root = path.join(dir, "proj"); + const flows = path.join(root, ".argent", "flows"); + await fs.mkdir(flows, { recursive: true }); + const first = path.join(dir, "first.yaml"); + const second = path.join(dir, "second.yaml"); + for (const f of [first, second]) await fs.writeFile(f, "steps: []\n", "utf8"); + const link = path.join(flows, "shared.yaml"); + await fs.symlink(first, link); + + await startRecordingSession({ + name: "shared", + projectRoot: root, + persist: "host", + filePath: link, + flow: emptyFlow(), + }); + const session = (await getRecordingSession(root, "shared"))!; + + await fs.rm(link); + await fs.symlink(second, link); + // The spelling now resolves to a different file entirely. + expect(await getRecordingSession(root, "shared")).toBeUndefined(); + + clearRecordingSession(session); + expect(listActiveRecordings()).toEqual([]); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + it("keeps same-named recordings under different project roots independent", async () => { await start("/tmp/proj-a", "my-flow", { executionPrerequisite: "A", steps: [] }); await start("/tmp/proj-b", "my-flow", { executionPrerequisite: "B", steps: [] }); @@ -1172,7 +1210,7 @@ describe("recording sessions", () => { (await requireRecordingSession("/tmp/proj-b", "my-flow")).flow.executionPrerequisite ).toBe("B"); // Finishing one leaves the other recording. - await clearRecordingSession("/tmp/proj-a", "my-flow"); + clearRecordingSession(await requireRecordingSession("/tmp/proj-a", "my-flow")); expect(await getRecordingSession("/tmp/proj-a", "my-flow")).toBeUndefined(); expect( (await requireRecordingSession("/tmp/proj-b", "my-flow")).flow.executionPrerequisite @@ -1235,7 +1273,7 @@ describe("recording sessions", () => { { name: "my-flow", projectRoot: "/tmp/proj-a", steps: 1 }, { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, ]); - await clearRecordingSession("/tmp/proj-a", "my-flow"); + clearRecordingSession(await requireRecordingSession("/tmp/proj-a", "my-flow")); expect(listActiveRecordings()).toEqual([ { name: "my-flow", projectRoot: "/tmp/proj-b", steps: 0 }, ]); From 15d42801e07a7b9aeed113447ec6b3b35a341106 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:21:39 +0200 Subject: [PATCH 52/98] test(flow): pin that a deleted vault target no longer orphans the recording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dangling-symlink resolution keeps the recording key stable when the target goes away mid-take, so the session stays addressable and the append fails as the missing file it is rather than as a recording that was never started — the second answer sends the agent to flow-start-recording, which truncates. Restoring the target resumes the same take. --- .../flows/flow-concurrent-recording.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index a6884c06b..4a0a8e943 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -482,6 +482,41 @@ describe("two recording keys that resolve to one file", () => { expect((await fs.lstat(flowPath(rootA, "checkout"))).isSymbolicLink()).toBe(true); }); + it("keeps a recording reachable when its vault target is deleted mid-take", async () => { + // The link is still there and still names the same file, so the recording's + // identity has not moved — it is only the target that is momentarily + // absent. Resolving that back to the link's own path made the key move, + // orphaning the live session behind a generic "no active recording". + const vault = await makeRoot("deleted-target-vault"); + const root = await makeRoot("deleted-target-proj"); + const target = path.join(vault, "checkout.yaml"); + await fs.mkdir(path.dirname(flowPath(root, "checkout")), { recursive: true }); + await fs.symlink(target, flowPath(root, "checkout")); + + await start(root, "checkout"); + await addEcho(root, "checkout", "c1"); + await fs.rm(target); + + // Still addressable under the spelling it was started with. + expect((await getRecordingSession(root, "checkout"))?.name).toBe("checkout"); + + // The append does fail — its file really is gone — but as the missing file + // it is, not as a recording that was never started. The distinction is the + // whole point: the second answer sends the agent to flow-start-recording, + // which truncates. + const err = await captureFailure(addEcho(root, "checkout", "c2")); + expect(getFailureSignal(err)?.error_code).not.toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + expect((err as Error).message).toMatch(/ENOENT/); + + // And restoring the target resumes the same take. + await fs.writeFile(target, "steps: []\n", "utf8"); + await addEcho(root, "checkout", "c3"); + const finished = await finish(root, "checkout"); + expect(markers(parseFlow(finished.flowFile).steps)).toEqual(["echo:c3"]); + expect((await fs.lstat(flowPath(root, "checkout"))).isSymbolicLink()).toBe(true); + expect(await getRecordingSession(root, "checkout")).toBeUndefined(); + }); + it("keeps two genuinely distinct flows independent", async () => { // The control: no symlink, no case variance, so nothing is canonicalized // together and the isolation guarantee holds exactly as stated. From c368be59557d935c10f65bb791e3b6ea2fbfbb7c Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:23:34 +0200 Subject: [PATCH 53/98] fix(stop): honour the request abort signal during the sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit execute never took ctx, so a caller that had given up — an MCP client timing out, a cancelled CLI run — was still billed for a full loop of awaited disposals across thirteen namespaces. Check the signal between disposals (a dispose already under way finishes, since abandoning a blueprint mid-teardown leaks the handles this tool exists to free) and report the partial teardown as { aborted: true } rather than computing `unmatched` / `left_running` from a snapshot the sweep never finished reading. Also moves the `unmatched` caveats back next to `unmatched` in the description, where the left_running sentence had split them. --- .../simulator/stop-all-simulator-servers.ts | 23 ++++++- packages/tool-server/test/stop-tools.test.ts | 66 +++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts index 73a221c26..a304f4219 100644 --- a/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts +++ b/packages/tool-server/src/tools/simulator/stop-all-simulator-servers.ts @@ -35,7 +35,7 @@ export function createStopAllSimulatorServersTool( registry: Registry ): ToolDefinition< z.infer, - { stopped: string[]; unmatched?: string[]; left_running?: string[] } + { stopped: string[]; unmatched?: string[]; left_running?: string[]; aborted?: true } > { return { id: "stop-all-simulator-servers", @@ -80,10 +80,10 @@ export function createStopAllSimulatorServersTool( description: `Stop the services a device owns - simulator-server processes (iOS + Android), native devtools, the iOS accessibility service, TV-control daemons, Chromium CDP sessions, screen recordings, native profiler sessions, and JS-runtime debugger sessions along with the network inspectors and React profiler sessions that ride on them - freeing their spawned processes, sockets and ports. Call this when your session ends or the user says they are done. PASS \`devices\` with the device ids this session used — one tool-server serves every agent, subagent and CLI call using this argent install, and an unscoped call tears down THEIR devices too (a mid-recording devtools teardown degrades another agent's flow to brittle coordinate taps; that agent is warned, but its recorded steps are already the worse kind). Omit \`devices\` only when a machine-wide cleanup is what you actually want. Passing an EMPTY array scopes to nothing and stops nothing - it is not a way to ask for the machine-wide sweep. A JS-runtime debugger session is keyed by the id you called \`debugger-connect\` with. On a Metro serving two or more devices that id is not a udid or serial - connect refuses those and tells you to re-target with the \`logicalDeviceId\` it returns - so a scope built from \`list-devices\` ids cannot reach that session. Pass any such \`logicalDeviceId\` in \`devices\` ALONGSIDE the device id; { left_running } names the ones you missed. -Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. { left_running } lists live debugger sessions (and the network inspectors / React profiler sessions riding on them) whose id no device scope can name - re-call with that id to reap them. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. Past the schema - which rejects an unknown key outright, so the \`udids\` slip is an error rather than a silent machine-wide sweep - the call always succeeds; reaping nothing is a result, not a failure.`, +Returns { stopped } - the URNs of the services that were actually live and got shut down; an ERROR node is disposed too but never appears there, so an empty \`stopped\` only means nothing was still running. { unmatched } lists supplied ids that own no service here, so a mistyped id - or a device NAME passed where an id was expected - does not read as a clean machine. It is NOT proof the id is wrong: a Vega device is driven through CLI/adb shell-outs, so one you only booted and drove with the remote registers no service and always lands here — as does a real device of any platform this session never started anything on. Present ONLY when \`devices\` was supplied AND at least one id matched nothing - absent on an unscoped call and when every id matched. Stopping the same device twice does not report it unmatched: ownership counts regardless of service state. { left_running } lists live debugger sessions (and the network inspectors / React profiler sessions riding on them) whose id no device scope can name - re-call with that id to reap them. { aborted: true } means the caller cancelled the request part-way, so the rest of the machine was left untouched and neither of the other two fields was computed. Past the schema - which rejects an unknown key outright, so the \`udids\` slip is an error rather than a silent machine-wide sweep - the call always succeeds; reaping nothing is a result, not a failure.`, zodSchema, services: () => ({}), - async execute(_services, params) { + async execute(_services, params, ctx) { const devices = params.devices; // Present-but-empty scopes to nothing rather than falling back to the // machine-wide sweep: a caller that computed a device list and got none @@ -96,7 +96,19 @@ Returns { stopped } - the URNs of the services that were actually live and got s // ones are ever reported (see `unnameableSessionUrns`) — the rest are // other agents' devices, which a scoped stop leaves alone by design. const survivors: string[] = []; + let aborted = false; for (const [urn, entry] of snapshot.services) { + // A sweep is a loop of awaited disposals — thirteen namespaces, each + // reaping spawned processes and sockets — so a caller that has given up + // (an MCP client timing out, a cancelled CLI run) would otherwise be + // billed for the whole of it. Checked between disposals rather than + // inside one: a dispose already under way finishes, since abandoning a + // blueprint mid-teardown is what leaks the handles this tool exists to + // free. + if (ctx?.signal?.aborted) { + aborted = true; + break; + } const matchedId = scoped ? deviceIdOwningUrn(urn, DEVICE_OWNED_NAMESPACES, devices) : undefined; @@ -125,6 +137,11 @@ Returns { stopped } - the URNs of the services that were actually live and got s survivors.push(urn); } } + // An abort left the rest of the snapshot untouched, so neither `unmatched` + // nor `left_running` can be computed — an id whose only service the sweep + // never reached would read as a typo, and every namespace past the break + // would read as unreachable. Report the partial teardown as partial. + if (aborted) return { stopped, aborted: true }; if (!scoped) return { stopped }; // A scoped stop that named an id owning nothing is indistinguishable from // a clean machine unless we say so — and when that id is a typo, or a diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 72ea3fd7c..a75d7a901 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -1108,6 +1108,72 @@ describe("stop-all-simulator-servers unmatched ids", () => { }); }); +describe("stop-all-simulator-servers abort", () => { + // A sweep is a loop of awaited disposals across thirteen namespaces, each + // reaping spawned processes and sockets. Ignoring the request signal billed a + // caller who had already given up — an MCP client timing out, a cancelled CLI + // run — for the whole of it. + + it("stops sweeping once the request is aborted, and says the teardown is partial", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const controller = new AbortController(); + // Abort as soon as the first disposal has happened. + vi.mocked(registry.disposeService).mockImplementationOnce(async (urn: string) => { + services.get(urn)!.state = ServiceState.IDLE; + controller.abort(); + }); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }, { + signal: controller.signal, + } as never); + + expect(result).toEqual({ stopped: [`SimulatorServer:${MINE}`], aborted: true }); + expect(registry.disposeService).toHaveBeenCalledTimes(1); + }); + + it("does not report `unmatched` for a partial sweep it never finished reading", async () => { + // The id may well own a service further down the snapshot, so calling it a + // typo here would be a guess — and `left_running` would name every + // namespace past the break. + const services = new Map([ + [`SimulatorServer:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const controller = new AbortController(); + controller.abort(); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }, { + signal: controller.signal, + } as never); + + expect(result).toEqual({ stopped: [], aborted: true }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + + it("sweeps to completion when no signal is supplied", async () => { + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`NativeDevtools:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ + stopped: [`SimulatorServer:${MINE}`, `NativeDevtools:${MINE}`], + }); + }); +}); + describe("stop-all-simulator-servers left_running", () => { // With two or more devices on one Metro, `debugger-connect` refuses a udid / // serial and instructs the caller to re-target with the `logicalDeviceId` From b0b2c09cc0946a61138481d0fc73d628954843c3 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:28:00 +0200 Subject: [PATCH 54/98] fix(flow): keep a flow file's mode, and refuse to overwrite a read-only one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scratch file is created under this process's umask and rename carries ITS mode over, so every append quietly rewrote the flow file's permissions to 0644 — and since the swap needs permission on the DIRECTORY rather than on the file, a `chmod 0444` that a plain write refused now succeeded. Preserve the target's mode on the scratch file before the rename, and refuse up front when the existing flow file is not writable, so a read-only flow goes on meaning what it meant before the write became atomic. --- .../tool-server/src/tools/flows/flow-utils.ts | 47 ++++++++++-- .../tool-server/test/flows/flow-utils.test.ts | 71 +++++++++++++++++++ 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index b7dc79dc1..3b42e93d6 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -1,5 +1,6 @@ import * as path from "node:path"; import * as fs from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; import { FAILURE_CODES, FailureError } from "@argent/registry"; import { stringify as yamlStringify, parse as yamlParse } from "yaml"; import { @@ -2657,6 +2658,14 @@ async function followDanglingLink(linkPath: string): Promise { return current; } +/** Whether this process may write `filePath` — its mode as the kernel reads it. */ +async function isWritable(filePath: string): Promise { + return fs.access(filePath, fsConstants.W_OK).then( + () => true, + () => false + ); +} + /** * Replace a flow file's contents so no reader can ever observe it half-written. * @@ -2692,19 +2701,49 @@ async function followDanglingLink(linkPath: string): Promise { * tool-server (a different install bundle) that could be writing the same * directory. * - * The swap costs two things a write-through would have kept, both accepted for - * the atomicity: it needs write permission on the DIRECTORY rather than on the - * file, and it replaces the inode, so a chmod on the flow file or a hardlink to - * it does not survive an append. + * The swap costs one thing a write-through would have kept, accepted for the + * atomicity: it needs write permission on the DIRECTORY rather than on the + * file, and it replaces the inode, so a hardlink to the flow file does not + * survive an append. The file's own MODE is not among the costs — see below. */ async function writeFlowFile(filePath: string, content: string): Promise { const { dir: resolvedDir, target } = await canonicalFlowTarget(filePath); + // Null when the flow file does not exist yet (the first write of a recording), + // which has no mode to preserve and nothing to be refused by. + const previousMode = await fs.stat(target).then( + (s) => s.mode & 0o7777, + () => null + ); + if (previousMode !== null && !(await isWritable(target))) { + // The swap needs permission on the directory, not on the file, so it would + // replace a `chmod 0444` flow file regardless — turning a plain write's + // EACCES into a silent success that also relaxed the mode to the umask + // default. Refuse instead, so a read-only flow file goes on meaning what it + // meant before the write became atomic. + throw new FailureError( + `Failed to write flow file ${filePath} (EACCES) — ${target} is not writable ` + + `(mode ${previousMode.toString(8).padStart(4, "0")}). An append replaces the file via a ` + + `sibling temp file and rename, which needs permission on the directory rather than on ` + + `the file — so this is refused explicitly rather than quietly overwriting a flow you ` + + `made read-only. chmod it writable to record over it.`, + { + error_code: FAILURE_CODES.FLOW_FILE_WRITE_FAILED, + failure_stage: "flow_file_write", + failure_area: "tool_server", + error_kind: "unknown", + } + ); + } const tmpPath = path.join( path.dirname(target), `.argent-flow-${process.pid}-${++flowWriteSeq}.tmp` ); try { await fs.writeFile(tmpPath, content, "utf8"); + // The scratch file was created under this process's umask, and rename + // carries ITS mode over — so without this every append would quietly + // rewrite the flow file's permissions to 0644. + if (previousMode !== null) await fs.chmod(tmpPath, previousMode); // Atomic within a filesystem, and the temp file is a sibling of the target, // so it is always the same one. await fs.rename(tmpPath, target); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index b23a44f11..c2328b680 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import * as fs from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; @@ -1883,3 +1884,73 @@ describe("writeFlowFile failure hints", () => { expect(message).toContain(await fs.realpath(vault)); }); }); + +describe("flow file permissions across an atomic append", () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "flow-mode-")); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + /** Whether mode bits can refuse this process at all (root ignores them). */ + async function modeBitsBite(file: string): Promise { + return fs + .access(file, fsConstants.W_OK) + .then(() => false) + .catch(() => true); + } + + it("carries the flow file's mode across the swap", async () => { + // The scratch file is created under the process umask and rename carries + // ITS mode over, so without preserving it every append quietly rewrote the + // flow file's permissions to 0644. + const file = path.join(root, "flow.yaml"); + await fs.writeFile(file, "steps: []\n", "utf8"); + await fs.chmod(file, 0o600); + + await writeNewFlowFile(file, "steps: []\n"); + + expect((await fs.stat(file)).mode & 0o777).toBe(0o600); + }); + + it("refuses to overwrite a read-only flow file", async () => { + // The swap needs permission on the DIRECTORY, so it would replace a + // `chmod 0444` file regardless — turning a plain write's EACCES into a + // silent success that also relaxed the mode. + const file = path.join(root, "flow.yaml"); + await fs.writeFile(file, "steps: []\nkeep: me\n", "utf8"); + await fs.chmod(file, 0o444); + if (!(await modeBitsBite(file))) return; + + const err = await writeNewFlowFile(file, "steps: []\n").catch((e: unknown) => e); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + expect((err as Error).message).toMatch(/not writable \(mode 0444\)/); + // And it really did not touch the file. + expect(await fs.readFile(file, "utf8")).toContain("keep: me"); + }); + + it("leaves no scratch file behind when it refuses", async () => { + const flows = path.join(root, ".argent", "flows"); + await fs.mkdir(flows, { recursive: true }); + const file = path.join(flows, "flow.yaml"); + await fs.writeFile(file, "steps: []\n", "utf8"); + await fs.chmod(file, 0o444); + if (!(await modeBitsBite(file))) return; + + await writeNewFlowFile(file, "steps: []\n").catch(() => {}); + + expect(await fs.readdir(flows)).toEqual(["flow.yaml"]); + }); + + it("still creates a flow file that does not exist yet", async () => { + // The control: nothing to preserve and nothing to be refused by. + const file = path.join(root, "fresh.yaml"); + await writeNewFlowFile(file, "steps: []\n"); + expect(await fs.readFile(file, "utf8")).toBe("steps: []\n"); + }); +}); From 20358cb098bebd9945b6d6c9efe79822fae5f574 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:29:50 +0200 Subject: [PATCH 55/98] test(flow): make the finish-vs-append race pin both outcomes, not one seven times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The microtask loop claimed to exercise both, but every iteration produced `fulfilled` and the `if (rejected)` branch never ran: the finish awaits a real realpath before joining the lock queue, so no microtask tuning can make it overtake an append already queued. Replace it with two cases fixed by the lock rather than by timing — an append that wins the queue and must appear in what the finish reports, and one parked in its live phase across a completed finish, which must be rejected and must not be on disk — sharing one helper for the report-matches-disk invariant. --- .../flows/flow-concurrent-recording.test.ts | 103 +++++++++++------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 4a0a8e943..1990d554e 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -1458,46 +1458,71 @@ describe("a restart that lands while a step is still running", () => { // ── A finish landing on top of an in-flight append ─────────────────── describe("a finish that lands while a step is still running", () => { - it("never reports steps, a summary or YAML the file disagrees with", async () => { - // Vary how far the append has progressed when the finish arrives, so both - // outcomes are exercised: the append wins the lock (and must be included in - // what finish reports) or the finish wins it (and the append must fail). - for (const microtasks of [0, 1, 2, 3, 4, 6, 8]) { - const root = await makeRoot(`finish-inflight-${microtasks}`); - await start(root, "alpha"); - await addStep(root, "alpha", "a1"); - - const gate = gateNextSubTool(); - const appending = addStep(root, "alpha", "a2"); - await gate.reached; - gate.release(); - for (let i = 0; i < microtasks; i++) await Promise.resolve(); - - const [appended, finished] = await Promise.allSettled([appending, finish(root, "alpha")]); - - if (finished.status === "rejected") throw finished.reason; - const report = finished.value; - const onDisk = await readMarkers(root, "alpha"); - - // The whole report is one snapshot of one file state. - expect(markers(parseFlow(report.flowFile).steps)).toEqual(onDisk); - expect(report.steps).toBe(onDisk.length); - expect(report.summary).toHaveLength(onDisk.length); - expect(report.path).toBe(flowPath(root, "alpha")); - expect(report.savedTo).toBe(flowPath(root, "alpha")); - - if (appended.status === "rejected") { - expect(getFailureSignal(appended.reason)?.error_code).toBe( - FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING - ); - expect(onDisk).toEqual(["tool:a1"]); - } else { - expect(onDisk).toEqual(["tool:a1", "tool:a2"]); - } + /** + * The invariant both outcomes share: the whole report is one snapshot of one + * file state, and the recording is gone afterwards either way. + */ + async function expectReportMatchesDisk( + root: string, + report: Awaited> + ): Promise { + const onDisk = await readMarkers(root, "alpha"); + expect(markers(parseFlow(report.flowFile).steps)).toEqual(onDisk); + expect(report.steps).toBe(onDisk.length); + expect(report.summary).toHaveLength(onDisk.length); + expect(report.path).toBe(flowPath(root, "alpha")); + expect(report.savedTo).toBe(flowPath(root, "alpha")); + expect(await getRecordingSession(root, "alpha")).toBeUndefined(); + return onDisk; + } - // Either way the recording is gone, and nothing can be appended to it. - expect(await getRecordingSession(root, "alpha")).toBeUndefined(); - } + // Both outcomes are pinned, each by the lock rather than by timing. An + // earlier version varied a microtask count instead and only ever produced the + // first one: the finish awaits a real `realpath` before joining the lock + // queue, so no amount of microtask tuning can make it overtake an append + // already queued — the "append rejected" branch simply never ran. + + it("includes an append that WON the lock in everything it reports", async () => { + const root = await makeRoot("finish-inflight-append-wins"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + // A parked holder fixes the queue order: append, then finish. + const holder = openGate(); + const held = withFlowFileLock(root, "alpha", () => holder.promise); + const appending = addStep(root, "alpha", "a2"); + await settle(); + const finishing = finish(root, "alpha"); + holder.open(); + + const [appended, finished] = await Promise.allSettled([appending, finishing]); + await held; + + if (appended.status === "rejected") throw appended.reason; + if (finished.status === "rejected") throw finished.reason; + expect(await expectReportMatchesDisk(root, finished.value)).toEqual(["tool:a1", "tool:a2"]); + }); + + it("reports the file without an append that LOST, and rejects that append", async () => { + const root = await makeRoot("finish-inflight-finish-wins"); + await start(root, "alpha"); + await addStep(root, "alpha", "a1"); + + // Parked in its LIVE phase, before it has taken the lock — a real step can + // sit here for minutes on a device. The finish runs to completion across it. + const gate = gateNextSubTool(); + const appending = addStep(root, "alpha", "a2"); + await gate.reached; + + const report = await finish(root, "alpha"); + expect(await expectReportMatchesDisk(root, report)).toEqual(["tool:a1"]); + + gate.release(); + const appended = await captureFailure(appending); + expect(getFailureSignal(appended)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + // The step ran on the device but is in no take, and the file the finish + // reported is still exactly what is on disk. + expect(await readMarkers(root, "alpha")).toEqual(["tool:a1"]); }); it("reads the file back and clears the session only once the lock is free", async () => { From 266a62f30c9ca14b1c213b99c8a7817d3dae12ee Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:31:29 +0200 Subject: [PATCH 56/98] fix(test): repair the test-only typecheck after the session-api change `tsc --build` covers src only, so the new required `disposed` field on NativeProfilerSessionApi and the event-map generic in the ReactProfilerSession dispose test only surfaced under `typecheck:tests`. --- .../test/ios-instruments/analyze-freshness.test.ts | 1 + .../test/ios-instruments/malloc-stack-logging.test.ts | 1 + .../tool-server/test/native-profiler-analyze-failure.test.ts | 1 + .../tool-server/test/native-profiler-missing-trace.test.ts | 1 + .../tool-server/test/react-profiler/session-dispose.test.ts | 4 +++- 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/test/ios-instruments/analyze-freshness.test.ts b/packages/tool-server/test/ios-instruments/analyze-freshness.test.ts index f22b6d550..411e5294c 100644 --- a/packages/tool-server/test/ios-instruments/analyze-freshness.test.ts +++ b/packages/tool-server/test/ios-instruments/analyze-freshness.test.ts @@ -32,6 +32,7 @@ function makeApi(wallClockStartMs: number | null): NativeProfilerSessionApi { // null exporter paths → checkExportFileMissing short-circuits (no fs access); // the freshness note still renders in the all-clear header regardless. exportedFiles: { cpu: null, hangs: null, leaks: null }, + disposed: false, profilingActive: false, wallClockStartMs, parsedData: null, diff --git a/packages/tool-server/test/ios-instruments/malloc-stack-logging.test.ts b/packages/tool-server/test/ios-instruments/malloc-stack-logging.test.ts index 0337cca2d..baef551e6 100644 --- a/packages/tool-server/test/ios-instruments/malloc-stack-logging.test.ts +++ b/packages/tool-server/test/ios-instruments/malloc-stack-logging.test.ts @@ -35,6 +35,7 @@ function fakeApi(): NativeProfilerSessionApi { mallocStackLogging: null, traceFile: null, exportedFiles: null, + disposed: false, profilingActive: false, wallClockStartMs: null, parsedData: null, diff --git a/packages/tool-server/test/native-profiler-analyze-failure.test.ts b/packages/tool-server/test/native-profiler-analyze-failure.test.ts index 41524063a..7ca138e58 100644 --- a/packages/tool-server/test/native-profiler-analyze-failure.test.ts +++ b/packages/tool-server/test/native-profiler-analyze-failure.test.ts @@ -90,6 +90,7 @@ async function buildSessionWithTrace(): Promise<{ captureProcess: null, traceFile: tracePath, exportedFiles: { pftrace: tracePath }, + disposed: false, profilingActive: false, wallClockStartMs: null, parsedData: null, diff --git a/packages/tool-server/test/native-profiler-missing-trace.test.ts b/packages/tool-server/test/native-profiler-missing-trace.test.ts index 62d1fd145..1bc22874b 100644 --- a/packages/tool-server/test/native-profiler-missing-trace.test.ts +++ b/packages/tool-server/test/native-profiler-missing-trace.test.ts @@ -59,6 +59,7 @@ describe("native-profiler-analyze: missing trace file", () => { captureProcess: null, traceFile, exportedFiles: { cpu: cpuPath, hangs: hangsPath, leaks: leaksPath }, + disposed: false, profilingActive: false, wallClockStartMs: null, parsedData: null, diff --git a/packages/tool-server/test/react-profiler/session-dispose.test.ts b/packages/tool-server/test/react-profiler/session-dispose.test.ts index 2f266392d..4cc0c0c71 100644 --- a/packages/tool-server/test/react-profiler/session-dispose.test.ts +++ b/packages/tool-server/test/react-profiler/session-dispose.test.ts @@ -22,7 +22,9 @@ interface SentCall { } function fakeDebuggerApi(sent: SentCall[]): JsRuntimeDebuggerApi { - const events = new TypedEventEmitter void>>(); + // The CDP event map is not exported; nothing here subscribes, the emitter only + // has to exist for the factory's `cdp.events.on(...)` calls. + const events = new TypedEventEmitter void>>(); const cdp = { events, send: async (method: string, params?: Record) => { From e1712c6728a45f0bd047f3e3a4906f632c6d2517 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:34:50 +0200 Subject: [PATCH 57/98] test(stop): make the scoped-stop cases prove scoping, and the mock mirror the registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four cases survived an always-match matcher because each snapshot held only the target device's URNs — and AXService, ScreenRecordingSession, NativeProfilerSession and ChromiumJsRuntimeDebugger were covered only that way. Each now carries a second-device control. The registry mock is also brought back in line with Registry._teardown, which early-returns for TERMINATING as well as IDLE, and the fabricated `Metro:8081` node is gone: it is not a registry namespace, every namespace a blueprint declares as a dependency is device-owned, and what that case asserted was the mock's own recursion rather than any production line. It is replaced by the IDLE-dependent and TERMINATING cases, which are production lines that had no coverage. --- packages/tool-server/test/stop-tools.test.ts | 81 ++++++++++++++++---- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index a75d7a901..00bcea2a4 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -32,7 +32,11 @@ function createMockRegistry(services: Map { // the answer must not depend on which happened. const CDP = "ChromiumCdp:chromium-cdp-9222"; const CHROMIUM_DEBUGGER = "ChromiumJsRuntimeDebugger:chromium-cdp-9222"; + /** A second Electron instance, belonging to somebody else. */ + const OTHER_CDP = "ChromiumCdp:chromium-cdp-9333"; const live = () => ({ state: ServiceState.RUNNING, dependents: [] as string[] }); const cdpWithDependent = () => ({ state: ServiceState.RUNNING, @@ -327,9 +333,14 @@ describe("stop-all-simulator-servers", () => { // Both URNs carry the device id, so each is matched DIRECTLY; the // cascade is incidental here and this case is about insertion order not // changing membership. What the cascade alone decides is pinned below. - const services = new Map( - order.map((urn) => [urn, urn === CDP ? cdpWithDependent() : live()] as const) - ); + // + // A second chromium instance is the control: with only the target's URNs + // in the snapshot an always-match matcher passes this case, and + // `ChromiumJsRuntimeDebugger` is a namespace nothing else here scopes. + const services = new Map([ + ...order.map((urn) => [urn, urn === CDP ? cdpWithDependent() : live()] as const), + [OTHER_CDP, live()] as const, + ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -340,22 +351,28 @@ describe("stop-all-simulator-servers", () => { [CDP, CHROMIUM_DEBUGGER].sort() ); expect(result).not.toHaveProperty("unmatched"); - expect(services.get(CHROMIUM_DEBUGGER)?.state).toBe(ServiceState.IDLE); - expect(services.get(CDP)?.state).toBe(ServiceState.IDLE); + expect(registry.disposeService).not.toHaveBeenCalledWith(OTHER_CDP); } ); - it("takes a non-device dependent down with its dependency without claiming to have reaped it", async () => { - // The distinction the mock's recursion exists for, and the one the case - // above cannot make: a dependent this tool does NOT match by device. It - // still dies — the registry cascades — but it is somebody else's - // dependent, not something the teardown reaped by name, so it must not - // appear in `stopped`. Reporting it there would tell an agent a - // device-scoped teardown deliberately killed its Metro session. - const METRO = "Metro:8081"; + it("does not credit `stopped` with a dependent that was already IDLE", async () => { + // The distinction the case above cannot make. `ChromiumJsRuntimeDebugger` + // declares `ChromiumCdp` as its dependency, so the transport's teardown + // takes it down as a dependent — but it was already IDLE, so it was not a + // running service this call shut down and must not be named. `stopped` + // reports what this teardown found LIVE, not everything the graph touched; + // naming it would tell an agent a session it had already stopped was still + // up a moment ago. + // + // The earlier version of this case fabricated a `Metro:8081` node to stand + // in for a non-device dependent. There is no such thing: every namespace a + // blueprint declares as a dependency is itself in DEVICE_OWNED_NAMESPACES, + // and `Metro:8081` is not a registry namespace at all — so what it asserted + // (`services.get(METRO)?.state`) was the mock's own recursion, which no + // production line reads. const services = new Map([ - [CDP, { state: ServiceState.RUNNING, dependents: [METRO] }], - [METRO, { state: ServiceState.RUNNING, dependents: [] as string[] }], + [CDP, { state: ServiceState.RUNNING, dependents: [CHROMIUM_DEBUGGER] }], + [CHROMIUM_DEBUGGER, { state: ServiceState.IDLE, dependents: [] as string[] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -363,7 +380,25 @@ describe("stop-all-simulator-servers", () => { const result = await tool.execute!({}, { devices: ["chromium-cdp-9222"] }); expect(result).toEqual({ stopped: [CDP] }); - expect(services.get(METRO)?.state).toBe(ServiceState.IDLE); + // Matched, so not a mistyped id — the device owns both URNs either way. + expect(result).not.toHaveProperty("unmatched"); + }); + + it("disposes a TERMINATING node without reporting it as stopped", async () => { + // A node already being torn down is not live, so `isLiveServiceState` keeps + // it out of `stopped` — but it is not IDLE either, so the sweep still calls + // `disposeService` on it (which the real `_teardown` then no-ops). Both + // halves are production lines; neither had coverage. + const services = new Map([ + [`SimulatorServer:${MINE}`, { state: ServiceState.TERMINATING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + const result = await tool.execute!({}, { devices: [MINE] }); + + expect(result).toEqual({ stopped: [] }); + expect(registry.disposeService).toHaveBeenCalledWith(`SimulatorServer:${MINE}`); }); it("returns empty list when no simulators are running", async () => { @@ -834,8 +869,12 @@ describe("stop-all-simulator-servers unmatched ids", () => { // case). `AXService` is a device-owned namespace holding the in-sim ax // daemon (spawned --timeout 3600), so a scoped stop reaps it AND does not // report the correct UDID as unmatched: it owns a real service, not a typo. + // A second device's AXService is the control: without it an always-match + // matcher passes this case, and AXService is one of the namespaces nothing + // else here scopes. const services = new Map([ [`AXService:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`AXService:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -845,6 +884,7 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(result).toEqual({ stopped: [`AXService:${MINE}`] }); expect(result).not.toHaveProperty("unmatched"); expect(registry.disposeService).toHaveBeenCalledWith(`AXService:${MINE}`); + expect(registry.disposeService).not.toHaveBeenCalledWith(`AXService:${THEIRS}`); }); it("scopes the tcp-transport AXService URN to its own device", async () => { @@ -873,8 +913,11 @@ describe("stop-all-simulator-servers unmatched ids", () => { // cascades to it. It is a device-owned namespace, so a session that ran // screen-recording-start is correctly reaped by a scoped stop and its // serial is not reported as a mistyped id. + // Second device as the control — an always-match matcher would otherwise + // pass, and nothing else here scopes this namespace. const services = new Map([ [`ScreenRecordingSession:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + [`ScreenRecordingSession:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -883,6 +926,7 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(result).toEqual({ stopped: [`ScreenRecordingSession:${MINE}`] }); expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).not.toHaveBeenCalledWith(`ScreenRecordingSession:${THEIRS}`); }); it("owns and stops a device whose only service is a native profiler session", async () => { @@ -890,6 +934,8 @@ describe("stop-all-simulator-servers unmatched ids", () => { // its trace file on Android. const services = new Map([ [`NativeProfilerSession:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], + // Control, as above. + [`NativeProfilerSession:${THEIRS}`, { state: ServiceState.RUNNING, dependents: [] }], ]); const registry = createMockRegistry(services); const tool = createStopAllSimulatorServersTool(registry); @@ -898,6 +944,7 @@ describe("stop-all-simulator-servers unmatched ids", () => { expect(result).toEqual({ stopped: [`NativeProfilerSession:${MINE}`] }); expect(result).not.toHaveProperty("unmatched"); + expect(registry.disposeService).not.toHaveBeenCalledWith(`NativeProfilerSession:${THEIRS}`); }); it("scopes the port-keyed debugger URNs to the right device", async () => { From fbc61e7490fd9aedf484535d4ef67a2e4e440cbe Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:44:44 +0200 Subject: [PATCH 58/98] test: cover the changed lines the suite was leaving to inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - device-services' `afterPort < 0` arm: a port-keyed URN missing its device half owns no device, so the literal Metro port cannot claim it. - the keyResolutions FIFO sequencer, which nothing imported: realpath mocked to complete in decreasing time, so removing the sequencer inverts the lock queue deterministically rather than flakily. - the iOS breadcrumb clear (and the iOS half of the disposed-session guard), driven through the real startNativeProfilerIos with xctrace, simctl, the readiness handshake and the capture strategy stubbed at their boundaries — the only start-side test file was Android-only. - failedMsg for both stop tools. - the ENAMETOOLONG and missing-vault arms of writeFailureHint, and the ENOTDIR / unwritable-parent / ENAMETOOLONG arms of mkdirFailureHint. - the `flow_recording_key_aliased` failure stage, which shares its error code with a key that was never started and wants a different fix. --- .../flows/flow-concurrent-recording.test.ts | 3 + .../test/flows/flow-key-sequencer.test.ts | 107 ++++++++++++ .../tool-server/test/flows/flow-utils.test.ts | 103 ++++++++++++ .../test/native-profiler-ios-start.test.ts | 153 ++++++++++++++++++ packages/tool-server/test/stop-tools.test.ts | 43 +++++ 5 files changed, 409 insertions(+) create mode 100644 packages/tool-server/test/flows/flow-key-sequencer.test.ts create mode 100644 packages/tool-server/test/native-profiler-ios-start.test.ts diff --git a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts index 1990d554e..d9e88da71 100644 --- a/packages/tool-server/test/flows/flow-concurrent-recording.test.ts +++ b/packages/tool-server/test/flows/flow-concurrent-recording.test.ts @@ -428,6 +428,9 @@ describe("two recording keys that resolve to one file", () => { const err = await captureFailure(addEcho(realRoot, "checkout", "c2")); expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_NO_ACTIVE_RECORDING); + // Its own stage, so telemetry can tell an aliased key from a key that was + // never started — the two share an error code and want different fixes. + expect(getFailureSignal(err)?.failure_stage).toBe("flow_recording_key_aliased"); const message = formatErrorForAgent(err); expect(message).toContain("re-address it exactly as you passed it to flow-start-recording"); expect(message).toContain("the take is intact and still recording"); diff --git a/packages/tool-server/test/flows/flow-key-sequencer.test.ts b/packages/tool-server/test/flows/flow-key-sequencer.test.ts new file mode 100644 index 000000000..477571579 --- /dev/null +++ b/packages/tool-server/test/flows/flow-key-sequencer.test.ts @@ -0,0 +1,107 @@ +/** + * `keyResolutions` — the in-flight map every recording tool's key resolution + * passes through. + * + * Resolution is `realpath`, which runs on libuv's threadpool and completes in + * an order unrelated to the order it was requested in. Every recording tool + * resolves its key BEFORE joining its flow file's lock queue, so without the + * sequencer which of two calls acquires the lock first is decided by threadpool + * scheduling rather than by which was issued first — and a restart can land + * behind the append it is supposed to discard. + * + * Nothing imported it, so the property had no test at all. Here `realpath` is + * mocked to complete in decreasing time, which inverts the issue order + * deterministically: without the sequencer the second caller wins. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as os from "node:os"; +import * as path from "node:path"; +import * as realFs from "node:fs/promises"; + +/** + * How long each `realpath` call takes, by call index — strictly decreasing, so + * a later request always finishes before an earlier one. + */ +const DELAYS = [40, 30, 20, 10, 8, 6, 4, 2]; +let realpathCalls = 0; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: (p: string) => { + const delay = DELAYS[Math.min(realpathCalls++, DELAYS.length - 1)]!; + return new Promise((resolve, reject) => { + setTimeout(() => { + actual.realpath(p).then(resolve, reject); + }, delay); + }); + }, + }; +}); + +import { withFlowFileLock, __resetRecordingsForTesting } from "../../src/tools/flows/flow-utils"; + +let root: string; + +beforeEach(async () => { + __resetRecordingsForTesting(); + realpathCalls = 0; + root = await realFs.mkdtemp(path.join(os.tmpdir(), "flow-key-seq-")); + await realFs.mkdir(path.join(root, ".argent", "flows"), { recursive: true }); + await realFs.writeFile(path.join(root, ".argent", "flows", "alpha.yaml"), "steps: []\n", "utf8"); +}); + +afterEach(async () => { + await realFs.rm(root, { recursive: true, force: true }); +}); + +describe("the flow-key resolution sequencer", () => { + it("keeps the lock queue in the order the calls were issued", async () => { + const order: string[] = []; + const enter = (label: string) => + withFlowFileLock(root, "alpha", async () => { + order.push(label); + }); + + // Issued first, resolves SLOWEST if it resolves on its own. + const first = enter("first"); + const second = enter("second"); + const third = enter("third"); + await Promise.all([first, second, third]); + + expect(order).toEqual(["first", "second", "third"]); + }); + + it("shares one resolution between callers spelling the path the same way", async () => { + const before = realpathCalls; + await Promise.all([ + withFlowFileLock(root, "alpha", async () => {}), + withFlowFileLock(root, "alpha", async () => {}), + withFlowFileLock(root, "alpha", async () => {}), + ]); + // One resolution — a dir + a file realpath — not three. + expect(realpathCalls - before).toBe(2); + }); + + it("drops the entry once it settles, so a repointed link is seen next time", async () => { + // Not a cache: a second round must resolve again rather than reuse the + // first round's answer. + await withFlowFileLock(root, "alpha", async () => {}); + const after = realpathCalls; + await withFlowFileLock(root, "alpha", async () => {}); + expect(realpathCalls).toBeGreaterThan(after); + }); + + it("resolves two DIFFERENT flows separately rather than sharing one answer", async () => { + // Keyed by the SPELLED path, so two flows never collapse onto one + // resolution — which would hand one file's key to the other's lock. + await realFs.writeFile(path.join(root, ".argent", "flows", "beta.yaml"), "steps: []\n", "utf8"); + const before = realpathCalls; + await Promise.all([ + withFlowFileLock(root, "alpha", async () => {}), + withFlowFileLock(root, "beta", async () => {}), + ]); + expect(realpathCalls - before).toBe(4); + }); +}); diff --git a/packages/tool-server/test/flows/flow-utils.test.ts b/packages/tool-server/test/flows/flow-utils.test.ts index c2328b680..93bfc2b98 100644 --- a/packages/tool-server/test/flows/flow-utils.test.ts +++ b/packages/tool-server/test/flows/flow-utils.test.ts @@ -1863,6 +1863,43 @@ describe("writeFlowFile failure hints", () => { expect(message).not.toMatch(/is a symlink/); }); + it("blames the name length, not the directory, on ENAMETOOLONG", async () => { + // The arm the hint was split for: an over-long flow name comes out of + // `rename` (the scratch name is short), and reporting it as a + // directory-permissions problem sent the reader looking for one that is not + // there. + const flowsDir = path.join(root, ".argent", "flows"); + await fs.mkdir(flowsDir, { recursive: true }); + + const err = await writeNewFlowFile( + path.join(flowsDir, `${"n".repeat(400)}.yaml`), + "steps: []\n" + ).catch((e: unknown) => e); + + const message = (err as Error).message; + expect(message).toContain("(ENAMETOOLONG)"); + expect(message).toContain("use a shorter name"); + expect(message).not.toContain("must be writable"); + }); + + it("names the missing VAULT directory when the link points into one", async () => { + // ENOENT out of the scratch write, in the directory the swap actually uses. + // Naming `.argent/flows` here would point at a directory that exists. + const flowsDir = path.join(root, ".argent", "flows"); + await fs.mkdir(flowsDir, { recursive: true }); + const absentVault = path.join(root, "no-such-vault"); + await fs.symlink(path.join(absentVault, "shared.yaml"), path.join(flowsDir, "shared.yaml")); + + const err = await writeNewFlowFile(path.join(flowsDir, "shared.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + const message = (err as Error).message; + expect(message).toContain("(ENOENT)"); + expect(message).toContain(`${absentVault} does not exist`); + expect(message).toContain("shared.yaml is a symlink"); + }); + it("still points at the vault when the flow file really is a symlink", async () => { // The case the clause exists for: naming `.argent/flows` here would send the // reader to a directory that is already writable while the vault, the only @@ -1954,3 +1991,69 @@ describe("flow file permissions across an atomic append", () => { expect(await fs.readFile(file, "utf8")).toBe("steps: []\n"); }); }); + +describe("mkdirFailureHint arms", () => { + // The flows-directory half of writeNewFlowFile's classification. Only its + // wrapping was covered; each errno arm names a different cause, and the + // ENOTDIR one — a `project_root` that names a FILE — is the mistake the hint + // exists for. + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "flow-mkdir-hint-")); + }); + + afterEach(async () => { + await fs.chmod(root, 0o755).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); + }); + + it("blames a project_root that names a file, not a directory", async () => { + const asFile = path.join(root, "notadir"); + await fs.writeFile(asFile, "", "utf8"); + const flows = path.join(asFile, ".argent", "flows"); + + const err = await writeNewFlowFile(path.join(flows, "x.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.FLOW_FILE_WRITE_FAILED); + expect(getFailureSignal(err)?.failure_stage).toBe("flow_dir_create"); + expect((err as Error).message).toContain("(ENOTDIR)"); + expect((err as Error).message).toContain( + "check that project_root names a directory rather than a file" + ); + }); + + it("blames the nearest existing parent when it is not writable", async () => { + await fs.chmod(root, 0o555); + if ( + await fs.access(root, fsConstants.W_OK).then( + () => true, + () => false + ) + ) + return; + const flows = path.join(root, ".argent", "flows"); + + const err = await writeNewFlowFile(path.join(flows, "x.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + expect((err as Error).message).toMatch(/\((EACCES|EPERM)\)/); + expect((err as Error).message).toContain("nearest existing parent"); + }); + + it("blames the name length when the path is too long for the filesystem", async () => { + // ENAMETOOLONG out of mkdir -p, which must not read as a permissions + // problem the user would then go and not find. + const tooLong = path.join(root, "d".repeat(512), ".argent", "flows"); + + const err = await writeNewFlowFile(path.join(tooLong, "x.yaml"), "steps: []\n").catch( + (e: unknown) => e + ); + + expect((err as Error).message).toContain("(ENAMETOOLONG)"); + expect((err as Error).message).toContain("longer than this filesystem allows"); + }); +}); diff --git a/packages/tool-server/test/native-profiler-ios-start.test.ts b/packages/tool-server/test/native-profiler-ios-start.test.ts new file mode 100644 index 000000000..06518f034 --- /dev/null +++ b/packages/tool-server/test/native-profiler-ios-start.test.ts @@ -0,0 +1,153 @@ +/** + * The iOS half of `native-profiler-start`, driven at its module boundaries + * (xctrace spawn, the readiness handshake, the capture strategy, simctl). + * + * Two of its lines had no coverage at all, because the only start-side test + * file is Android-only: + * + * - the teardown-breadcrumb clear. A breadcrumb explains ONE confusing + * answer — the "no active session" a reaped capture's own stop would get — + * so a start that succeeds afterwards makes it unconsumable, and it would + * sit in the process-global map until some genuinely unrelated later + * absence collected it and blamed a teardown that had nothing to do with + * it. The Android twin clears it and is tested; iOS was not. + * - the disposed-session guard, which turns a start whose session a teardown + * destroyed mid-handshake into a failure instead of a `status: "recording"` + * nothing can stop. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { EventEmitter } from "events"; +import type { ChildProcess } from "child_process"; +import * as os from "node:os"; +import { FAILURE_CODES, getFailureSignal, type DeviceInfo } from "@argent/registry"; + +class FakeXctrace extends EventEmitter { + pid = 4242; + stdout = new EventEmitter(); + stderr = new EventEmitter(); + kill = vi.fn(() => true); +} + +vi.mock("child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: vi.fn(() => new FakeXctrace()), + // Every simctl helper on this path goes through execFileSync. Failing it puts + // `resolveExplicitApp` on its documented "app is not running yet" fallback, + // which attaches by name — the cold-start-retry shape. + execFileSync: vi.fn(() => { + throw new Error("simctl unavailable in this test"); + }), +})); +vi.mock("../src/utils/ios-device-sets", () => ({ + deviceSetForUdid: vi.fn(async () => undefined), + simctlArgsForUdidSync: vi.fn((_udid: string, args: string[]) => args), +})); +vi.mock("../src/utils/react-profiler/debug/dump", () => ({ + getDebugDir: vi.fn(async () => os.tmpdir()), +})); +vi.mock("../src/utils/ios-profiler/notify", () => ({ + // Null handle: the start falls back to the stdout substring match, and + // `waitForXctraceReady` below is what decides readiness either way. + listenForDarwinNotification: vi.fn(() => { + throw new Error("notifyutil unavailable in this test"); + }), +})); +vi.mock("../src/utils/ios-profiler/startup", () => ({ + waitForXctraceReady: vi.fn(async () => ({ stderrBuffer: "" })), +})); +vi.mock("../src/utils/ios-profiler/capture-strategy", () => ({ + selectIosCaptureStrategy: vi.fn(() => ({ + name: "device", + attachesByName: true, + cpuFilterPid: () => null, + buildRecordArgs: () => ["record", "--device", "UDID"], + })), + resolveIosCaptureStrategy: vi.fn(() => ({ name: "device" })), + warnIfInvalidCaptureOverride: vi.fn(), +})); + +import { + nativeProfilerSessionBlueprint, + type NativeProfilerSessionApi, +} from "../src/blueprints/native-profiler-session"; +import { startNativeProfilerIos } from "../src/tools/profiler/native-profiler/platforms/ios"; +import { + recordReapedSession, + takeReapedSession, + __resetReapedSessionsForTesting, +} from "../src/utils/reaped-sessions"; + +const iosDevice = { id: "6DBF83B4-0000-0000-0000-000000000000", platform: "ios" } as DeviceInfo; + +async function session() { + return nativeProfilerSessionBlueprint.factory({}, iosDevice, { device: iosDevice } as never); +} + +const startParams = { + device_id: iosDevice.id, + app_process: "Bluesky", + template_path: "/tmp/Argent.tracetemplate", +}; + +beforeEach(() => { + __resetReapedSessionsForTesting(); +}); + +describe("startNativeProfilerIos", () => { + it("clears the teardown breadcrumb its own success would make unconsumable", async () => { + const instance = await session(); + const api = instance.api as NativeProfilerSessionApi; + recordReapedSession("native-profiler", api.deviceId, "an earlier trace"); + + const result = await startNativeProfilerIos(api, startParams); + + expect(result.status).toBe("recording"); + expect(takeReapedSession("native-profiler", api.deviceId)).toBeUndefined(); + if (api.recordingTimeout) clearTimeout(api.recordingTimeout); + }); + + it("leaves another device's breadcrumb alone", async () => { + const instance = await session(); + const api = instance.api as NativeProfilerSessionApi; + recordReapedSession("native-profiler", "emulator-5554", "somebody else's trace"); + + await startNativeProfilerIos(api, startParams); + + expect(takeReapedSession("native-profiler", "emulator-5554")).toBeDefined(); + if (api.recordingTimeout) clearTimeout(api.recordingTimeout); + }); + + it("fails, rather than reporting a recording, when a teardown lands mid-handshake", async () => { + const instance = await session(); + const api = instance.api as NativeProfilerSessionApi; + const startup = await import("../src/utils/ios-profiler/startup"); + vi.mocked(startup.waitForXctraceReady).mockImplementationOnce(async () => { + await instance.dispose(); + return { stderrBuffer: "" }; + }); + + const err = await startNativeProfilerIos(api, startParams).catch((e: unknown) => e); + + expect(getFailureSignal(err)?.error_code).toBe(FAILURE_CODES.NATIVE_PROFILER_SESSION_TORN_DOWN); + expect(api.profilingActive).toBe(false); + expect(api.captureProcess).toBeNull(); + expect(api.recordingTimeout).toBeNull(); + }); + + it("kills the xctrace it spawned rather than leaving it recording", async () => { + const instance = await session(); + const api = instance.api as NativeProfilerSessionApi; + const child = new FakeXctrace(); + const cp = await import("child_process"); + vi.mocked(cp.spawn).mockReturnValueOnce(child as unknown as ChildProcess); + const startup = await import("../src/utils/ios-profiler/startup"); + vi.mocked(startup.waitForXctraceReady).mockImplementationOnce(async () => { + await instance.dispose(); + return { stderrBuffer: "" }; + }); + + await startNativeProfilerIos(api, startParams).catch(() => {}); + + expect(child.kill).toHaveBeenCalled(); + }); +}); diff --git a/packages/tool-server/test/stop-tools.test.ts b/packages/tool-server/test/stop-tools.test.ts index 00bcea2a4..9e6a9c3bf 100644 --- a/packages/tool-server/test/stop-tools.test.ts +++ b/packages/tool-server/test/stop-tools.test.ts @@ -167,6 +167,18 @@ describe("stop-simulator-server", () => { expect(registry.disposeService).toHaveBeenCalledWith("ChromiumCdp:chromium-cdp-9222"); }); + it("names the device and the error code in failedMsg", () => { + // The one formatter with no coverage — flattening it to a constant left the + // suite green, and it is the line an agent reads when a teardown fails. + const tool = createStopSimulatorServerTool(createMockRegistry(new Map())); + expect( + tool.interaction!.failedMsg!({ + params: { udid: "AAAA-BBBB" }, + failureSignal: { error_code: "REGISTRY_TOOL_EXECUTION_FAILED" }, + } as never) + ).toBe("Failed to stop simulator server for AAAA-BBBB: REGISTRY_TOOL_EXECUTION_FAILED"); + }); + // Both stop tools resolve "which services does this device own" through the // one shared matcher in device-services.ts, so a given udid — whatever its // case — reaches the same services through either. Case-insensitivity is the @@ -764,6 +776,25 @@ describe("stop-all-simulator-servers unmatched ids", () => { // meant to reap (on tvOS, two spawned --timeout 3600 daemons) stayed running. // `unmatched` names them, so scoping cannot fail silently. + it("owns no device from a port-keyed URN missing its device half", async () => { + // `:` with nothing after the port is malformed — the device + // portion is what follows the FIRST colon, and there is none. Reading the + // tail as the device id instead would let the literal Metro port `8081` + // claim it, so a `devices: ["8081"]` typo would silently reap a debugger + // session and report a clean scope. + const services = new Map([ + ["JsRuntimeDebugger:8081", { state: ServiceState.RUNNING, dependents: [] }], + ]); + const registry = createMockRegistry(services); + const tool = createStopAllSimulatorServersTool(registry); + + expect(await tool.execute!({}, { devices: ["8081"] })).toEqual({ + stopped: [], + unmatched: ["8081"], + }); + expect(registry.disposeService).not.toHaveBeenCalled(); + }); + it("names an unknown id in unmatched while still stopping the live device", async () => { const services = new Map([ [`SimulatorServer:${MINE}`, { state: ServiceState.RUNNING, dependents: [] }], @@ -1462,6 +1493,18 @@ describe("stop-all-simulator-servers interaction messages", () => { ).toBe("Stopped 0 simulator servers (2 debugger sessions left running)"); }); + it("failedMsg names the error code", () => { + // The one formatter of the three with no coverage — flattening it to a + // constant left the suite green. + const failedMsg = tool().interaction!.failedMsg!; + expect( + failedMsg({ + params: {}, + failureSignal: { error_code: "REGISTRY_TOOL_EXECUTION_FAILED" }, + } as never) + ).toBe("Failed to stop simulator servers: REGISTRY_TOOL_EXECUTION_FAILED"); + }); + it("completedMsg reports both clauses when a call hits both", () => { const completedMsg = tool().interaction!.completedMsg!; expect( From 1344af0a5405f63b0298c734b60ec5e7bbdee9ad Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:49:21 +0200 Subject: [PATCH 59/98] test(flow): guard the flow-add-step schema the CLI tests hand-copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three CLI test files encode that schema as a fixture, because @argent/cli does not depend on the tool-server and cannot derive it. Drift was therefore silent in the direction that matters: relaxing the real schema — making project_root optional, renaming `args` — left all three green while the CLI's --args handling and help output were decided by a schema nothing resembled any more. Put the guard where the schema is, asserting the exact property and required sets those fixtures encode plus the description sentence they quote, and point each fixture at it. --- packages/argent-cli/test/flag-parser.test.ts | 5 +++ .../test/run-flow-add-step-payload.test.ts | 6 ++++ packages/argent-cli/test/run-help.test.ts | 5 +++ .../tool-server/test/flows/flow-tools.test.ts | 36 ++++++++++++++++++- 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/packages/argent-cli/test/flag-parser.test.ts b/packages/argent-cli/test/flag-parser.test.ts index fcb1d9ed7..999cf1e83 100644 --- a/packages/argent-cli/test/flag-parser.test.ts +++ b/packages/argent-cli/test/flag-parser.test.ts @@ -118,6 +118,11 @@ describe("flag-parser array + -json interleave never throws a raw error", () => // registry advertises for the real tool (zodObjectToJsonSchema over // packages/tool-server/src/tools/flows/flow-add-step.ts): recordings are keyed by // `name` + `project_root`, so both are required alongside `command`. +// +// This fixture is hand-copied: `@argent/cli` does not depend on the tool-server, +// so it cannot derive the schema. The guard that catches drift lives where the +// schema does — `flow-tools.test.ts`'s "the flow-add-step schema the CLI tests +// hand-copy". If that fails, this fixture is what it is telling you to update. const flowAddStepSchema: JsonSchema = { type: "object", properties: { diff --git a/packages/argent-cli/test/run-flow-add-step-payload.test.ts b/packages/argent-cli/test/run-flow-add-step-payload.test.ts index dc965b26d..485cccf22 100644 --- a/packages/argent-cli/test/run-flow-add-step-payload.test.ts +++ b/packages/argent-cli/test/run-flow-add-step-payload.test.ts @@ -50,6 +50,12 @@ function startServer(cap: Captured): Promise<{ url: string; close: () => Promise // covered by run-help.test.ts). The array is kept faithful so the // fixture stays readable as the real schema, not because dropping // an entry would fail here. + // + // Hand-copied because `@argent/cli` does not depend on the + // tool-server. The guard that catches drift lives where the schema + // does — flow-tools.test.ts's "the flow-add-step schema the CLI + // tests hand-copy"; if that fails, this is one of the fixtures it + // is telling you to update. inputSchema: { type: "object", properties: { diff --git a/packages/argent-cli/test/run-help.test.ts b/packages/argent-cli/test/run-help.test.ts index ab26911d8..55773fe35 100644 --- a/packages/argent-cli/test/run-help.test.ts +++ b/packages/argent-cli/test/run-help.test.ts @@ -35,6 +35,11 @@ vi.mock("@argent/telemetry", () => telemetryMock); // The drift that does pass silently is the opposite one: if the real // flow-add-step schema ever relaxes, nothing here notices this fixture went // stale. +// +// This fixture is hand-copied: `@argent/cli` does not depend on the tool-server, +// so it cannot derive the schema. The guard that catches drift lives where the +// schema does — `flow-tools.test.ts`'s "the flow-add-step schema the CLI tests +// hand-copy". If that fails, this fixture is what it is telling you to update. const flowAddStepMeta = { name: "flow-add-step", // Leading sentence of the real tool description, verbatim. diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index c7b343ef7..afdc03d46 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -3,7 +3,7 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import type { Registry, ToolContext } from "@argent/registry"; -import { ArtifactStore } from "@argent/registry"; +import { ArtifactStore, zodObjectToJsonSchema } from "@argent/registry"; import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording"; import { flowInsertEchoTool } from "../../src/tools/flows/flow-insert-echo"; @@ -2466,3 +2466,37 @@ describe("flow-read-prerequisite", () => { ).rejects.toThrow("exactly one flow source"); }); }); + +describe("the flow-add-step schema the CLI tests hand-copy", () => { + // Three CLI test files encode this schema as a fixture — `run-help.test.ts`, + // `flag-parser.test.ts` and `run-flow-add-step-payload.test.ts` — because + // `@argent/cli` does not depend on the tool-server and so cannot derive it. + // That makes drift silent in the direction that matters: relaxing the real + // schema here (making `project_root` optional, renaming `args`) leaves all + // three green while the CLI's `--args` handling and help output are decided + // by a schema nothing resembles any more. + // + // So the guard lives on this side, where the schema is. If this fails, + // update those three fixtures in the same change. + const CLI_FIXTURE_PROPERTIES = ["name", "project_root", "command", "args", "delayMs"]; + const CLI_FIXTURE_REQUIRED = ["name", "project_root", "command"]; + + it("still declares exactly the properties and required keys those fixtures encode", () => { + const schema = zodObjectToJsonSchema( + createFlowAddStepTool({} as unknown as Registry).zodSchema! + ) as { properties: Record; required?: string[] }; + + expect(Object.keys(schema.properties).sort()).toEqual([...CLI_FIXTURE_PROPERTIES].sort()); + expect([...(schema.required ?? [])].sort()).toEqual([...CLI_FIXTURE_REQUIRED].sort()); + // `parseFlags` branches on this one specifically: a tool that declares its + // own `args` must not also advertise the whole-payload `--args ` + // escape hatch. + expect(schema.properties["args"]).toMatchObject({ type: "string" }); + }); + + it("still opens its description with the sentence those fixtures quote verbatim", () => { + expect(createFlowAddStepTool({} as unknown as Registry).description).toContain( + "Execute a tool call and record it as a step in the flow named by `name` + `project_root`" + ); + }); +}); From 4c30ae25db327c23b2be5f6b5dae0a9730c11004 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:51:41 +0200 Subject: [PATCH 60/98] docs: correct what the deviceless test and the e2e cleanup phase claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - flow-deviceless.test.ts said the `devices` key is "stripped at record time and re-injected at replay". It is not stripped — stripDeviceKeys touches only the target keys, and flow-tools.test.ts asserts the opposite. The empty args in that fixture are a recording of the UNSCOPED sweep. - 90-cleanup.sh said it stops "any simulator-servers this run started". The unscoped `{}` is now the machine-wide sweep across every device-owned namespace. Say so, and say why unscoped is right in that one place (the run's HOME is the sandbox, so the server it discovers is its own). - 20-validation.sh skipped stop-all-simulator-servers entirely, so neither the `.strict()` rejection of the `udids` slip nor the `unmatched` report had any E2E coverage. Both now have a targeted case; the bogus-id scope reaps nothing, so it is safe to run there. --- .../test/flows/flow-deviceless.test.ts | 8 +++++-- scripts/e2e-full/phases/20-validation.sh | 24 ++++++++++++++++++- scripts/e2e-full/phases/90-cleanup.sh | 12 +++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/packages/tool-server/test/flows/flow-deviceless.test.ts b/packages/tool-server/test/flows/flow-deviceless.test.ts index c97573188..5f1bc5716 100644 --- a/packages/tool-server/test/flows/flow-deviceless.test.ts +++ b/packages/tool-server/test/flows/flow-deviceless.test.ts @@ -327,8 +327,12 @@ describe("stepRequiresDevice", () => { describe("a cleanup flow whose only step is stop-all-simulator-servers", () => { const teardownOnly: FlowStep[] = [ - // What the recorder writes for a `stop-all-simulator-servers`: the - // `devices` key is stripped at record time and re-injected at replay. + // What the recorder writes for an UNSCOPED `stop-all-simulator-servers`. + // A scoped one keeps its `devices` in the YAML — `stripDeviceKeys` touches + // only the target keys, and `flow-tools.test.ts`'s "keeps the devices list + // when recording a scoped teardown" pins that — so the empty args here are + // the recording of the machine-wide sweep, which replay then NARROWS onto + // the run device. { kind: "tool", name: "stop-all-simulator-servers", args: {} }, ]; diff --git a/scripts/e2e-full/phases/20-validation.sh b/scripts/e2e-full/phases/20-validation.sh index d54647a93..68e373d64 100644 --- a/scripts/e2e-full/phases/20-validation.sh +++ b/scripts/e2e-full/phases/20-validation.sh @@ -13,7 +13,9 @@ # Tools with no required flags that would actually EXECUTE (touch a device / # network / state) if called empty — excluded from missing-required here and -# covered by the device tiers instead. +# covered by the device tiers instead. `stop-all-simulator-servers` gets its own +# targeted cases below, since skipping it outright left its `.strict()` +# rejection and its `unmatched` report with no coverage anywhere. _VAL_EXCLUDE_MISSING="list-devices stop-all-simulator-servers stop-metro native-devtools-status update-argent" # Build a JSON object with valid dummies for every required flag in a model, @@ -86,6 +88,26 @@ run_phase() { assert_reject "$P" "$t" "bad-enum:$ef" "$args" "$ef" "invalid_value" done + # --- stop-all-simulator-servers' strict schema and unmatched report ----- + # It is on _VAL_EXCLUDE_MISSING (calling it empty would sweep the machine), + # so the generated matrix skips it entirely — leaving the two properties + # that make its `devices` scope safe with no E2E coverage at all. + if [ "$t" = "stop-all-simulator-servers" ]; then + # `.strict()`: `udids` is the natural slip (every sibling tool spells the + # device parameter `udid`), and under a stripping schema that typo would + # be a silent machine-wide sweep. + assert_reject "$P" "$t" strict-unknown-key '{"udids":["nope"]}' "udids" "unrecognized_keys" + # `unmatched`: an id owning nothing must not read as a clean machine. A + # scope of one bogus id reaps nothing and touches no device, so this is + # safe to run here. + run_tool "$t" '{"devices":["__e2e_no_such_device__"]}' + if [ "$RT_RC" -eq 0 ] && [ "$(printf '%s' "$RT_JSON" | jq -r '.unmatched[0] // ""')" = "__e2e_no_such_device__" ]; then + pass "$P" "$t" unmatched "bogus id reported, not silently clean" + else + fail "$P" "$t" unmatched "expected unmatched:[__e2e_no_such_device__], got rc=$RT_RC $RT_JSON" + fi + fi + # --- bad-type (first required number flag gets a string) --------------- local nf nf="$(model_number_flags "$model" | while read -r f; do diff --git a/scripts/e2e-full/phases/90-cleanup.sh b/scripts/e2e-full/phases/90-cleanup.sh index 0588fcff8..7844a3706 100644 --- a/scripts/e2e-full/phases/90-cleanup.sh +++ b/scripts/e2e-full/phases/90-cleanup.sh @@ -5,7 +5,17 @@ run_phase() { local P=cleanup - # Stop any simulator-servers this run started (Android/iOS backends). + # Drain the run's own tool-server. The unscoped `{}` is the machine-wide sweep + # across every device-owned namespace — simulator-servers, native devtools, AX, + # TV-control daemons, Chromium CDP, screen recordings, native-profiler and + # JS-runtime debugger sessions — not just "the simulator-servers this run + # started", which is what this said while the tool only reached the transports. + # + # Unscoped is nonetheless right HERE, and only here: the run's HOME is the + # sandbox, so the server it discovers is this run's own (see ensure_server's + # note) and the sweep cannot reach another agent's devices. Anywhere an agent + # is talking to the shared install, pass `devices` — that is what the tool's + # own description and the skills tell agents to do. if [ -n "${ARGENT_TOOLS_URL:-}" ]; then run_tool stop-all-simulator-servers '{}' >/dev/null 2>&1 && pass "$P" stop-all-simulator-servers teardown || skip "$P" stop-all-simulator-servers teardown "no server/none running" run_tool stop-metro '{}' >/dev/null 2>&1 && pass "$P" stop-metro teardown || skip "$P" stop-metro teardown "no metro" From c45f98212c27e89cc3be81fa92eee3d2198a433a Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 23:17:25 +0200 Subject: [PATCH 61/98] feat(flow): add an idle condition that waits for the screen to stop moving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `await: { idle: true }` is the one check a selector condition cannot express: has the screen stopped moving? Flows were substituting fixed `wait:` steps for it, which either wait too long on every run or too little on the run that mattered. It waits on BOTH the UI tree and the rendered pixels, because each is blind to what the other sees. The tree cannot see presentation-layer motion: an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — so a tree-only settle reports a screen that is still sliding and the next tap lands on a moving target. Measured on a real Android emulator: a page animating a large element continuously reads as a single static `WebView` node, and the existing `await-screen-idle` tool called it settled in 465ms, five times out of five. Pixels alone would not do either — they cannot see a tree still churning behind an unchanged surface, and anything animated forever (a video, a shimmer) would make a pixel-only settle unsatisfiable on a screen the tree calls ready. Unlike that tool, this FAILS on timeout, which is what makes it safe to persist in a flow: a soft `settled: false` cannot carry a regression verdict on an unattended replay. Everything about the design follows from one rule — the absence of evidence is never evidence. A capture that did not arrive cannot complete the hold. A single agreeing pair of captures cannot either: every animation that reverses has a turning point, and two samples straddling it come back identical while the screen is moving, which on a live 3s cross-fade passed a default-shaped step on roughly one run in three. Settling takes two consecutive still intervals, so `minStableMs: 0` still means three reads. A round is never started without the budget to observe it with, since a round begun with nothing left neither captures nor reads, and both absences would otherwise be recorded as facts about the device. And no verdict comes from a latch: a screen that settles and then moves again, or goes blank, has not settled. `timeout:` is a real bound. No describe path takes an abort signal, so the tree read is raced against what is left of the budget — without that, a wedged ViewInspector RPC ran 2.25s past an 8000ms budget. A read that runs out of step budget is the step ending, not the source failing, and only the latter is reported as an environment problem. The failure modes stay apart because they call for opposite responses: a screen that never stopped moving is a verdict about the app; a tree that could not be read is not, and it names the foreground check first, because a backgrounded app reads identically to an uninstrumented one and relaunching is the wrong repair. A run whose captures never produced a comparable pair still settles on the tree and says so in a warning, rather than passing off half a proof as the whole one. The pixel comparison owns its tolerance rather than borrowing screenshot-diff's. That one holds a baseline stored across sessions, machines and OS versions against a live capture and must absorb real drift; this one holds two captures one poll apart from a single session, where a static screen reads back byte-identical (measured: zero changed pixels across five consecutive pairs on an idle simulator). The margin is load-bearing — uniform change is all-or-nothing, so at the baseline tolerance any cross-fade slower than ~2s counted zero changed pixels and read as settled mid-animation. Captures route exactly as the `screenshot` tool routes them, so tvOS and Vega — which have no simulator-server backend but are perfectly screenshottable through `xcrun` and the emulator console — are covered rather than written off. It is deliberately NOT a screen check: a dropped tap leaves the source screen perfectly idle. It belongs after the element check that names the destination, never instead of one. --- .../skills/skills/argent-create-flow/SKILL.md | 6 +- .../src/tools/flows/flow-actions.ts | 308 ++++++++++++- .../src/tools/flows/flow-device.ts | 5 +- .../src/tools/flows/flow-finish-recording.ts | 2 + .../src/tools/flows/flow-pixels.ts | 188 ++++++++ .../tool-server/src/tools/flows/flow-run.ts | 37 +- .../tool-server/src/tools/flows/flow-utils.ts | 186 +++++++- .../tools/screenshot-diff/screenshot-diff.ts | 4 + .../tool-server/src/tools/screenshot/index.ts | 5 +- .../tool-server/src/utils/simulator-client.ts | 2 +- .../test/flows/flow-deviceless.test.ts | 2 + .../test/flows/flow-idle-condition.test.ts | 138 ++++++ .../test/flows/flow-idle-run.test.ts | 412 ++++++++++++++++++ .../test/flows/flow-pixels.test.ts | 327 ++++++++++++++ 14 files changed, 1593 insertions(+), 29 deletions(-) create mode 100644 packages/tool-server/src/tools/flows/flow-pixels.ts create mode 100644 packages/tool-server/test/flows/flow-idle-condition.test.ts create mode 100644 packages/tool-server/test/flows/flow-idle-run.test.ts create mode 100644 packages/tool-server/test/flows/flow-pixels.test.ts diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 00876ee6c..65c53f69a 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -31,7 +31,7 @@ Beyond raw `tool:` steps and `echo:`, flows support declarative directives inter | `scroll-to` | `- scroll-to: "Order #1234"` (scrolls down) or `- scroll-to: { target: …, direction: right, within: … }` | momentum-free scroll until the target is visible | | `pinch` | `- pinch: { on: "Map", scale: 3 }` or `- pinch: { scale: 0.5 }` | two-finger zoom in (`scale` > 1) or out (`< 1`); big scales chain gestures; `on` optional — defaults to screen center; open-loop — assert the visible result | | `rotate` | `- rotate: { on: "Map", by: 90 }` or `- rotate: { by: -45 }` | two-finger rotation by degrees (+ CW, − CCW, within ±3000°; options map only); `on` optional — screen center default; not `tool: rotate` (orientation) | -| `await` | `- await: { visible: Home }` | wait for a UI condition | +| `await` | `- await: { visible: Home }` or `- await: { idle: true }` | wait for a UI condition, or for the screen to stop moving | | `wait` | `- wait: 500` | pause for a fixed number of milliseconds (last resort — prefer `await`) | | `assert` | `- assert: { visible: Welcome }` | check a condition, hard-fail if it never holds | | `snapshot` | `- snapshot: home` or `- snapshot: { name: home, maxMismatch: 0.5, cropOn: { id: order-summary } }` | diff a screenshot — or one element's region — against a stored baseline | @@ -85,6 +85,10 @@ This condition-as-key form is the only spelling. `await` also accepts an optiona For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. +**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels, then hard-fails if it never does (that hard failure is what makes it safe to persist, unlike the soft `await-screen-idle` tool). Options: `minStableMs` (how long stillness must hold, default 250 — it has to be shorter than the timeout, or the gate could never pass) and `timeout` (default 7500). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. + +It is **not** a screen check: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. Do not sprinkle it after every step — each one costs a settle; add one where a step actually proved flaky. If its captures never produced a comparable pair, it still passes on the tree alone but reports a warning saying so, because it then only proved the hierarchy held still. + ### `type` and `scroll-to` `type` presses Enter after typing to commit the value and dismiss the keyboard, so it can't cover later targets. For a chained form whose fields feed one explicit submit — e.g. email then password then a `tap: "Log in"` — set `submit: false` on the intermediate fields so a premature Enter doesn't fire the form early: `type: { into: password, text: "hunter2", submit: false }`. diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index f45054b17..1436276c7 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -19,10 +19,11 @@ import { type WaitCondition, type TextMatchMode, } from "../../utils/ui-tree-match"; -import { sleepOrAbort } from "../../utils/timing"; +import { settleWithin, sleepOrAbort } from "../../utils/timing"; import { invokeSubTool } from "../../utils/sub-invoke"; import { bindDeviceArgs } from "./flow-device"; import { fetchFlowTree } from "./flow-tree"; +import { capturePixelsWithin, pixelsDiffer, type PixelFrame } from "./flow-pixels"; import { buildAxisCandidate, decomposePinch, @@ -40,6 +41,8 @@ import { import { describeSelector, describeTextExpectation, + IDLE_DEFAULT_MIN_STABLE_MS, + IDLE_DEFAULT_TIMEOUT_MS, SELECTOR_RELATIONS, type FlowSelector, type FlowStep, @@ -66,9 +69,16 @@ export interface DirectiveOutcome { * blind/degraded tree), or a `hidden` check ended on a blind or failed * read after the element had matched. Read by the `when:` guard probe, * which must error rather than silently skip a block a broken tree source - * can't vouch for; a plain `assert` reports it as an ordinary failure. + * can't vouch for; a plain `assert` reports it as an ordinary failure. An + * `idle` step is the exception — it has no condition to fall back on, so the + * runner scores its indeterminate outcome `error` rather than `fail`. */ indeterminate?: boolean; + /** + * The step passed, but the WAY it passed weakens it as proof — carried into + * the step report so the author is told what the green actually bought. + */ + warning?: string; } /** @@ -84,10 +94,21 @@ export const ABORTED_OUTCOME: DirectiveOutcome = { reason: "run aborted", }; -/** The selector-acting steps {@link runDirective} handles. */ +/** The condition/action steps {@link runDirective} handles. */ export type DirectiveStep = Extract< FlowStep, - { kind: "tap" | "long-press" | "type" | "await" | "assert" | "scroll-to" | "pinch" | "rotate" } + { + kind: + | "tap" + | "long-press" + | "type" + | "await" + | "assert" + | "idle" + | "scroll-to" + | "pinch" + | "rotate"; + } >; /** Dispatch a tool with the run's resolved device id bound into its args. */ @@ -621,7 +642,11 @@ export function offscreenHint(sel: FlowSelector): string { return `no visible element matched selector ${describeSelector(sel)} — if it is off-screen, add a scroll-to step before this one`; } -/** Execute one selector-acting directive (`tap` / `long-press` / `type` / `await` / `assert` / `scroll-to` / `pinch` / `rotate`). */ +/** + * Execute one directive step: the selector-acting ones (`tap` / `long-press` / + * `type` / `await` / `assert` / `scroll-to` / `pinch` / `rotate`) plus `idle`, + * which takes no selector because stillness is a property of the whole screen. + */ export async function runDirective(env: ActionEnv, step: DirectiveStep): Promise { // Vega is remote-driven — there is no touch input, so the touch directives // can never act on it. Fail upfront with authoring guidance instead of a @@ -663,6 +688,8 @@ export async function runDirective(env: ActionEnv, step: DirectiveStep): Promise return waitForCondition(env, step, step.timeout ?? DEFAULT_ACTION_TIMEOUT_MS); case "assert": return waitForCondition(env, step, DEFAULT_ASSERT_TIMEOUT_MS); + case "idle": + return waitForIdle(env, step); case "scroll-to": { const r = await scrollToVisible(env, step.target, step.direction, step.within); if (r.aborted) return ABORTED_OUTCOME; @@ -1117,6 +1144,277 @@ async function waitForCondition( }; } +// ── Screen readiness ───────────────────────────────────────────────── +// +// `await: { idle: true }` asks one question a selector condition cannot: has +// the screen stopped moving? It is deliberately NOT an identity check — a +// dropped tap leaves the source screen perfectly idle — so it belongs next to +// the element check that says WHICH screen, never instead of it. + +/** + * `idle` poll cadence, matching `await-screen-idle`'s own. The timeout and hold + * defaults live in flow-utils beside the parser, which needs the timeout to + * reject a hold that could never fit inside the wait. + */ +const IDLE_POLL_MS = 200; + +/** + * How many consecutive intervals must read as still before the screen is + * called settled. Two, not one, because a single agreeing pair of captures is + * not evidence of stillness: any animation that reverses — a cross-fade, a + * pulse, a bounce — has a turning point, and two samples straddling it come + * back identical while the screen is very much moving. Observed on a 3s + * white/indigo cross-fade, where a default-shaped step passed on roughly one + * run in three. A second agreeing interval needs a third sample, which the + * same phase symmetry cannot supply unless the animation's period happens to + * match the poll — so the aliasing that survives one comparison does not + * survive two. + */ +const MIN_STILL_INTERVALS = 2; + +/** + * The smallest budget a poll round is allowed to start with. A round begun + * with nothing left cannot capture and cannot read, and both absences were + * being recorded as facts about the device: the skipped capture latched + * "captures do not work here", and the abandoned read latched "the tree source + * is not answering". Neither was true — the step had simply run out of time. + * The first round always runs, so an unusually short `timeout:` still buys one + * honest look, and ending up to one round early is strictly better than + * judging a screen nobody managed to observe. + */ +const MIN_ROUND_BUDGET_MS = IDLE_POLL_MS; + +/** How the last tree read ended. Only `value` licenses a verdict about the app. */ +type TreeReadOutcome = "value" | "error" | "timeout"; + +/** + * Wait until the screen has content and stops moving — in the UI tree AND in + * the rendered pixels. + * + * Both signals are required because each is blind to what the other sees. The + * tree cannot see presentation-layer motion: an iOS push or modal dismissal + * commits its hierarchy up front and then animates a layer over ~300-500ms, and + * a cross-fade or a scrim moves no node at all — so a tree-only settle reports + * a screen that is still sliding. Pixels cannot see a tree that is still + * churning behind an unchanged-looking surface, and anything genuinely animated + * forever (a video, a shimmer) would make a pixel-only settle unsatisfiable on + * a screen the tree calls ready. + * + * This is `await-screen-idle`'s question asked against the tree the directives + * actually resolve against, and — unlike that tool — it FAILS when the screen + * never settles, which is what makes it safe to persist in a flow. + * + * Every verdict is drawn from the LAST round that observed something, never + * from a latch remembering that the screen was once still: a screen that + * settles and then moves again has not settled. + */ +async function waitForIdle( + env: ActionEnv, + step: Extract +): Promise { + const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; + const minStableMs = step.minStableMs ?? IDLE_DEFAULT_MIN_STABLE_MS; + const deadline = Date.now() + timeoutMs; + + // Two hold clocks, because the tree can settle while the pixels have not. + // The combined one decides; the tree-only one feeds the degraded report at + // the bottom, for a run whose captures never produced a comparable pair. + let treeSignature: string | undefined; + let treeSince = 0; + let treeStillIntervals = 0; + let treeSettledAtLastRead = false; + let previousFrame: PixelFrame | undefined; + let bothSince = 0; + let stillIntervals = 0; + + let readsSucceeded = 0; + // Definitely assigned: the loop below always completes at least one round, + // and every arm of that round sets it. + let lastRead!: TreeReadOutcome; + let treeErrorMessage: string | undefined; + let sawContent = false; + let pixelsEverMoved = false; + let captureFailed = false; + let firstCapture = true; + + for (;;) { + if (env.signal?.aborted) return ABORTED_OUTCOME; + // The tree read is bounded by what is left of the step's budget, the same + // way the capture is. Without that bound `timeout:` was not an upper bound + // at all: no describe path takes a signal, and a wedged one (a hung + // ViewInspector RPC, an `adb` that has stopped answering) ran the round + // past the deadline — measured at 2.25s over an 8000ms budget. + const roundBudget = Math.max(1, deadline - Date.now()); + // Read both signals from as close to one instant as possible: they describe + // the same screen, and any gap between them is a window motion hides in. + // They also travel over different channels (tree source vs. capture + // backend), so serializing them would double the round without buying + // anything. + const [read, frame] = await Promise.all([ + settleWithin(fetchFlowTree(env.registry, env.device), roundBudget, env.signal), + capturePixelsWithin(env, deadline, firstCapture), + ]); + firstCapture = false; + // Before anything is concluded from this round: a capture abandoned by an + // abort comes back indistinguishable from one that failed, and a verdict + // about the app must never be derived from a run that was cancelled. + if (env.signal?.aborted || read.type === "aborted") return ABORTED_OUTCOME; + + if (read.type === "timeout") { + // The read did not come back inside the round. That is the absence of an + // observation, not an observation: it neither refutes the last known + // tree state nor stands in for one, so the hold state is left as it was + // and the bottom decides what, if anything, it means. + lastRead = "timeout"; + } else if (read.type === "error") { + // A tree-source blip mid-animation is expected; keep polling. Only its + // presence on the LAST read is reportable. + lastRead = "error"; + treeErrorMessage = read.error; + treeSignature = undefined; + previousFrame = undefined; + treeSince = 0; + treeStillIntervals = 0; + treeSettledAtLastRead = false; + bothSince = 0; + stillIntervals = 0; + } else { + lastRead = "value"; + readsSucceeded += 1; + treeErrorMessage = undefined; + const tree = read.value.tree; + if (tree.children.length === 0) { + // Blank or still loading — never "settled", and it resets both holds. + // Unlike a failed read this IS an observation, so it also clears the + // tree-only verdict: a screen showing nothing has not settled on + // anything. + treeSignature = undefined; + previousFrame = undefined; + treeSince = 0; + treeStillIntervals = 0; + treeSettledAtLastRead = false; + bothSince = 0; + stillIntervals = 0; + } else { + sawContent = true; + const signature = treeFingerprint(tree); + const now = Date.now(); + + // Stillness is a property of an INTERVAL, so no verdict comes from one + // observation — and, per MIN_STILL_INTERVALS, none comes from one + // interval either. `minStableMs: 0` therefore still means three reads: + // a single sample proves nothing about motion, and a single agreeing + // pair can be two points of an animation that reversed between them. + const treeHeld = signature === treeSignature; + treeSignature = signature; + if (!treeHeld) { + treeSince = now; + treeStillIntervals = 0; + } else { + treeStillIntervals += 1; + } + treeSettledAtLastRead = + treeStillIntervals >= MIN_STILL_INTERVALS && now - treeSince >= minStableMs; + + // A missing frame is the ABSENCE of visual evidence, never evidence of + // stillness. Letting it stand in for "the pixels held" is what turned a + // screen that never stopped moving into a pass. + let pixelsHeld = false; + if (frame === undefined) { + captureFailed = true; + } else if (previousFrame !== undefined) { + if (pixelsDiffer(previousFrame, frame)) pixelsEverMoved = true; + else pixelsHeld = true; + } + previousFrame = frame; + + if (treeHeld && pixelsHeld) { + stillIntervals += 1; + if (stillIntervals >= MIN_STILL_INTERVALS && now - bothSince >= minStableMs) { + return { ok: true }; + } + } else { + bothSince = now; + stillIntervals = 0; + } + } + } + + if (env.signal?.aborted) return ABORTED_OUTCOME; + const left = deadline - Date.now(); + if (left < MIN_ROUND_BUDGET_MS) break; + if (!(await sleepOrAbort(Math.min(IDLE_POLL_MS, left), env.signal))) return ABORTED_OUTCOME; + } + + // An unreadable window is never a verdict about the app. Which flavour of + // unreadable it was decides the repair, so they stay apart. + const unreadable = (underlying: string): DirectiveOutcome => ({ + ok: false, + indeterminate: true, + // The underlying reader reports an instrumentation failure, whose remedy + // (relaunch the app) is the wrong repair for the commonest cause of it + // here: the app is simply not in the foreground, which reads exactly the + // same from the tree source. Name that first so the author checks it + // before relaunching anything. + reason: + `could not read the UI tree while waiting for the screen to settle — check the app is ` + + `still in the foreground (a backgrounded app reads the same as an uninstrumented one). ` + + `Underlying error: ${underlying}`, + }); + + if (readsSucceeded === 0) { + if (treeErrorMessage !== undefined) return unreadable(treeErrorMessage); + return { + ok: false, + indeterminate: true, + reason: + `the tree source never answered within the step's ${timeoutMs}ms — raise this step's ` + + `\`timeout:\` if it is merely slow (a tree read on a busy screen can take seconds), or ` + + `repair it if it has stopped answering altogether`, + }; + } + // Reads worked, then stopped: a backgrounded app, a dropped instrumentation + // session. One early success does not license a verdict drawn from a window + // that went dark afterwards. (A read that merely ran out of budget is NOT + // this case — it is the step ending, and the evidence below still stands.) + if (lastRead === "error" && treeErrorMessage !== undefined) { + return unreadable(treeErrorMessage); + } + // Readable throughout and never once carrying content: the screen rendered + // nothing, which is not the same claim as "it never stopped moving". + if (!sawContent) { + return { + ok: false, + indeterminate: true, + reason: `the UI tree stayed empty for ${timeoutMs}ms — the screen never rendered content`, + }; + } + // The tree was settled as of the last read and no pair of captures ever + // showed motion, yet the combined hold never completed. With captures + // arriving this is unreachable: a pair either agrees — and the tree was + // holding, so the hold would have run — or disagrees, which sets + // pixelsEverMoved. So `captureFailed` is what is left, and it is required + // here rather than assumed. The hierarchy genuinely held still, so this is a + // pass; half of the proof is missing, so it is a warned one. + if (treeSettledAtLastRead && !pixelsEverMoved && captureFailed) { + return { + ok: true, + warning: + `settled on the UI tree alone — no screenshot of this screen could be read, so animation ` + + `that moves pixels without moving nodes (a push, a fade, a dismissing modal) was not ` + + `waited out. Follow this with the element check the next step actually needs.`, + }; + } + return { + ok: false, + reason: + `the screen never held still for ${minStableMs}ms within ${timeoutMs}ms — the UI tree or ` + + `the pixels kept changing, so it is still animating, or something on it never stops moving ` + + `(a looping animation, a video, a carousel that keeps advancing). Gate on the element you ` + + `actually need instead of on stillness.`, + }; +} + function assertReason( condition: WaitCondition, selector: FlowSelector, diff --git a/packages/tool-server/src/tools/flows/flow-device.ts b/packages/tool-server/src/tools/flows/flow-device.ts index 22a2d1da4..93a34b574 100644 --- a/packages/tool-server/src/tools/flows/flow-device.ts +++ b/packages/tool-server/src/tools/flows/flow-device.ts @@ -201,10 +201,12 @@ export function stripDeviceKeys(args: Record): Record { + if (env.device.platform === "chromium") { + const ref = chromiumCdpRef(env.device); + const api = (await env.registry.resolveService(ref.urn, ref.options)) as ChromiumCdpApi; + const { path } = await api.captureScreenshot({ scale: CAPTURE_SCALE }); + return path; + } + if (env.device.platform === "vega") { + return captureVegaScreenshotPng({ scale: CAPTURE_SCALE }); + } + // Shape alone cannot tell tvOS from iOS — both are 8-4-4-4-12 UUIDs tagged + // `platform: "ios"` — so ask the runtime, which is memoized per UDID. + if (env.device.platform === "ios" && (await isTvOsSimulator(env.device.id))) { + return tvScreenshot(env.device.id, CAPTURE_SCALE, undefined); + } + const ref = simulatorServerRef(env.device); + const api = (await env.registry.resolveService(ref.urn, ref.options)) as SimulatorServerApi; + // Deliberately NOT threading env.signal into this capture: the + // simulator-server writes its temp PNG to disk before replying, and the + // reply is the only place the path is learned — severing the fetch on abort + // would orphan that file. Cancellation stays responsive regardless, because + // callers abandon this promise via settleWithin; the capture just runs to + // completion on its own bounds, learns the path, and the `finally` in + // capturePixels removes the file — the same ownership the Chromium arm above + // has, whose captureScreenshot takes no signal either. + const { path } = await httpScreenshot(api, undefined, undefined, CAPTURE_SCALE); + return path; +} + +/** + * One capture as decoded pixels, or `undefined` when the pixels could not be + * read (any capture or decode failure). Soft by design — the caller treats it + * as the ABSENCE of visual evidence, never as evidence of stillness. + */ +async function capturePixels(env: ActionEnv): Promise { + try { + const file = await captureFile(env); + try { + const png = PNG.sync.read(await fs.readFile(file)); + return { width: png.width, height: png.height, data: png.data }; + } finally { + await fs.rm(file, { force: true }).catch(() => {}); + } + } catch { + return undefined; + } +} + +/** + * One capture bounded by both its own per-capture budget and the caller's + * deadline. Every way of not getting a frame collapses to `undefined` — the + * caller has one response to all of them (no visual evidence this round), so + * distinguishing them here would only invent a difference it cannot act on. + * Abort is the caller's to notice: it holds the signal and checks it either + * side of this call. + */ +export async function capturePixelsWithin( + env: ActionEnv, + deadline: number, + firstCapture: boolean +): Promise { + const budget = Math.min(deadline - Date.now(), pixelCaptureTimeoutMs(env.device, firstCapture)); + if (budget <= 0) return undefined; + const result = await settleWithin(capturePixels(env), budget, env.signal); + return result.type === "value" ? result.value : undefined; +} + +/** + * Did the screen move between two captures? Different dimensions count as + * motion; otherwise the changed-pixel fraction is compared against + * {@link MOTION_FRACTION}. Alpha is ignored — a screen capture is opaque. + * + * The dimension branch covers a resized window (Chromium). It is NOT how a + * device rotation is caught: the Android capture keeps its portrait shape + * across one, so rotation registers through content change like anything else. + */ +export function pixelsDiffer(a: PixelFrame, b: PixelFrame): boolean { + if (a.width !== b.width || a.height !== b.height) return true; + const total = a.width * a.height; + if (total === 0) return false; + const limit = Math.min(a.data.length, b.data.length); + let changed = 0; + for (let o = 0; o + 2 < limit; o += 4) { + const dr = a.data[o] - b.data[o]; + const dg = a.data[o + 1] - b.data[o + 1]; + const db = a.data[o + 2] - b.data[o + 2]; + if (dr * dr + dg * dg + db * db > PIXEL_THRESHOLD_SQUARED) changed++; + } + return changed / total > MOTION_FRACTION; +} diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 6e10c88f0..6e19cdbf5 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -188,6 +188,14 @@ export interface StepReport { * the runner does not own reports no reason. */ reason?: string; + /** + * The step passed, but the WAY it passed weakens it as proof. Rendered as a + * "⚠" suffix by the MCP client. Raised by an `await: { idle: true }` whose + * captures never produced a comparable pair, so it proved stillness on the + * UI tree alone and never saw the presentation-layer motion it exists to + * catch. + */ + warning?: string; /** Underlying tool id for `tool` steps. */ tool?: string; /** Tool result for `tool` steps. */ @@ -915,7 +923,11 @@ reads "inside card inside list", each container's frame inside the next); (\`pinch: { on?, scale }\` — scale > 1 in, < 1 out; screen center when \`on\` is omitted); \`rotate\` is the two-finger rotation gesture (\`rotate: { on?, by }\` — degrees, + clockwise, within ±3000°; screen center when \`on\` is omitted; distinct from the \`rotate\` tool, which changes device orientation); \`await\` waits -for a UI condition; \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` +for a UI condition, and additionally takes the one condition that has no selector: \`idle: true\` waits +until the screen has content and stops moving in BOTH the UI tree and the rendered pixels (unlike the +\`await-screen-idle\` tool it FAILS on timeout, so it is safe to persist; it says nothing about WHICH +screen settled — a dropped tap leaves the source screen perfectly idle — so pair it with the element +check that names the destination); \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` diffs a screenshot — or, with \`cropOn: \`, one element's cropped region — against a stored baseline (a missing baseline fails the step — set updateBaselines to adopt the current screen; a cropped element whose size drifted fails on dimensions); \`echo\` annotates; \`run\` executes another flow @@ -1549,6 +1561,10 @@ function stepTarget(step: FlowStep): string | undefined { case "await": case "assert": return conditionLabel(step, selectorLabel); + case "idle": + // The caller already prints the kind, and this step has no target beyond + // the screen itself: returning one would render as "idle screen idle". + return undefined; case "when": return step.condition.kind === "platform" ? `platform ${step.condition.platform}` @@ -2098,6 +2114,7 @@ async function execLeafStep( case "type": case "await": case "assert": + case "idle": case "scroll-to": case "pinch": case "rotate": { @@ -2109,7 +2126,23 @@ async function execLeafStep( // A run cancelled mid-directive is a skip (matching the pre-step guard // and `wait`), never a step failure — the app did nothing wrong. if (r.aborted) return { ...base, status: "skip", reason: r.reason }; - return { ...base, status: r.ok ? "pass" : "fail", reason: r.reason }; + // An INDETERMINATE `idle` outcome is not a verdict about the app: the + // wait could not run at all (an unreadable or degraded tree, a screen + // nobody managed to observe). Reporting it as `fail` makes CI read an + // environment problem as a regression and a QA author reset a pass + // streak over it. `error` keeps the run non-ok while saying plainly + // that the app was never judged. Scoped to `idle`, whose whole verdict + // rests on being able to observe the screen; the selector conditions + // keep their existing `fail` mapping. + if (!r.ok && r.indeterminate && step.kind === "idle") { + return { ...base, status: "error", reason: r.reason }; + } + return { + ...base, + status: r.ok ? "pass" : "fail", + reason: r.reason, + ...(r.warning !== undefined ? { warning: r.warning } : {}), + }; } catch (err) { return { ...base, status: "error", reason: errMsg(err) }; } diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 3b42e93d6..d35d3fa31 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -706,6 +706,14 @@ export type FlowStep = expectedText?: string; textMatch?: TextMatchMode; } + /** + * Screen READINESS: the UI tree has content, and neither it nor the rendered + * pixels are still changing. Spelled `await: { idle: true }` — a condition + * like any other, but the only one that takes no selector, because stillness + * is a property of the whole screen. There is no `assert` form: "has it + * stopped moving yet" is inherently a wait. + */ + | { kind: "idle"; timeout?: number; minStableMs?: number } | { kind: "wait"; ms: number } | { kind: "scroll-to"; target: FlowSelector; direction: ScrollDirection; within?: FlowSelector } | { kind: "pinch"; selector?: FlowSelector; scale: number } @@ -843,6 +851,14 @@ type YamlWaitCondition = type YamlTextWaitCondition = Extract; +/** + * The one condition that takes no selector. It shares the `await:` key with + * {@link YamlWaitCondition} but is deliberately NOT part of that union — a step + * body carries either a selector condition or this one, never a mix — so it is + * parsed by {@link parseIdleFields} rather than by parseWaitFields. + */ +type YamlIdleCondition = { idle: true; minStableMs?: number; timeout?: number }; + /** `scroll-to` body: a bare target (scrolls down), or a map with options. */ type YamlScrollBody = | YamlSelector @@ -867,7 +883,7 @@ type YamlStep = | { tap: TapBody } | { "long-press": YamlTarget | { on: YamlTarget; duration?: number } } | { type: { into: YamlSelector; text: string; submit?: boolean } } - | { await: YamlWaitCondition & { timeout?: number } } + | { await: (YamlWaitCondition & { timeout?: number }) | YamlIdleCondition } | { assert: YamlWaitCondition } | { wait: number } | { "scroll-to": YamlScrollBody } @@ -1150,10 +1166,24 @@ function waitToYaml( return body; } +/** + * Sugar an `idle` step back under its `await:` key. Optional fields are emitted + * only when set, so the canonical minimal spelling (`await: { idle: true }`) + * round-trips unchanged. + */ +function idleToYaml(step: Extract): YamlStep { + const body: YamlIdleCondition = { idle: true }; + if (step.minStableMs !== undefined) body.minStableMs = step.minStableMs; + if (step.timeout !== undefined) body.timeout = step.timeout; + return { await: body }; +} + function toYamlStep(step: FlowStep): YamlStep { switch (step.kind) { case "echo": return { echo: step.message }; + case "idle": + return idleToYaml(step); case "launch": return { launch: step.app }; case "run": @@ -1568,18 +1598,22 @@ type WaitFields = { * `assert` carrying one is rejected rather than silently ignored. */ function parseWaitFields(raw: unknown, kind: "await" | "assert" | "when"): WaitFields { + // What the author is allowed to write, which is NOT the same as what this + // function parses: a body naming `idle` is routed to parseIdleFields before + // we get here, so this list is only ever read by an author whose body named + // no legal condition, or more than one — and omitting `idle` from it left the + // one condition they may have been reaching for out of the answer. Only + // `await` gains it; `assert` and `when:` genuinely have no readiness form. + const legalKeys = kind === "await" ? [...WAIT_CONDITIONS, IDLE_CONDITION] : WAIT_CONDITIONS; if (raw === null || typeof raw !== "object") { - badEntry({ [kind]: raw }, `${kind} needs a condition (${WAIT_CONDITIONS.join(", ")})`); + badEntry({ [kind]: raw }, `${kind} needs a condition (${legalKeys.join(", ")})`); } const b = raw as Record; // The condition is the key; its value is the selector. const present = WAIT_CONDITIONS.filter((c) => c in b); if (present.length !== 1) { - badEntry( - { [kind]: b }, - `${kind} needs exactly one condition key (${WAIT_CONDITIONS.join(", ")})` - ); + badEntry({ [kind]: b }, `${kind} needs exactly one condition key (${legalKeys.join(", ")})`); } const condition = present[0]!; @@ -1591,16 +1625,7 @@ function parseWaitFields(raw: unknown, kind: "await" | "assert" | "when"): WaitF "assert has no timeout — it is an immediate check; use `await` for a timed wait" ); } - // Like `wait`, reject non-finite values: YAML `.inf` (or an overflowing - // literal like 1e400) parses to Infinity — typeof number and > 0 — which - // would make the runner's poll deadline unreachable and the await unbounded. - if (typeof b.timeout !== "number" || !Number.isFinite(b.timeout) || b.timeout <= 0) { - badEntry( - { [kind]: b }, - "await.timeout needs a positive number of milliseconds (e.g. `timeout: 10000`)" - ); - } - timeout = b.timeout as number; + timeout = parseAwaitTimeout({ [kind]: b }, b.timeout); } // `await` takes the condition key plus `timeout`; `assert` the condition key @@ -1658,6 +1683,119 @@ function parseWaitFields(raw: unknown, kind: "await" | "assert" | "when"): WaitF return { condition, selector: parseSelector(b[condition], `${kind}.${condition}`), timeout }; } +/** + * The one condition key that takes no selector. Its presence in an + * `await`/`assert` body routes parsing away from {@link parseWaitFields} — see + * {@link parseIdleFields}. + */ +const IDLE_CONDITION = "idle"; + +/** + * `idle`'s defaults, spelled here rather than beside the runner because the + * parser needs the timeout one: a hold that cannot fit inside the wait is a + * gate that fails on every run, and this file rejects unsatisfiable gates. + */ +export const IDLE_DEFAULT_TIMEOUT_MS = 7500; +export const IDLE_DEFAULT_MIN_STABLE_MS = 250; + +/** + * The `timeout` sibling key an `await` may carry, spelled once for both the + * selector conditions and `idle`. + * + * Non-finite values are rejected alongside non-positive ones: YAML `.inf` (or + * an overflowing literal like 1e400) parses to Infinity — typeof number and + * > 0 — which would make the runner's poll deadline unreachable and the await + * unbounded. + */ +function parseAwaitTimeout(entry: unknown, value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + badEntry( + entry, + "await.timeout needs a positive number of milliseconds (e.g. `timeout: 10000`)" + ); + } + return value as number; +} + +/** Bounded non-negative integer option, in milliseconds. */ +function parseBoundedMs( + entry: unknown, + value: unknown, + where: string, + max: number, + why?: string +): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > max) { + badEntry( + entry, + `${where} needs an integer between 0 and ${max} (milliseconds)${why ? ` — ${why}` : ""}` + ); + } + return value as number; +} + +/** + * Parse an `await`/`assert` body carrying the `idle` condition. Returns the + * finished step, because unlike the selector conditions it has no selector to + * hand back as fields. + * + * `assert` is rejected outright: waiting is the whole point of the check. + */ +function parseIdleFields(raw: Record, kind: "await" | "assert"): FlowStep { + const entry = { [kind]: raw }; + + if (kind !== "await") { + badEntry( + entry, + "idle has no assert form — it waits for the screen to stop changing, which is an `await`" + ); + } + rejectUnknownKeys(entry, raw, ["idle", "minStableMs", "timeout"], kind); + + // `idle: true` only. A falsey value would spell "assert the screen is NOT + // settled", which no flow wants and the runner cannot answer. + if (raw.idle !== true) { + badEntry(entry, "idle takes only `true` (`await: { idle: true }`)"); + } + + const step: Extract = { kind: "idle" }; + if ("timeout" in raw) step.timeout = parseAwaitTimeout(entry, raw.timeout); + if (raw.minStableMs !== undefined) { + // A hold as long as the whole wait can never be observed within it, so the + // step would fail on every run — and fail blaming the app, which is the one + // thing such a failure is not evidence about. Caught here, deviceless. + const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; + step.minStableMs = parseBoundedMs( + entry, + raw.minStableMs, + "idle.minStableMs", + timeoutMs - 1, + `the hold has to fit inside the ${step.timeout === undefined ? "default " : ""}${timeoutMs}ms timeout` + ); + } + return step; +} + +/** + * Whether an `await`/`assert` body names the `idle` condition rather than an + * ordinary selector one. Rejects a body that mixes the two rather than silently + * preferring one. + */ +function isIdleCondition(raw: unknown, kind: "await" | "assert"): boolean { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return false; + const body = raw as Record; + if (!(IDLE_CONDITION in body)) return false; + const selectorConditions = WAIT_CONDITIONS.filter((c) => c in body); + if (selectorConditions.length > 0) { + badEntry( + { [kind]: body }, + `${kind} mixes \`${IDLE_CONDITION}\` with \`${selectorConditions.join("`, `")}\` — a step ` + + `checks exactly one condition` + ); + } + return true; +} + /** * The platform set, spelled once: launch maps, `when: { platform }` guards * ({@link WhenPlatform}), flow-device's `FlowPlatform`, and flow-run's @@ -2267,12 +2405,24 @@ function fromYamlStep(raw: YamlStep, whenDepth = 0): FlowStep { return step; } + // `await:` / `assert:` carry two families of condition: the selector ones + // (visible/hidden/exists/text, matched against the UI tree) and `idle`, which + // takes no selector because stillness is a property of the whole screen. The + // body's key decides which. if ("await" in raw) { - return { kind: "await", ...parseWaitFields((raw as { await: unknown }).await, "await") }; + const body = (raw as { await: unknown }).await; + if (isIdleCondition(body, "await")) { + return parseIdleFields(body as Record, "await"); + } + return { kind: "await", ...parseWaitFields(body, "await") }; } if ("assert" in raw) { - return { kind: "assert", ...parseWaitFields((raw as { assert: unknown }).assert, "assert") }; + const body = (raw as { assert: unknown }).assert; + if (isIdleCondition(body, "assert")) { + return parseIdleFields(body as Record, "assert"); + } + return { kind: "assert", ...parseWaitFields(body, "assert") }; } if ("wait" in raw) { diff --git a/packages/tool-server/src/tools/screenshot-diff/screenshot-diff.ts b/packages/tool-server/src/tools/screenshot-diff/screenshot-diff.ts index eb7d22cd1..f0e4d0d56 100644 --- a/packages/tool-server/src/tools/screenshot-diff/screenshot-diff.ts +++ b/packages/tool-server/src/tools/screenshot-diff/screenshot-diff.ts @@ -105,6 +105,10 @@ interface DiffArtifactPaths { } const MAX_RGB_DISTANCE_SQUARED = 255 * 255 * 3; +// Sized for a baseline PNG stored across sessions, machines and OS versions, +// so it absorbs real drift. flow-pixels' PIXEL_THRESHOLD deliberately does NOT +// mirror it (it compares two captures from one live session, a far lower noise +// floor) — the two are independent by design; see the rationale there. const DEFAULT_THRESHOLD = 0.1; const DEFAULT_IGNORE_TOP_NORMALIZED_Y = 0.06; const DEFAULT_REGION_MERGE_DISTANCE = 8; diff --git a/packages/tool-server/src/tools/screenshot/index.ts b/packages/tool-server/src/tools/screenshot/index.ts index 7ee753085..38af5ae17 100644 --- a/packages/tool-server/src/tools/screenshot/index.ts +++ b/packages/tool-server/src/tools/screenshot/index.ts @@ -75,8 +75,11 @@ const capability: ToolCapability = { * tvOS screenshot path. The simulator-server backend does not support tvOS, so * capture via `xcrun simctl io screenshot` instead and (optionally) * downscale with `sips` to match the iOS/Android scale behaviour. + * + * Exported for the flow settle, which captures for motion detection rather than + * for an artifact and so cannot go through the tool wrapper above. */ -async function tvScreenshot( +export async function tvScreenshot( udid: string, scale: number, signal: AbortSignal | undefined diff --git a/packages/tool-server/src/utils/simulator-client.ts b/packages/tool-server/src/utils/simulator-client.ts index 788b85202..a0bd1bdd2 100644 --- a/packages/tool-server/src/utils/simulator-client.ts +++ b/packages/tool-server/src/utils/simulator-client.ts @@ -30,7 +30,7 @@ const DEFAULT_SCREENSHOT_SCALE = 0.3; // https://github.com/software-mansion/argent/issues/391). Poll past that // transient instead of surfacing it as a hard failure. const NO_IMAGE_ERROR = /no image to export/i; -const FIRST_FRAME_WAIT_MS = 6_000; +export const FIRST_FRAME_WAIT_MS = 6_000; const FIRST_FRAME_POLL_MS = 250; /** diff --git a/packages/tool-server/test/flows/flow-deviceless.test.ts b/packages/tool-server/test/flows/flow-deviceless.test.ts index 5f1bc5716..fe7c4773f 100644 --- a/packages/tool-server/test/flows/flow-deviceless.test.ts +++ b/packages/tool-server/test/flows/flow-deviceless.test.ts @@ -251,6 +251,7 @@ describe("stepRequiresDevice", () => { "type": true, "await": true, "assert": true, + "idle": true, "scroll-to": true, "pinch": true, "rotate": true, @@ -268,6 +269,7 @@ describe("stepRequiresDevice", () => { "type": { kind: "type", into: { text: "f" }, text: "hi" }, "await": { kind: "await", condition: "visible", selector: { text: "f" } }, "assert": { kind: "assert", condition: "visible", selector: { text: "f" } }, + "idle": { kind: "idle" }, "scroll-to": { kind: "scroll-to", target: { text: "f" }, direction: "down" }, "pinch": { kind: "pinch", scale: 2 }, "rotate": { kind: "rotate", by: 90 }, diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts new file mode 100644 index 000000000..91aa8085a --- /dev/null +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { parseFlow, serializeFlow, type FlowStep } from "../../src/tools/flows/flow-utils"; + +// `idle` is the one condition that takes no selector. It shares the `await:` +// key with the selector conditions, so the parse/serialize round trip and the +// mutual exclusion between the two families are the load-bearing behaviors. + +const flow = (steps: string): string => `executionPrerequisite: ""\nsteps:\n${steps}`; + +function parseSteps(steps: string): FlowStep[] { + return parseFlow(flow(steps)).steps; +} + +/** A flow's steps survive serialize → parse unchanged (canonical spelling). */ +function expectRoundTrip(steps: string): FlowStep[] { + const parsed = parseSteps(steps); + expect(parseFlow(serializeFlow({ executionPrerequisite: "", steps: parsed })).steps).toEqual( + parsed + ); + return parsed; +} + +describe("await { idle }", () => { + it("parses the readiness gate and round-trips its minimal spelling", () => { + const steps = expectRoundTrip(` - await: { idle: true }\n`); + expect(steps).toEqual([{ kind: "idle" }]); + // Minimal in, minimal out — no defaults materialize into the file. + expect(serializeFlow({ executionPrerequisite: "", steps })).toContain( + "await:\n idle: true" + ); + }); + + it("carries the optional hold and timeout", () => { + expect(expectRoundTrip(` - await: { idle: true, minStableMs: 400, timeout: 9000 }\n`)).toEqual( + [{ kind: "idle", minStableMs: 400, timeout: 9000 }] + ); + }); + + it("has no assert form — waiting is the whole point of the check", () => { + expect(() => parseSteps(` - assert: { idle: true }\n`)).toThrow(/idle has no assert form/); + }); + + it("takes only `true` — there is no useful 'prove the screen is moving'", () => { + expect(() => parseSteps(` - await: { idle: false }\n`)).toThrow(/idle takes only/); + }); + + it("bounds minStableMs", () => { + expect(() => parseSteps(` - await: { idle: true, minStableMs: -1 }\n`)).toThrow( + /idle.minStableMs/ + ); + expect(() => parseSteps(` - await: { idle: true, minStableMs: 1.5 }\n`)).toThrow( + /idle.minStableMs/ + ); + }); + + // A hold that cannot fit inside the wait is a gate that fails on every run — + // and fails blaming the app, which is the one thing it is not evidence + // about. Caught at parse, deviceless, rather than against a live screen. + it("rejects a hold that could never fit inside the timeout", () => { + expect(() => + parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 1000 }\n`) + ).toThrow( + /idle.minStableMs needs an integer between 0 and 499 .* fit inside the 500ms timeout/ + ); + // With no explicit timeout the default is what it has to fit inside, and + // the message says which number it is measuring against. + expect(() => parseSteps(` - await: { idle: true, minStableMs: 9000 }\n`)).toThrow( + /fit inside the default 7500ms timeout/ + ); + // The boundary itself is legal on both sides. + expect(parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 499 }\n`)).toEqual([ + { kind: "idle", timeout: 500, minStableMs: 499 }, + ]); + expect(() => parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 500 }\n`)).toThrow( + /idle.minStableMs/ + ); + }); + + it("rejects a non-positive timeout", () => { + expect(() => parseSteps(` - await: { idle: true, timeout: 0 }\n`)).toThrow(/await.timeout/); + expect(() => parseSteps(` - await: { idle: true, timeout: "soon" }\n`)).toThrow( + /await.timeout/ + ); + }); +}); + +describe("condition families are mutually exclusive", () => { + it("rejects mixing a selector condition with the readiness one", () => { + expect(() => parseSteps(` - await: { idle: true, visible: { id: x } }\n`)).toThrow( + /mixes `idle` with `visible`/ + ); + }); + + it("rejects a stray key rather than ignoring it", () => { + expect(() => parseSteps(` - await: { idle: true, settleMs: 500 }\n`)).toThrow(/settleMs/); + }); + + // A typo next to an `idle:` gate used to be told that `idle` itself was not a + // legal key — the parser lists what the AUTHOR may write, which is not the + // same set as what its selector-condition branch parses. + it("offers idle when an await names no legal condition", () => { + expect(() => parseSteps(` - await: { visble: { id: home } }\n`)).toThrow( + /await needs exactly one condition key \(exists, visible, hidden, text, idle\)/ + ); + // Same list when the body isn't a condition map at all. + expect(() => parseSteps(` - await: visible\n`)).toThrow( + /await needs a condition \(exists, visible, hidden, text, idle\)/ + ); + }); + + it("does not offer it to assert or to a `when:` guard, which have no readiness form", () => { + const assertMiss = (): FlowStep[] => parseSteps(` - assert: { visble: { id: home } }\n`); + expect(assertMiss).toThrow( + /assert needs exactly one condition key \(exists, visible, hidden, text\)/ + ); + expect(assertMiss).not.toThrow(/idle/); + + const guard = (body: string) => (): FlowStep[] => + parseSteps(` - when: ${body}\n steps:\n - echo: guarded\n`); + + // A stray key carrying neither substring, so the rejected entry echoed + // back into the message cannot satisfy the negative assertion. + const stray = guard("{ visble: { id: home } }"); + expect(stray).toThrow( + /when needs exactly one condition key \(exists, visible, hidden, text, platform\)/ + ); + expect(stray).not.toThrow(/idle/); + + // And `idle` itself is not a guard. + expect(guard("{ idle: true }")).toThrow(/when needs exactly one condition key/); + }); + + it("leaves the selector conditions untouched", () => { + expect(parseSteps(` - await: { visible: { id: home-screen } }\n`)).toEqual([ + { kind: "await", condition: "visible", selector: { identifier: "home-screen" } }, + ]); + }); +}); diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts new file mode 100644 index 000000000..6ddf89925 --- /dev/null +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -0,0 +1,412 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { Registry } from "@argent/registry"; +import type { DescribeNode, DescribeTreeData } from "../../src/tools/describe/contract"; +import type { PixelFrame } from "../../src/tools/flows/flow-pixels"; + +// Serve the flow tree directly (see flow-when.test.ts) — `idle` polls it. +let currentTree: () => DescribeNode; +/** Simulates a tree source that is slow, or wedged when it exceeds the step. */ +let treeDelayMs = 0; +vi.mock("../../src/tools/flows/flow-tree", () => ({ + fetchFlowTree: vi.fn(async (): Promise => { + if (treeDelayMs > 0) await new Promise((r) => setTimeout(r, treeDelayMs)); + return { + tree: currentTree(), + source: "native-devtools", + screen: { width: 390, height: 844 }, + }; + }), +})); + +// Stub only the capture; the real `pixelsDiffer` decides whether two frames +// moved, so the comparison the check depends on is the one under test. +let currentFrame: () => PixelFrame | undefined; +vi.mock("../../src/tools/flows/flow-pixels", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // Honours `deadline` exactly as the real one does — it returns undefined + // without capturing once the budget is gone. A stub that ignored it hid + // the case where the runner judges a screen on a round it had no time to + // observe, which is where a missing frame used to read as stillness. + capturePixelsWithin: vi.fn( + async (_env: unknown, deadline: number): Promise => + Date.now() >= deadline ? undefined : currentFrame() + ), + }; +}); + +import { createRunFlowTool, type FlowRunResult } from "../../src/tools/flows/flow-run"; + +const DEVICE = "00000000-0000-0000-0000-0000000000ab"; // iOS UDID shape +let tmpDir: string; + +function n(partial: Partial & { frame: DescribeNode["frame"] }): DescribeNode { + return { role: "AXOther", children: [], ...partial }; +} + +const FULL: DescribeNode["frame"] = { x: 0, y: 0, width: 1, height: 1 }; + +function screenWith(label: string): DescribeNode { + return n({ + role: "AXWindow", + frame: FULL, + children: [n({ frame: { x: 0, y: 0, width: 1, height: 0.1 }, label })], + }); +} + +/** A 10x10 frame filled with one grey level — a uniform "screen". */ +function frameAt(level: number): PixelFrame { + const data = Buffer.alloc(10 * 10 * 4, level); + return { width: 10, height: 10, data }; +} + +function mockRegistry(): Registry { + return { + invokeTool: vi.fn(async (id: string) => { + if (id === "list-devices") return { devices: [] }; + if (id === "restart-app") return { restarted: true }; + return { ok: true }; + }), + getTool: vi.fn(() => undefined), + resolveService: vi.fn(async () => ({ isConnected: () => true })), + } as unknown as Registry; +} + +async function writeFlow(name: string, yaml: string): Promise { + const dir = path.join(tmpDir, ".argent", "flows"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, `${name}.yaml`), yaml, "utf8"); +} + +async function run(name: string): Promise { + const tool = createRunFlowTool(mockRegistry()); + const result = await tool.execute({}, { name, project_root: tmpDir, device: DEVICE }, undefined); + if (!("steps" in result)) throw new Error("expected a run result"); + return result; +} + +beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-idle-")); + currentTree = () => screenWith("Home"); + currentFrame = () => frameAt(120); + treeDelayMs = 0; +}); + +afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + vi.clearAllMocks(); +}); + +// `await: { idle: true }` is the readiness check. Its whole reason to exist is +// that it FAILS — the `await-screen-idle` tool reports `settled: false` softly, +// which cannot carry a regression verdict on an unattended replay. +describe("await: { idle }", () => { + it("passes once both the tree and the pixels hold still", async () => { + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(true); + expect(r.steps.at(-1)).toMatchObject({ kind: "idle", status: "pass" }); + // Both signals were available, so nothing about this pass is weakened. + expect(r.steps.at(-1)!.warning).toBeUndefined(); + }); + + // Stillness is a property of an interval, and one interval can alias (see + // the reversing-animation case below), so `minStableMs: 0` still means "the + // first two agreeing intervals" — three reads — not "the first read". + it("never settles on one read or one interval, even with no hold requested", async () => { + let reads = 0; + currentTree = () => { + reads += 1; + return screenWith("Home"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + expect((await run("ready")).ok).toBe(true); + expect(reads).toBeGreaterThanOrEqual(3); + }); + + it("fails when the tree never stops changing", async () => { + let tick = 0; + currentTree = () => screenWith(`frame ${tick++}`); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 600, minStableMs: 300 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + const step = r.steps.at(-1)!; + expect(step.status).toBe("fail"); + expect(step.reason).toContain("never held still"); + }); + + // The reason this check reads pixels at all: an iOS push or modal dismissal + // commits its hierarchy up front and then animates a layer for a few hundred + // milliseconds. The tree is perfectly still the whole time. + it("fails when the pixels keep moving under a motionless tree", async () => { + let level = 0; + currentFrame = () => frameAt((level += 60) % 240); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 600, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + const step = r.steps.at(-1)!; + expect(step.status).toBe("fail"); + expect(step.reason).toContain("never held still"); + expect(step.reason).toContain("pixels"); + }); + + // Sub-threshold drift is encoder noise, not motion — treating it as motion + // would make the check unsatisfiable on a screen that is genuinely at rest. + it("tolerates capture noise below the motion threshold", async () => { + let level = 120; + currentFrame = () => frameAt((level += 1)); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 2000, minStableMs: 0 } +` + ); + expect((await run("ready")).ok).toBe(true); + }); + + // A reversing animation — a cross-fade, a pulse, a bounce — has a turning + // point, and two samples straddling it come back identical while the screen + // is still moving. Measured on a live 3s cross-fade: a default-shaped step + // passed on roughly one run in three until a second agreeing interval was + // required. Here every other capture repeats its predecessor, so a + // one-interval rule settles and a two-interval rule cannot. + it("does not settle on a single agreeing pair of a reversing animation", async () => { + let tick = 0; + currentFrame = () => { + // 0, 60, 60, 120, 120, 180, 180, … — one still interval, never two. + const level = Math.floor((tick + 1) / 2) * 60; + tick += 1; + return frameAt(level % 240); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1500, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + expect(r.steps.at(-1)!.reason).toContain("never held still"); + }); + + // `timeout:` is the author's answer to "how long may this take", so it has to + // be the answer. No describe path takes an abort signal, so a wedged tree + // source (a hung ViewInspector RPC, an adb that stopped answering) used to + // run the round past the deadline — measured at 2.25s over an 8s budget on a + // live simulator. + it("honours its timeout even when the tree source stops answering", async () => { + treeDelayMs = 5_000; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 800, minStableMs: 0 } +` + ); + const started = Date.now(); + const r = await run("ready"); + const elapsed = Date.now() - started; + + expect(elapsed).toBeLessThan(3_000); + const step = r.steps.at(-1)!; + expect(step.status).toBe("error"); + expect(step.reason).toContain("never answered within the step's 800ms"); + }); + + it("settles on the tree alone when no screenshot can be captured, and says so", async () => { + currentFrame = () => undefined; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 900, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(true); + const step = r.steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("UI tree alone"); + // Attributed to the capture, not to the platform: on a device where + // screenshots normally work this is a per-capture failure, not a property + // of the OS. + expect(step.warning).not.toContain("could not be captured on"); + }); + + // A capture that goes missing is the ABSENCE of visual evidence. Treating it + // as evidence of stillness is how a moving screen used to pass: the round + // that outran the deadline skipped its capture, and the skip stood in for + // "the pixels held". + it("never lets a missing capture stand in for stillness while the pixels move", async () => { + let level = 0; + currentFrame = () => frameAt((level += 60) % 240); + await writeFlow( + "ready", + // A default-shaped step, whose last poll round routinely starts with no + // capture budget left. + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1200 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + expect(r.steps.at(-1)).toMatchObject({ status: "fail" }); + expect(r.steps.at(-1)!.reason).toContain("never held still"); + }); + + // One good read early does not license an app verdict drawn from a window + // that went dark afterwards: a backgrounded app or a dropped instrumentation + // session reads as "unknown", never as "still animating". + it("reports a tree source that dies mid-wait as indeterminate, not as motion", async () => { + let reads = 0; + currentTree = () => { + reads += 1; + if (reads > 1) throw new Error("native-devtools is not connected"); + return screenWith("Home"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 700, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + const step = r.steps.at(-1)!; + // `indeterminate` is scored `error`, which is what stops a QA run rather + // than recording a regression the app never had. + expect(step.status).toBe("error"); + expect(step.reason).toContain("could not read the UI tree"); + expect(step.reason).toContain("foreground"); + }); + + // H1: a screen that settles and then moves again has NOT settled. The + // tree-only verdict used to be a write-once latch, so an early quiet stretch + // licensed a pass drawn from a window that spent the rest of its time + // churning — which is precisely the regression this step exists to catch. + it("does not pass on a screen that settled early and then started moving again", async () => { + currentFrame = () => undefined; // force the tree-only path + let reads = 0; + currentTree = () => { + reads += 1; + return reads <= 4 ? screenWith("Home") : screenWith(`churn ${reads}`); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 2500, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.steps.at(-1)!.status).toBe("fail"); + expect(r.steps.at(-1)!.reason).toContain("never held still"); + }); + + // H2: the same latch let a screen that had gone BLANK by the deadline report + // ready. A blank tree is an observation, not a gap — it clears the verdict. + it("does not pass on a screen that settled early and then went blank", async () => { + currentFrame = () => undefined; + let reads = 0; + currentTree = () => { + reads += 1; + return reads <= 4 ? screenWith("Home") : n({ role: "AXWindow", frame: FULL, children: [] }); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 2500, minStableMs: 0 } +` + ); + expect((await run("ready")).steps.at(-1)!.status).toBe("fail"); + }); + + // H3: bounding the tree read by the remaining budget made the LAST read + // time out on every run, which turned every honest timeout into an + // environment `error` — deleting the hard-fail that justifies this step over + // the soft `await-screen-idle` tool. A round is not started without a budget + // to observe it with, and a read that ran out of step budget is the step + // ending, not the source failing. + it("still fails, rather than erroring, when a slow tree source keeps changing", async () => { + // 300ms per read against a 200ms tail budget: the LAST read runs out of + // step budget, which is the step ending, not the source failing. Earlier + // reads landed and saw a moving screen, so the verdict is theirs. + treeDelayMs = 300; + let tick = 0; + currentTree = () => screenWith(`frame ${tick++}`); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1200, minStableMs: 0 } +` + ); + const r = await run("ready"); + expect(r.steps.at(-1)!.status).toBe("fail"); + expect(r.steps.at(-1)!.reason).toContain("never held still"); + }); + + // M1: the final round used to begin with no budget left, so its capture was + // skipped — and that skip was recorded as "this device cannot be + // screenshotted", warning about a capture path that had worked every round. + it("does not blame the capture when only the step's budget ran out", async () => { + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1000, minStableMs: 900 } +` + ); + const r = await run("ready"); + const step = r.steps.at(-1)!; + // Whatever the verdict, it must not claim the screen could not be captured. + expect(step.warning).toBeUndefined(); + }); + + it("distinguishes a screen that never rendered from one that never settled", async () => { + currentTree = () => n({ role: "AXWindow", frame: FULL, children: [] }); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 500 } +` + ); + const r = await run("ready"); + expect(r.steps.at(-1)!.status).toBe("error"); + expect(r.steps.at(-1)!.reason).toContain("never rendered content"); + }); +}); diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts new file mode 100644 index 000000000..abda26420 --- /dev/null +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -0,0 +1,327 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { PNG } from "pngjs"; +import type { DeviceInfo } from "@argent/registry"; +import type { ActionEnv } from "../../src/tools/flows/flow-actions"; +import { + capturePixelsWithin, + FIRST_PIXEL_CAPTURE_TIMEOUT_MS, + PIXEL_CAPTURE_TIMEOUT_MS, + PIXEL_THRESHOLD, + pixelsDiffer, + pixelCaptureTimeoutMs, + type PixelFrame, +} from "../../src/tools/flows/flow-pixels"; +import { isTvOsSimulator } from "../../src/utils/ios-devices"; +import { captureVegaScreenshotPng } from "../../src/utils/vega-screen"; +import { tvScreenshot } from "../../src/tools/screenshot"; +import { FIRST_FRAME_WAIT_MS } from "../../src/utils/simulator-client"; + +// The capture backends shell out to xcrun / adb / a live simulator-server, so +// stub the four routes and assert which one a device is sent down. +vi.mock("../../src/utils/ios-devices", async (importOriginal) => ({ + ...(await importOriginal()), + isTvOsSimulator: vi.fn(async () => false), +})); +vi.mock("../../src/utils/vega-screen", () => ({ + captureVegaScreenshotPng: vi.fn(), +})); +vi.mock("../../src/tools/screenshot", () => ({ tvScreenshot: vi.fn() })); + +let tmpDir: string; + +beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-pixels-")); + vi.mocked(isTvOsSimulator).mockReset().mockResolvedValue(false); + vi.mocked(captureVegaScreenshotPng).mockReset(); + vi.mocked(tvScreenshot).mockReset(); +}); + +afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); +}); + +/** A solid-color RGBA frame — the unit under test only compares RGB. */ +function solid(width: number, height: number, [r, g, b]: [number, number, number]): PixelFrame { + const data = Buffer.alloc(width * height * 4); + for (let i = 0; i < width * height; i++) { + data[i * 4] = r; + data[i * 4 + 1] = g; + data[i * 4 + 2] = b; + data[i * 4 + 3] = 255; + } + return { width, height, data }; +} + +/** Flip `count` pixels of `base` to `color`, in place, returning it. */ +function withChangedPixels(base: PixelFrame, count: number, color: number): PixelFrame { + for (let i = 0; i < count; i++) { + base.data[i * 4] = color; + base.data[i * 4 + 1] = color; + base.data[i * 4 + 2] = color; + } + return base; +} + +/** Rewrite every pixel's alpha in place, returning the frame. */ +function withAlpha(base: PixelFrame, alpha: number): PixelFrame { + for (let i = 0; i < base.width * base.height; i++) { + base.data[i * 4 + 3] = alpha; + } + return base; +} + +describe("pixelsDiffer", () => { + it("reports no motion for two identical frames", () => { + expect(pixelsDiffer(solid(30, 30, [10, 20, 30]), solid(30, 30, [10, 20, 30]))).toBe(false); + }); + + it("reports motion when the whole frame changes", () => { + expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 30, [255, 255, 255]))).toBe(true); + }); + + it.each<[string, [number, number, number]]>([ + ["red", [255, 0, 0]], + ["green", [0, 255, 0]], + ["blue", [0, 0, 255]], + ])("registers a full-frame change confined to the %s channel as motion", (_channel, color) => { + // Motion that lives in a single channel must clear the per-pixel gate on + // that channel's term alone — the other two contribute zero, so dropping + // any one term from the distance goes blind to exactly one of these. + expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 30, color))).toBe(true); + }); + + it("ignores a change confined to the alpha channel (a screen capture is opaque)", () => { + // Identical RGB, alpha 255 → 0 on every pixel: the docstring promises alpha + // is ignored, so this must read as still. This also pins the byte offsets — + // a comparator that read o+3 (alpha) where it meant o+2 (blue) would count + // every pixel here as changed. + expect( + pixelsDiffer(solid(30, 30, [10, 20, 30]), withAlpha(solid(30, 30, [10, 20, 30]), 0)) + ).toBe(false); + }); + + it("treats a dimension change as motion (a resized window)", () => { + expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 31, [0, 0, 0]))).toBe(true); + }); + + it("ignores a sub-threshold per-pixel color drift (encoder / resample noise)", () => { + // +5 on every channel is well under the per-pixel tolerance, so no pixel + // counts as changed — two captures of a static screen must read as still. + expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [105, 105, 105]))).toBe( + false + ); + }); + + it("brackets the per-pixel tolerance from both sides", () => { + // The tolerance is 0.03 x ~441.7 ~= 13.25. A uniform +7 per channel is a + // distance of ~12.1 (just under) and +8 is ~13.9 (just over), so this pair + // pins the constant tightly: loosening it trips the second expectation and + // tightening it trips the first. + expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [107, 107, 107]))).toBe( + false + ); + expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [108, 108, 108]))).toBe(true); + }); + + it("registers two consecutive samples of a slow uniform cross-fade as motion", () => { + // The case this tolerance sits at its current value for. A spatially + // uniform fade moves every pixel by the same amount, so it clears the + // per-pixel gate on all pixels or on none — the motion fraction is never + // the deciding term. These are two samples one poll apart of a 2s + // indigo-over-white dismissal, a per-channel delta of (16, 23, 12) — + // distance ~30.5. Above the current ~13.25 gate, but BELOW the ~44.2 that + // screenshot-diff's baseline-sized 0.1 imposes, where the settle would + // count zero pixels and report stillness while the overlay was still + // painted and still hit-testing. + expect(pixelsDiffer(solid(30, 30, [165, 128, 193]), solid(30, 30, [149, 105, 181]))).toBe(true); + }); + + it("keeps its tolerance pinned, and stricter than a stored-baseline one", () => { + // Pin the value so the gate — the whole motion oracle — cannot drift + // silently, and so "restore parity with screenshot-diff" (0.1) is a + // deliberate act that trips a test rather than a quiet 3.3x widening of + // the cross-fade blind spot the case above measures. + expect(PIXEL_THRESHOLD).toBe(0.03); + }); + + it("ignores a handful of changed pixels below the motion fraction", () => { + // 900 px, fraction 0.002 → ~1.8 px budget: one changed pixel stays "still" + // (a blinking cursor), three tips it over into motion. + const base = solid(30, 30, [0, 0, 0]); + expect(pixelsDiffer(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 1, 255))).toBe(false); + expect(pixelsDiffer(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 3, 255))).toBe(true); + }); +}); + +/** Write a decodable 2x1 PNG and return its path. */ +async function pngAt(dir: string, name: string): Promise { + const file = path.join(dir, name); + const png = new PNG({ width: 2, height: 1 }); + png.data.set([10, 20, 30, 255, 40, 50, 60, 255]); + await fs.writeFile(file, PNG.sync.write(png)); + return file; +} + +function envFor(device: DeviceInfo, resolveService?: unknown): ActionEnv { + return { device, registry: { resolveService } } as unknown as ActionEnv; +} + +/** The production call path, with a deadline generous enough to stay out of the way. */ +function capture(env: ActionEnv): Promise { + return capturePixelsWithin(env, Date.now() + 30_000, false); +} + +describe("capturePixels routing", () => { + // Every platform argent can screenshot has a route here, and each one is a + // different backend — sending a device down the wrong one silently costs the + // settle its visual half, which then degrades to a tree-only pass. + it("captures and cleans up decodable pixels through the simulator-server backend", async () => { + const file = await pngAt(tmpDir, "native.png"); + const screenshot = vi.fn(async () => ({ path: file, url: `file://${file}` })); + const device: DeviceInfo = { platform: "ios", kind: "simulator", id: "ios-device" }; + const resolveService = vi.fn(async () => ({ transport: { screenshot } })); + + const pixels = await capture(envFor(device, resolveService)); + + expect(pixels).toMatchObject({ width: 2, height: 1 }); + expect([...pixels!.data]).toEqual([10, 20, 30, 255, 40, 50, 60, 255]); + expect(resolveService).toHaveBeenCalledWith(`SimulatorServer:${device.id}`, { device }); + expect(screenshot).toHaveBeenCalledWith({ + rotation: undefined, + scale: 0.25, + signal: undefined, + }); + // The temp PNG is scratch, never an artifact — it must not outlive the decode. + await expect(fs.access(file)).rejects.toThrow(); + }); + + it("routes a tvOS simulator to xcrun, not to the simulator-server it has no backend for", async () => { + vi.mocked(isTvOsSimulator).mockResolvedValue(true); + vi.mocked(tvScreenshot).mockImplementation(async () => pngAt(tmpDir, "tv.png")); + const resolveService = vi.fn(() => { + throw new Error("simulator-server must not be resolved for tvOS"); + }); + // A tvOS simulator's platform is "ios" — only the runtime tells them apart. + const device: DeviceInfo = { platform: "ios", kind: "simulator", id: "tv-udid" }; + + await expect(capture(envFor(device, resolveService))).resolves.toMatchObject({ + width: 2, + height: 1, + }); + expect(tvScreenshot).toHaveBeenCalledWith("tv-udid", 0.25, undefined); + expect(resolveService).not.toHaveBeenCalled(); + }); + + it("routes Vega to the emulator console, and never probes the iOS runtime for it", async () => { + vi.mocked(captureVegaScreenshotPng).mockImplementation(async () => pngAt(tmpDir, "vega.png")); + const resolveService = vi.fn(() => { + throw new Error("simulator-server must not be resolved for vega"); + }); + + await expect( + capture(envFor({ platform: "vega", kind: "vvd", id: "vega-serial" }, resolveService)) + ).resolves.toMatchObject({ width: 2, height: 1 }); + expect(captureVegaScreenshotPng).toHaveBeenCalledWith({ scale: 0.25 }); + expect(isTvOsSimulator).not.toHaveBeenCalled(); + expect(resolveService).not.toHaveBeenCalled(); + }); + + it("leaves Android on the simulator-server route without an iOS runtime probe", async () => { + const file = await pngAt(tmpDir, "android.png"); + const resolveService = vi.fn(async () => ({ + transport: { screenshot: async () => ({ path: file, url: `file://${file}` }) }, + })); + + await expect( + capture( + envFor({ platform: "android", kind: "emulator", id: "emulator-5554" }, resolveService) + ) + ).resolves.toMatchObject({ width: 2, height: 1 }); + expect(isTvOsSimulator).not.toHaveBeenCalled(); + }); + + it.each(["ios", "android", "chromium", "vega"] as const)( + "returns undefined (never throws) on %s when the capture backend fails", + async (platform) => { + // Soft by design: the caller reads undefined as "no visual evidence", so + // a throw escaping here would fail the step on an environment problem. + vi.mocked(captureVegaScreenshotPng).mockRejectedValue(new Error("no vvd")); + const env = envFor({ platform, kind: "unknown", id: "some-device" }); // no resolveService + + expect(await capture(env)).toBeUndefined(); + } + ); + + it("returns undefined when the capture succeeds but the file is not a decodable PNG", async () => { + const file = path.join(tmpDir, "garbage.png"); + await fs.writeFile(file, "not a png"); + const resolveService = vi.fn(async () => ({ + transport: { screenshot: async () => ({ path: file, url: `file://${file}` }) }, + })); + + expect( + await capture(envFor({ platform: "ios", kind: "simulator", id: "x" }, resolveService)) + ).toBeUndefined(); + // Still cleaned up — a failed decode must not leak the file either. + await expect(fs.access(file)).rejects.toThrow(); + }); +}); + +describe("capturePixelsWithin", () => { + const iosDevice: DeviceInfo = { platform: "ios", kind: "simulator", id: "ios-udid" }; + + function envWith(screenshot: () => Promise<{ path: string; url: string }>): ActionEnv { + return envFor( + iosDevice, + vi.fn(async () => ({ transport: { screenshot } })) + ); + } + + it("returns the frame when the capture lands inside the deadline", async () => { + const env = envWith(async () => { + const file = await pngAt(tmpDir, "in-time.png"); + return { path: file, url: `file://${file}` }; + }); + await expect(capturePixelsWithin(env, Date.now() + 5_000, false)).resolves.toMatchObject({ + width: 1 + 1, + height: 1, + }); + }); + + it("gives up rather than overrunning the caller's deadline", async () => { + // A capture that never returns must not hold the settle past the step's + // own timeout — the caller degrades to tree-only, it does not wait. + const env = envWith(() => new Promise(() => {})); + const started = Date.now(); + await expect(capturePixelsWithin(env, started + 120, false)).resolves.toBeUndefined(); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + it("does not attempt a capture once the deadline has already passed", async () => { + const screenshot = vi.fn(async () => { + const file = await pngAt(tmpDir, "too-late.png"); + return { path: file, url: `file://${file}` }; + }); + await expect(capturePixelsWithin(envWith(screenshot), Date.now() - 1, false)).resolves.toBe( + undefined + ); + expect(screenshot).not.toHaveBeenCalled(); + }); + + it("allows the first capture the cold-stream wait and later ones the warm bound", () => { + // The simulator-server serves captures from a live frame stream, so the + // first read after it starts can spend the whole first-frame window; every + // later one is answered from a stream that is already producing. + expect(pixelCaptureTimeoutMs(iosDevice, true)).toBe(FIRST_PIXEL_CAPTURE_TIMEOUT_MS); + expect(FIRST_PIXEL_CAPTURE_TIMEOUT_MS).toBeGreaterThan(FIRST_FRAME_WAIT_MS); + expect(pixelCaptureTimeoutMs(iosDevice, false)).toBe(PIXEL_CAPTURE_TIMEOUT_MS); + // Chromium answers from CDP with no stream to warm up, so its first + // capture gets no extra grace. + expect( + pixelCaptureTimeoutMs({ platform: "chromium", kind: "app", id: "chromium-cdp-9222" }, true) + ).toBe(PIXEL_CAPTURE_TIMEOUT_MS); + }); +}); From 1eb9be5ef3eedf8545f72da8dbaf4d73d21b000f Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 5 Aug 2026 13:31:40 +0200 Subject: [PATCH 62/98] fix(flow): make the idle condition warn instead of failing the run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Readiness is not an acceptance criterion. A screen that never stops moving is usually the app working as built — a video, a shimmer, a carousel — so failing the run turns a healthy app into a red test, and the flow's verdict belongs to the identity and outcome checks around the gate rather than to the gate itself. Measured on real devices, the two signals are also not equally sensitive across platforms. The Android tree carries live text and live bounds: a running stopwatch moved its fingerprint on every read, and a fling moved 41 lines with genuine mid-flight positions. The iOS tree saw none of the equivalent motion — a Bluesky feed scrolled for 2.9s with the tree byte-identical across 41 consecutive intervals while the pixels moved in every one. Hard-failing on a signal that different per platform gives one flow file two verdicts: a ticking timestamp or a relative "5s ago" fails a gate on Android that iOS cannot even observe. Both an autoplaying video on iOS and a running stopwatch on Android hard-failed a default-shaped step before this change. So the timeout now reports a warning on a passing step, and the step keeps the part that was always the point: it returns the moment the screen is still, so the following tap resolves its target against a screen that has stopped instead of racing a transition still in flight. That is also why it stays worth placing after every navigation. The warning names what to look at rather than only reporting that the wait gave up, because the benign reading and the bug look identical from here: a stuck spinner is a screen that never finished loading, not a screen that is animating by design. An unreadable window is untouched — it remains `indeterminate`, scored `error`, because the check could not run at all, which is not a verdict about the app and is now idle's only non-passing outcome. --- .../skills/skills/argent-create-flow/SKILL.md | 6 +- .../src/tools/flows/flow-actions.ts | 33 +++++++-- .../tool-server/src/tools/flows/flow-run.ts | 26 ++++--- .../tool-server/src/tools/flows/flow-utils.ts | 9 ++- .../test/flows/flow-idle-run.test.ts | 74 +++++++++++++------ 5 files changed, 98 insertions(+), 50 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 65c53f69a..d9fa2b405 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -85,9 +85,11 @@ This condition-as-key form is the only spelling. `await` also accepts an optiona For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. -**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels, then hard-fails if it never does (that hard failure is what makes it safe to persist, unlike the soft `await-screen-idle` tool). Options: `minStableMs` (how long stillness must hold, default 250 — it has to be shorter than the timeout, or the gate could never pass) and `timeout` (default 7500). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. +**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250 — it has to be shorter than the timeout, or the gate could never pass) and `timeout` (default 7500). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. -It is **not** a screen check: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. Do not sprinkle it after every step — each one costs a settle; add one where a step actually proved flaky. If its captures never produced a comparable pair, it still passes on the tree alone but reports a warning saying so, because it then only proved the hierarchy held still. +It **never fails a run.** A screen that never settles spends the timeout, then passes with a ⚠ warning: readiness is not an acceptance criterion, and plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text). Read that warning rather than stepping over it — a stuck spinner looks exactly the same, and it means the screen never finished loading. Only an unreadable tree stops the run, as an `error`. + +It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. ### `type` and `scroll-to` diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 1436276c7..ec26c0685 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1150,6 +1150,16 @@ async function waitForCondition( // the screen stopped moving? It is deliberately NOT an identity check — a // dropped tap leaves the source screen perfectly idle — so it belongs next to // the element check that says WHICH screen, never instead of it. +// +// It never fails a run. Readiness is not an acceptance criterion: the flow's +// verdict belongs to the identity and outcome checks around it, and a screen +// that keeps moving is usually a property of the app rather than a regression +// — a video, a shimmer, a carousel. On Android it is also routine: that tree +// carries live text, so a ticking timer or a relative timestamp moves it on +// every read, where the iOS tree cannot see either. Hard-failing on a signal +// that sensitive, and that different per platform, turns one flow file into +// two verdicts. So a screen that never settles is reported as a WARNING on a +// passing step, naming what to look at. /** * `idle` poll cadence, matching `await-screen-idle`'s own. The timeout and hold @@ -1201,8 +1211,14 @@ type TreeReadOutcome = "value" | "error" | "timeout"; * a screen the tree calls ready. * * This is `await-screen-idle`'s question asked against the tree the directives - * actually resolve against, and — unlike that tool — it FAILS when the screen - * never settles, which is what makes it safe to persist in a flow. + * actually resolve against. It returns early the moment the screen is still, + * which is the point: the following tap resolves its target against a screen + * that has stopped, instead of racing a transition still in flight. + * + * A screen that never settles spends the whole timeout and then passes with a + * warning (see the section note above). Only an unreadable window is a hard + * stop, and it is `indeterminate` — the check could not run, which is not a + * verdict about the app. * * Every verdict is drawn from the LAST round that observed something, never * from a latch remembering that the screen was once still: a screen that @@ -1406,12 +1422,13 @@ async function waitForIdle( }; } return { - ok: false, - reason: - `the screen never held still for ${minStableMs}ms within ${timeoutMs}ms — the UI tree or ` + - `the pixels kept changing, so it is still animating, or something on it never stops moving ` + - `(a looping animation, a video, a carousel that keeps advancing). Gate on the element you ` + - `actually need instead of on stillness.`, + ok: true, + warning: + `the screen never held still for ${minStableMs}ms within ${timeoutMs}ms, so this step went ` + + `ahead without waiting it out. Either something on it never stops (a video, a looping ` + + `animation, a carousel, live-updating text) or the screen never finished loading — a stuck ` + + `spinner looks like both. Look at what is moving, and make sure the next action is gated on ` + + `a stable element rather than on stillness.`, }; } diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 6e19cdbf5..17b8cc635 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -190,10 +190,11 @@ export interface StepReport { reason?: string; /** * The step passed, but the WAY it passed weakens it as proof. Rendered as a - * "⚠" suffix by the MCP client. Raised by an `await: { idle: true }` whose - * captures never produced a comparable pair, so it proved stillness on the - * UI tree alone and never saw the presentation-layer motion it exists to - * catch. + * "⚠" suffix by the MCP client. Raised by `await: { idle: true }`, either + * because the screen never settled at all — it waits, then goes ahead — or + * because its captures never produced a comparable pair, leaving stillness + * proved on the UI tree alone without the presentation-layer motion the + * pixel half exists to catch. */ warning?: string; /** Underlying tool id for `tool` steps. */ @@ -2126,14 +2127,15 @@ async function execLeafStep( // A run cancelled mid-directive is a skip (matching the pre-step guard // and `wait`), never a step failure — the app did nothing wrong. if (r.aborted) return { ...base, status: "skip", reason: r.reason }; - // An INDETERMINATE `idle` outcome is not a verdict about the app: the - // wait could not run at all (an unreadable or degraded tree, a screen - // nobody managed to observe). Reporting it as `fail` makes CI read an - // environment problem as a regression and a QA author reset a pass - // streak over it. `error` keeps the run non-ok while saying plainly - // that the app was never judged. Scoped to `idle`, whose whole verdict - // rests on being able to observe the screen; the selector conditions - // keep their existing `fail` mapping. + // `indeterminate` is `idle`'s only non-passing outcome: a screen that + // merely kept moving passes with a warning, so what is left here is a + // wait that could not run at all (an unreadable or degraded tree, a + // screen nobody managed to observe). Scoring that `fail` would make CI + // read an environment problem as a regression and a QA author reset a + // pass streak over it. `error` keeps the run non-ok while saying + // plainly that the app was never judged. Scoped to `idle`, whose whole + // verdict rests on being able to observe the screen; the selector + // conditions keep their existing `fail` mapping. if (!r.ok && r.indeterminate && step.kind === "idle") { return { ...base, status: "error", reason: r.reason }; } diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index d35d3fa31..f85fbeaaf 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -1692,8 +1692,8 @@ const IDLE_CONDITION = "idle"; /** * `idle`'s defaults, spelled here rather than beside the runner because the - * parser needs the timeout one: a hold that cannot fit inside the wait is a - * gate that fails on every run, and this file rejects unsatisfiable gates. + * parser needs the timeout one: a hold that cannot fit inside the wait can + * never be satisfied, and this file rejects unsatisfiable gates. */ export const IDLE_DEFAULT_TIMEOUT_MS = 7500; export const IDLE_DEFAULT_MIN_STABLE_MS = 250; @@ -1762,8 +1762,9 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") if ("timeout" in raw) step.timeout = parseAwaitTimeout(entry, raw.timeout); if (raw.minStableMs !== undefined) { // A hold as long as the whole wait can never be observed within it, so the - // step would fail on every run — and fail blaming the app, which is the one - // thing such a failure is not evidence about. Caught here, deviceless. + // step would spend its full timeout and warn on every run, however still + // the screen was — a gate that cannot pass, reported as if the app were at + // fault. Caught here, deviceless. const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; step.minStableMs = parseBoundedMs( entry, diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index 6ddf89925..bb85f92bb 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -101,9 +101,12 @@ afterEach(async () => { vi.clearAllMocks(); }); -// `await: { idle: true }` is the readiness check. Its whole reason to exist is -// that it FAILS — the `await-screen-idle` tool reports `settled: false` softly, -// which cannot carry a regression verdict on an unattended replay. +// `await: { idle: true }` is the readiness check: it returns the moment the +// screen is still, so the next tap resolves against a screen that has stopped. +// It never fails a run — a screen that never settles passes with a warning, +// because readiness is not an acceptance criterion and a screen that keeps +// moving (a video, a shimmer, live-updating text on Android) is usually a +// property of the app. Only an unreadable window is a hard stop, as an error. describe("await: { idle }", () => { it("passes once both the tree and the pixels hold still", async () => { await writeFlow( @@ -140,7 +143,7 @@ steps: expect(reads).toBeGreaterThanOrEqual(3); }); - it("fails when the tree never stops changing", async () => { + it("warns, and does not fail, when the tree never stops changing", async () => { let tick = 0; currentTree = () => screenWith(`frame ${tick++}`); await writeFlow( @@ -151,16 +154,40 @@ steps: ` ); const r = await run("ready"); - expect(r.ok).toBe(false); + expect(r.ok).toBe(true); const step = r.steps.at(-1)!; - expect(step.status).toBe("fail"); - expect(step.reason).toContain("never held still"); + expect(step.status).toBe("pass"); + expect(step.warning).toContain("never held still"); + // The warning has to say what to do next, not merely that it gave up. + expect(step.warning).toContain("stable element"); + }); + + // The point of warning instead of failing: a screen that never stops moving + // is usually the app working as built (a video, a shimmer, a carousel, or — + // on Android, whose tree carries live text — a ticking timestamp). The run + // has to reach the checks that actually carry its verdict. + it("lets the rest of the flow run when the screen never settles", async () => { + let tick = 0; + currentTree = () => screenWith(`frame ${tick++}`); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 600, minStableMs: 300 } + - echo: reached +` + ); + const r = await run("ready"); + expect(r.ok).toBe(true); + expect(r.failed).toBe(0); + expect(r.errored).toBe(0); + expect(r.steps.at(-1)).toMatchObject({ kind: "echo", status: "pass", message: "reached" }); }); // The reason this check reads pixels at all: an iOS push or modal dismissal // commits its hierarchy up front and then animates a layer for a few hundred // milliseconds. The tree is perfectly still the whole time. - it("fails when the pixels keep moving under a motionless tree", async () => { + it("warns when the pixels keep moving under a motionless tree", async () => { let level = 0; currentFrame = () => frameAt((level += 60) % 240); await writeFlow( @@ -171,11 +198,10 @@ steps: ` ); const r = await run("ready"); - expect(r.ok).toBe(false); + expect(r.ok).toBe(true); const step = r.steps.at(-1)!; - expect(step.status).toBe("fail"); - expect(step.reason).toContain("never held still"); - expect(step.reason).toContain("pixels"); + expect(step.status).toBe("pass"); + expect(step.warning).toContain("never held still"); }); // Sub-threshold drift is encoder noise, not motion — treating it as motion @@ -215,8 +241,8 @@ steps: ` ); const r = await run("ready"); - expect(r.ok).toBe(false); - expect(r.steps.at(-1)!.reason).toContain("never held still"); + expect(r.steps.at(-1)!.status).toBe("pass"); + expect(r.steps.at(-1)!.warning).toContain("never held still"); }); // `timeout:` is the author's answer to "how long may this take", so it has to @@ -280,9 +306,10 @@ steps: ` ); const r = await run("ready"); - expect(r.ok).toBe(false); - expect(r.steps.at(-1)).toMatchObject({ status: "fail" }); - expect(r.steps.at(-1)!.reason).toContain("never held still"); + // Passing is fine; claiming the screen SETTLED is not — the warning must + // still report the motion, not the tree-only settle. + expect(r.steps.at(-1)!.warning).toContain("never held still"); + expect(r.steps.at(-1)!.warning).not.toContain("UI tree alone"); }); // One good read early does not license an app verdict drawn from a window @@ -331,8 +358,7 @@ steps: ` ); const r = await run("ready"); - expect(r.steps.at(-1)!.status).toBe("fail"); - expect(r.steps.at(-1)!.reason).toContain("never held still"); + expect(r.steps.at(-1)!.warning).toContain("never held still"); }); // H2: the same latch let a screen that had gone BLANK by the deadline report @@ -351,7 +377,7 @@ steps: - await: { idle: true, timeout: 2500, minStableMs: 0 } ` ); - expect((await run("ready")).steps.at(-1)!.status).toBe("fail"); + expect((await run("ready")).steps.at(-1)!.warning).toContain("never held still"); }); // H3: bounding the tree read by the remaining budget made the LAST read @@ -360,7 +386,7 @@ steps: // the soft `await-screen-idle` tool. A round is not started without a budget // to observe it with, and a read that ran out of step budget is the step // ending, not the source failing. - it("still fails, rather than erroring, when a slow tree source keeps changing", async () => { + it("still warns, rather than erroring, when a slow tree source keeps changing", async () => { // 300ms per read against a 200ms tail budget: the LAST read runs out of // step budget, which is the step ending, not the source failing. Earlier // reads landed and saw a moving screen, so the verdict is theirs. @@ -375,8 +401,8 @@ steps: ` ); const r = await run("ready"); - expect(r.steps.at(-1)!.status).toBe("fail"); - expect(r.steps.at(-1)!.reason).toContain("never held still"); + expect(r.steps.at(-1)!.status).toBe("pass"); + expect(r.steps.at(-1)!.warning).toContain("never held still"); }); // M1: the final round used to begin with no budget left, so its capture was @@ -393,7 +419,7 @@ steps: const r = await run("ready"); const step = r.steps.at(-1)!; // Whatever the verdict, it must not claim the screen could not be captured. - expect(step.warning).toBeUndefined(); + expect(step.warning ?? "").not.toContain("no screenshot"); }); it("distinguishes a screen that never rendered from one that never settled", async () => { From 5a59d31d4b99096a36c6c38f8354dd4f9551da44 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Wed, 5 Aug 2026 13:44:54 +0200 Subject: [PATCH 63/98] docs(skills): name the step warning field rather than its rendered glyph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent reads the step report, where the outcome is a `warning` on a passing step; the "⚠" is only how the MCP client and the CLI render that field. Naming the glyph pointed the reader at the presentation instead of the thing it can actually check. --- packages/skills/skills/argent-create-flow/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index d9fa2b405..97e3f2fd3 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -87,7 +87,7 @@ For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-el **`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250 — it has to be shorter than the timeout, or the gate could never pass) and `timeout` (default 7500). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. -It **never fails a run.** A screen that never settles spends the timeout, then passes with a ⚠ warning: readiness is not an acceptance criterion, and plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text). Read that warning rather than stepping over it — a stuck spinner looks exactly the same, and it means the screen never finished loading. Only an unreadable tree stops the run, as an `error`. +It **never fails a run.** A screen that never settles spends the timeout, then passes with a `warning` on the step: readiness is not an acceptance criterion, and plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text). Read that warning rather than stepping over it — a stuck spinner looks exactly the same, and it means the screen never finished loading. Only an unreadable tree stops the run, as an `error`. It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. From c7a14d4a05f28110bb702b2ce23347135adb117c Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:04:21 +0200 Subject: [PATCH 64/98] fix(flow): report the small, permanent motion a spinner makes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readiness check compared changed pixels against 0.2% of the whole screen, which no spinner reaches: a stock one covers 0.03-0.15% of a phone display, and it does not move the UI tree either, since it spins in a layer whose box never changes. Both halves of the check therefore agreed a still-loading screen was at rest and passed it clean, with none of the warning both doc surfaces send the author to read. Split the comparison in two. Above the motion fraction the screen is moving, as before. Below it but above a noise floor the change is localized — too small to be the screen moving, too large to be a capture that is not bit-exact — and the settle still completes, because holding a flow for the full timeout over a caret is worse than saying so, but it now says so. Measured on a Chromium app pointed at a page whose only motion is a 40px CSS spinner: 3 runs of 3 warn where 3 of 3 passed silently before. A static page stays clean 4 runs of 4, and a full-screen pulse still reports that the screen never held still. --- .../src/tools/flows/flow-actions.ts | 35 +++++++- .../src/tools/flows/flow-pixels.ts | 61 +++++++++---- .../test/flows/flow-idle-run.test.ts | 58 +++++++++++++ .../test/flows/flow-pixels.test.ts | 85 +++++++++++++++---- 4 files changed, 202 insertions(+), 37 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index ec26c0685..8d9e3fd00 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -23,7 +23,7 @@ import { settleWithin, sleepOrAbort } from "../../utils/timing"; import { invokeSubTool } from "../../utils/sub-invoke"; import { bindDeviceArgs } from "./flow-device"; import { fetchFlowTree } from "./flow-tree"; -import { capturePixelsWithin, pixelsDiffer, type PixelFrame } from "./flow-pixels"; +import { capturePixelsWithin, comparePixels, type PixelFrame } from "./flow-pixels"; import { buildAxisCandidate, decomposePinch, @@ -1194,6 +1194,20 @@ const MIN_STILL_INTERVALS = 2; */ const MIN_ROUND_BUDGET_MS = IDLE_POLL_MS; +/** + * The screen settled, but something small on it never stopped. A spinner is the + * case that matters: it is far too small to move the screen (a stock one covers + * ~0.1% of a phone display) and it does not move the tree either, since it + * spins in a layer without its box ever changing — so both halves of the check + * agree the screen is at rest while it is still loading. This is the only place + * that difference is visible, so it is said outright. + */ +const LOCALIZED_MOTION_WARNING = + `the screen settled, but a small part of it kept changing the whole time — a spinner, a ` + + `caret, a progress dot. If it is a loading spinner then the screen had not finished loading, ` + + `and stillness cannot tell those apart: look at what is moving, and gate the next action on ` + + `the element the loading produces rather than on this settle.`; + /** How the last tree read ended. Only `value` licenses a verdict about the app. */ type TreeReadOutcome = "value" | "error" | "timeout"; @@ -1242,6 +1256,10 @@ async function waitForIdle( let previousFrame: PixelFrame | undefined; let bothSince = 0; let stillIntervals = 0; + // Small, persistent motion seen across the intervals that produced the + // current hold. Cleared with the hold, so it only ever describes the settle + // actually being reported. + let localizedMotionDuringHold = false; let readsSucceeded = 0; // Definitely assigned: the loop below always completes at least one round, @@ -1336,22 +1354,31 @@ async function waitForIdle( // stillness. Letting it stand in for "the pixels held" is what turned a // screen that never stopped moving into a pass. let pixelsHeld = false; + let localizedThisInterval = false; if (frame === undefined) { captureFailed = true; } else if (previousFrame !== undefined) { - if (pixelsDiffer(previousFrame, frame)) pixelsEverMoved = true; - else pixelsHeld = true; + const change = comparePixels(previousFrame, frame); + if (change === "moving") pixelsEverMoved = true; + else { + pixelsHeld = true; + localizedThisInterval = change === "localized"; + } } previousFrame = frame; if (treeHeld && pixelsHeld) { stillIntervals += 1; + if (localizedThisInterval) localizedMotionDuringHold = true; if (stillIntervals >= MIN_STILL_INTERVALS && now - bothSince >= minStableMs) { - return { ok: true }; + return localizedMotionDuringHold + ? { ok: true, warning: LOCALIZED_MOTION_WARNING } + : { ok: true }; } } else { bothSince = now; stillIntervals = 0; + localizedMotionDuringHold = false; } } } diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index dc368b9bb..e3bbd0293 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -19,6 +19,9 @@ export interface PixelFrame { data: Buffer; } +/** What {@link comparePixels} saw between two captures. */ +export type PixelChange = "still" | "localized" | "moving"; + // Hard downscale: motion detection only needs to see a large region moving, // and a quarter-scale frame decodes ~16x faster. (Chromium without `sharp` // ignores the scale and returns full-res — the comparison is scale-agnostic.) @@ -48,11 +51,13 @@ export const PIXEL_THRESHOLD = 0.03; const MAX_RGB_DISTANCE_SQUARED = 255 * 255 * 3; const PIXEL_THRESHOLD_SQUARED = PIXEL_THRESHOLD * PIXEL_THRESHOLD * MAX_RGB_DISTANCE_SQUARED; -// Captures match when fewer than this fraction of pixels changed — counting -// only pixels that individually clear the per-pixel gate above. That sits -// above the noise of a blinking cursor and catches localized motion and moving -// edges at any speed, plus screen-filling changes whose per-interval rate -// clears the gate. +// The screen is MOVING when at least this fraction of pixels changed — +// counting only pixels that individually clear the per-pixel gate above. It is +// sized for motion worth waiting out: a transition, a scroll, a fade, a +// carousel. Anything smaller is reported separately (see below) rather than +// resetting the settle, because a caret or a spinner never stops, and holding +// a flow for the full timeout over one is worse than telling the author about +// it. // // This fraction is NOT what bounds spatially uniform change (a fade, dim, tint // or scrim): such a change clears the per-pixel gate on either 100% of pixels @@ -61,6 +66,22 @@ const PIXEL_THRESHOLD_SQUARED = PIXEL_THRESHOLD * PIXEL_THRESHOLD * MAX_RGB_DIST // the gate alone; loosening this fraction does not widen that blind spot. const MOTION_FRACTION = 0.002; +// Below MOTION_FRACTION but at or above this one, the change is LOCALIZED: too +// small to be the screen moving, too large to be capture noise. A stock 40pt +// spinner measures 0.03-0.15% of a phone screen and a text caret about 0.01%, +// both under MOTION_FRACTION — which is how a still-loading screen used to +// report as settled with nothing said about it. +// +// Both fractions are of frame AREA, which makes them resolution-independent: +// an object of a fixed on-screen size covers the same fraction of the frame +// whatever the capture scale, so the same numbers hold on a 158k-pixel Pixel +// capture and a full-resolution desktop one. +// +// The floor is not zero because a capture pair is not guaranteed byte-identical +// on every backend; it is two orders of magnitude below the smallest spinner +// and one below a caret, which is the widest margin that still sees them. +const LOCALIZED_MOTION_FRACTION = 0.00005; + // `httpScreenshot` may spend its full first-frame wait before it even returns // a file path. Leave a separate completion margin for reading, decoding, and // removing that PNG. Warm captures get the tighter bound below. @@ -164,18 +185,26 @@ export async function capturePixelsWithin( } /** - * Did the screen move between two captures? Different dimensions count as - * motion; otherwise the changed-pixel fraction is compared against - * {@link MOTION_FRACTION}. Alpha is ignored — a screen capture is opaque. + * How much of the screen changed between two captures. + * + * - `moving` — the screen is in motion and has not settled. + * - `localized` — something small never stopped: a spinner, a caret, a + * progress dot. Not enough to call the screen unsettled, but the caller + * reports it, because a spinner means the screen never finished loading and + * nothing else here can see the difference. + * - `still`. + * + * Alpha is ignored — a screen capture is opaque. * - * The dimension branch covers a resized window (Chromium). It is NOT how a - * device rotation is caught: the Android capture keeps its portrait shape - * across one, so rotation registers through content change like anything else. + * Different dimensions count as motion. That branch covers a resized window + * (Chromium). It is NOT how a device rotation is caught: the Android capture + * keeps its portrait shape across one, so rotation registers through content + * change like anything else. */ -export function pixelsDiffer(a: PixelFrame, b: PixelFrame): boolean { - if (a.width !== b.width || a.height !== b.height) return true; +export function comparePixels(a: PixelFrame, b: PixelFrame): PixelChange { + if (a.width !== b.width || a.height !== b.height) return "moving"; const total = a.width * a.height; - if (total === 0) return false; + if (total === 0) return "still"; const limit = Math.min(a.data.length, b.data.length); let changed = 0; for (let o = 0; o + 2 < limit; o += 4) { @@ -184,5 +213,7 @@ export function pixelsDiffer(a: PixelFrame, b: PixelFrame): boolean { const db = a.data[o + 2] - b.data[o + 2]; if (dr * dr + dg * dg + db * db > PIXEL_THRESHOLD_SQUARED) changed++; } - return changed / total > MOTION_FRACTION; + const fraction = changed / total; + if (fraction > MOTION_FRACTION) return "moving"; + return fraction >= LOCALIZED_MOTION_FRACTION ? "localized" : "still"; } diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index bb85f92bb..f9783d10e 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -64,6 +64,23 @@ function frameAt(level: number): PixelFrame { return { width: 10, height: 10, data }; } +/** + * A capture-sized frame (180k pixels, the order a real one has at + * CAPTURE_SCALE) that is still apart from `movingPixels` of it. Sized so a + * spinner's share of a screen can be expressed at all: on the 10x10 frames + * above, one pixel is already 1% of the screen. + */ +function frameWithMovingPixels(movingPixels: number, level: number): PixelFrame { + const [width, height] = [300, 600]; + const data = Buffer.alloc(width * height * 4, 255); + for (let i = 0; i < movingPixels; i++) { + data[i * 4] = level; + data[i * 4 + 1] = level; + data[i * 4 + 2] = level; + } + return { width, height, data }; +} + function mockRegistry(): Registry { return { invokeTool: vi.fn(async (id: string) => { @@ -204,6 +221,47 @@ steps: expect(step.warning).toContain("never held still"); }); + // A spinner is the reason this warning exists. It is far too small to move + // the screen (a stock one covers ~0.1% of a phone display) and it does not + // move the tree either — it spins in a layer whose box never changes — so + // both halves of the check call the screen settled while it is still + // loading. The step still passes, because waiting out a caret or a spinner + // that never stops is worse than saying so, but it must SAY so. + it("warns when the screen settles with something small still moving on it", async () => { + let tick = 0; + // 40 of 180_000 pixels (0.022%) alternating: an order of magnitude under + // the motion fraction, an order above the noise floor. + currentFrame = () => frameWithMovingPixels(40, tick++ % 2 === 0 ? 0 : 40); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } + - echo: reached +` + ); + const r = await run("ready"); + expect(r.ok).toBe(true); + const step = r.steps.at(-2)!; + expect(step).toMatchObject({ kind: "idle", status: "pass" }); + expect(step.warning).toContain("small part of it kept changing"); + expect(step.warning).toContain("spinner"); + // The warning is about how the settle was reached, not a refusal to settle. + expect(step.warning).not.toContain("never held still"); + }); + + it("says nothing about small motion when the screen is genuinely still", async () => { + currentFrame = () => frameWithMovingPixels(0, 0); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + expect((await run("ready")).steps.at(-1)!.warning).toBeUndefined(); + }); + // Sub-threshold drift is encoder noise, not motion — treating it as motion // would make the check unsatisfiable on a screen that is genuinely at rest. it("tolerates capture noise below the motion threshold", async () => { diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index abda26420..6f50d8b9d 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -10,7 +10,7 @@ import { FIRST_PIXEL_CAPTURE_TIMEOUT_MS, PIXEL_CAPTURE_TIMEOUT_MS, PIXEL_THRESHOLD, - pixelsDiffer, + comparePixels, pixelCaptureTimeoutMs, type PixelFrame, } from "../../src/tools/flows/flow-pixels"; @@ -73,13 +73,13 @@ function withAlpha(base: PixelFrame, alpha: number): PixelFrame { return base; } -describe("pixelsDiffer", () => { +describe("comparePixels", () => { it("reports no motion for two identical frames", () => { - expect(pixelsDiffer(solid(30, 30, [10, 20, 30]), solid(30, 30, [10, 20, 30]))).toBe(false); + expect(comparePixels(solid(30, 30, [10, 20, 30]), solid(30, 30, [10, 20, 30]))).toBe("still"); }); it("reports motion when the whole frame changes", () => { - expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 30, [255, 255, 255]))).toBe(true); + expect(comparePixels(solid(30, 30, [0, 0, 0]), solid(30, 30, [255, 255, 255]))).toBe("moving"); }); it.each<[string, [number, number, number]]>([ @@ -90,7 +90,7 @@ describe("pixelsDiffer", () => { // Motion that lives in a single channel must clear the per-pixel gate on // that channel's term alone — the other two contribute zero, so dropping // any one term from the distance goes blind to exactly one of these. - expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 30, color))).toBe(true); + expect(comparePixels(solid(30, 30, [0, 0, 0]), solid(30, 30, color))).toBe("moving"); }); it("ignores a change confined to the alpha channel (a screen capture is opaque)", () => { @@ -99,19 +99,19 @@ describe("pixelsDiffer", () => { // a comparator that read o+3 (alpha) where it meant o+2 (blue) would count // every pixel here as changed. expect( - pixelsDiffer(solid(30, 30, [10, 20, 30]), withAlpha(solid(30, 30, [10, 20, 30]), 0)) - ).toBe(false); + comparePixels(solid(30, 30, [10, 20, 30]), withAlpha(solid(30, 30, [10, 20, 30]), 0)) + ).toBe("still"); }); it("treats a dimension change as motion (a resized window)", () => { - expect(pixelsDiffer(solid(30, 30, [0, 0, 0]), solid(30, 31, [0, 0, 0]))).toBe(true); + expect(comparePixels(solid(30, 30, [0, 0, 0]), solid(30, 31, [0, 0, 0]))).toBe("moving"); }); it("ignores a sub-threshold per-pixel color drift (encoder / resample noise)", () => { // +5 on every channel is well under the per-pixel tolerance, so no pixel // counts as changed — two captures of a static screen must read as still. - expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [105, 105, 105]))).toBe( - false + expect(comparePixels(solid(30, 30, [100, 100, 100]), solid(30, 30, [105, 105, 105]))).toBe( + "still" ); }); @@ -120,10 +120,12 @@ describe("pixelsDiffer", () => { // distance of ~12.1 (just under) and +8 is ~13.9 (just over), so this pair // pins the constant tightly: loosening it trips the second expectation and // tightening it trips the first. - expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [107, 107, 107]))).toBe( - false + expect(comparePixels(solid(30, 30, [100, 100, 100]), solid(30, 30, [107, 107, 107]))).toBe( + "still" + ); + expect(comparePixels(solid(30, 30, [100, 100, 100]), solid(30, 30, [108, 108, 108]))).toBe( + "moving" ); - expect(pixelsDiffer(solid(30, 30, [100, 100, 100]), solid(30, 30, [108, 108, 108]))).toBe(true); }); it("registers two consecutive samples of a slow uniform cross-fade as motion", () => { @@ -136,7 +138,9 @@ describe("pixelsDiffer", () => { // screenshot-diff's baseline-sized 0.1 imposes, where the settle would // count zero pixels and report stillness while the overlay was still // painted and still hit-testing. - expect(pixelsDiffer(solid(30, 30, [165, 128, 193]), solid(30, 30, [149, 105, 181]))).toBe(true); + expect(comparePixels(solid(30, 30, [165, 128, 193]), solid(30, 30, [149, 105, 181]))).toBe( + "moving" + ); }); it("keeps its tolerance pinned, and stricter than a stored-baseline one", () => { @@ -148,11 +152,56 @@ describe("pixelsDiffer", () => { }); it("ignores a handful of changed pixels below the motion fraction", () => { - // 900 px, fraction 0.002 → ~1.8 px budget: one changed pixel stays "still" - // (a blinking cursor), three tips it over into motion. + // 900 px, fraction 0.002 → ~1.8 px budget: one changed pixel is not the + // screen moving, three is. const base = solid(30, 30, [0, 0, 0]); - expect(pixelsDiffer(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 1, 255))).toBe(false); - expect(pixelsDiffer(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 3, 255))).toBe(true); + expect(comparePixels(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 1, 255))).not.toBe( + "moving" + ); + expect(comparePixels(base, withChangedPixels(solid(30, 30, [0, 0, 0]), 3, 255))).toBe("moving"); + }); + + // Every case above runs on a 30x30 frame, where the motion budget is 1.8 + // pixels and anything visible trips it. A real capture at CAPTURE_SCALE is + // 158k-198k pixels, and at that size the small-but-permanent movers a + // readiness check exists to notice — a spinner above all — sit two orders of + // magnitude below the same fraction. Measured on real captures taken at the + // scale the check uses: a stock spinner moved 66 pixels of an iPhone 16 Pro + // frame (302x656) and 57 of a Pixel 5 one (270x585). + describe("at a real capture size", () => { + const IPHONE = [302, 656] as const; // 198k px: 396 px of motion budget + const PIXEL5 = [270, 585] as const; // 158k px: 316 px of motion budget + + function changed([w, h]: readonly [number, number], count: number): [PixelFrame, PixelFrame] { + return [ + solid(w, h, [255, 255, 255]), + withChangedPixels(solid(w, h, [255, 255, 255]), count, 0), + ]; + } + + it.each([ + ["iPhone 16 Pro", IPHONE, 66], + ["Pixel 5", PIXEL5, 57], + ] as const)("sees a spinner on a %s frame", (_device, size, spinnerPixels) => { + // Under the motion fraction, so the screen is not called unsettled — but + // never "still", which is what let a still-loading screen report ready + // with nothing said about it. + expect(comparePixels(...changed(size, spinnerPixels))).toBe("localized"); + }); + + it("still calls a real transition motion at that size", () => { + // 1% of the frame — a sheet edge, a scrolling row, a moving cursor bar. + expect(comparePixels(...changed(IPHONE, Math.round(IPHONE[0] * IPHONE[1] * 0.01)))).toBe( + "moving" + ); + }); + + it("keeps a few stray pixels below even the localized floor", () => { + // The floor exists so a backend that is not bit-exact between two + // captures of a static screen does not warn on every settle. Three + // pixels of 198k is an order of magnitude under a caret. + expect(comparePixels(...changed(IPHONE, 3))).toBe("still"); + }); }); }); From 0d98a25c4691f83ddf34437df6687c82f653db76 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:07:49 +0200 Subject: [PATCH 65/98] fix(flow): stop calling a wedged app an animating screen A tree source that fails has a dedicated hard error. One that HANGS is a different outcome internally, and once any earlier read had succeeded it was never routed there: the step passed and told the author the screen never stopped moving, listing a video, a carousel and live-updating text, on a screen frozen solid. The two are told apart by how much budget the abandoned read had. The last read of a step routinely times out with a couple of hundred milliseconds to its name, and that is the step ending; one abandoned with seconds in hand is a source that wedged. The read still gets the whole remaining budget, since a tree read on a busy screen genuinely takes seconds. Verified against a Chromium app whose renderer enters an infinite loop 400ms into the idle step: 2 runs of 2 now report the source stopped answering, where 4 of 4 used to pass blaming animation. --- .../src/tools/flows/flow-actions.ts | 38 +++++++++++++++++++ .../test/flows/flow-idle-run.test.ts | 32 ++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 8d9e3fd00..5244dca8f 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1208,6 +1208,21 @@ const LOCALIZED_MOTION_WARNING = `and stillness cannot tell those apart: look at what is moving, and gate the next action on ` + `the element the loading produces rather than on this settle.`; +/** + * How long a tree read may go unanswered before the SOURCE is what stopped + * working, rather than the step running out of time. + * + * A read is still given the whole remaining budget — a tree read on a busy + * screen genuinely takes seconds, and Android's `uiautomator dump` allows + * itself twenty — so this is not a bound on the read. It is the size of the + * gap that separates the two reasons a read fails to come back: the last read + * of a step routinely times out with a couple of hundred milliseconds to its + * name, and that is the step ending. One abandoned with seconds of budget in + * hand is a source that has wedged, and no verdict about the app may be drawn + * from a window nobody could see through. + */ +const HUNG_TREE_READ_MS = 2_000; + /** How the last tree read ended. Only `value` licenses a verdict about the app. */ type TreeReadOutcome = "value" | "error" | "timeout"; @@ -1266,6 +1281,7 @@ async function waitForIdle( // and every arm of that round sets it. let lastRead!: TreeReadOutcome; let treeErrorMessage: string | undefined; + let treeReadHung = false; let sawContent = false; let pixelsEverMoved = false; let captureFailed = false; @@ -1300,6 +1316,11 @@ async function waitForIdle( // tree state nor stands in for one, so the hold state is left as it was // and the bottom decides what, if anything, it means. lastRead = "timeout"; + // ...except for one thing it does say. A read abandoned with seconds of + // budget left is a source that has wedged, not a step that ran out of + // time, and the difference decides whether the bottom may describe the + // app at all. + if (roundBudget >= HUNG_TREE_READ_MS) treeReadHung = true; } else if (read.type === "error") { // A tree-source blip mid-animation is expected; keep polling. Only its // presence on the LAST read is reportable. @@ -1316,6 +1337,8 @@ async function waitForIdle( lastRead = "value"; readsSucceeded += 1; treeErrorMessage = undefined; + // It answered, so whatever wedged it has cleared. + treeReadHung = false; const tree = read.value.tree; if (tree.children.length === 0) { // Blank or still loading — never "settled", and it resets both holds. @@ -1423,6 +1446,21 @@ async function waitForIdle( if (lastRead === "error" && treeErrorMessage !== undefined) { return unreadable(treeErrorMessage); } + // The same window going dark the other way: the source answered, then stopped + // answering with seconds of budget still in hand. A failing read has a + // dedicated error above, but a HANGING one used to fall through to the + // motion warning and tell the author a frozen screen was a carousel. + if (lastRead === "timeout" && treeReadHung) { + return { + ok: false, + indeterminate: true, + reason: + `the UI tree source answered and then stopped: a read given at least ` + + `${HUNG_TREE_READ_MS}ms never came back, so the screen could not be observed for the ` + + `rest of the wait — check the app is still in the foreground and responding (a wedged ` + + `app reads the same as a backgrounded one)`, + }; + } // Readable throughout and never once carrying content: the screen rendered // nothing, which is not the same claim as "it never stopped moving". if (!sawContent) { diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index f9783d10e..f8ead9f2f 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -327,6 +327,38 @@ steps: expect(step.reason).toContain("never answered within the step's 800ms"); }); + // The sibling of the case above, and the one that used to slip through: a + // source that FAILS is caught by the unreadable-tree error, but one that + // HANGS is a different outcome internally, and after any earlier read had + // succeeded it fell through to the motion warning — telling the author that + // a screen frozen by a wedged renderer was a video or a carousel. + it("reports a tree source that wedges mid-wait as indeterminate, not as motion", async () => { + let reads = 0; + currentTree = () => { + reads += 1; + // Answer the first read, then wedge for longer than the step can wait. + treeDelayMs = 60_000; + return screenWith("Home"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 2500, minStableMs: 0 } + - echo: reached +` + ); + const r = await run("ready"); + expect(reads).toBe(1); + expect(r.ok).toBe(false); + const step = r.steps.find((s) => s.kind === "idle")!; + expect(step.status).toBe("error"); + expect(step.reason).toContain("answered and then stopped"); + expect(step.reason).toContain("foreground"); + // And it must not be dressed up as a verdict about what was on screen. + expect(step.reason).not.toContain("never held still"); + }); + it("settles on the tree alone when no screenshot can be captured, and says so", async () => { currentFrame = () => undefined; await writeFlow( From 7cb9afd8320cf6ee902b04c2643e3bbb0452f38d Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:13:35 +0200 Subject: [PATCH 66/98] fix(flow): reject an idle wait that cannot contain its own settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A settle costs 600ms before any hold is counted: three reads spanning two 200ms polls, plus the budget the closing round needs to start with. Under that, no screen can produce a clean settle however still it is, and the step reported one of two claims about an app that had done nothing — that it never stopped moving, or that it could not be screenshotted. Which one it picked depended on where the budget ran out, so one file gave different verdicts run to run. The parser had a guard for exactly this, but it compared `minStableMs` against `timeout` and only when `minStableMs` was written out — so `timeout: 100` was accepted while the identical `timeout: 100, minStableMs: 250` was rejected. It now checks the effective hold plus what the settle itself costs, and the poll cadence moved next to the defaults it is computed from. Slow sources can still leave a legal step short of reads, so the runner says that too, instead of blaming the app for a screen it barely looked at. Verified on a Chromium app: `timeout: 400`, which warned "never held still" 4 runs of 4 on a completely static page, is now refused at parse with the number to raise it to. --- .../src/tools/flows/flow-actions.ts | 41 +++++---- .../tool-server/src/tools/flows/flow-utils.ts | 88 ++++++++++++++----- .../test/flows/flow-idle-condition.test.ts | 37 +++++--- .../test/flows/flow-idle-run.test.ts | 51 ++++++++--- 4 files changed, 153 insertions(+), 64 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 5244dca8f..59a1299e4 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -43,6 +43,8 @@ import { describeTextExpectation, IDLE_DEFAULT_MIN_STABLE_MS, IDLE_DEFAULT_TIMEOUT_MS, + IDLE_MIN_STILL_INTERVALS, + IDLE_POLL_MS, SELECTOR_RELATIONS, type FlowSelector, type FlowStep, @@ -1162,25 +1164,11 @@ async function waitForCondition( // passing step, naming what to look at. /** - * `idle` poll cadence, matching `await-screen-idle`'s own. The timeout and hold - * defaults live in flow-utils beside the parser, which needs the timeout to - * reject a hold that could never fit inside the wait. + * The cadence, the interval count and the defaults all live in flow-utils + * beside the parser, which needs every one of them to reject a wait that could + * never contain the settle it asks for. Aliased here for readability. */ -const IDLE_POLL_MS = 200; - -/** - * How many consecutive intervals must read as still before the screen is - * called settled. Two, not one, because a single agreeing pair of captures is - * not evidence of stillness: any animation that reverses — a cross-fade, a - * pulse, a bounce — has a turning point, and two samples straddling it come - * back identical while the screen is very much moving. Observed on a 3s - * white/indigo cross-fade, where a default-shaped step passed on roughly one - * run in three. A second agreeing interval needs a third sample, which the - * same phase symmetry cannot supply unless the animation's period happens to - * match the poll — so the aliasing that survives one comparison does not - * survive two. - */ -const MIN_STILL_INTERVALS = 2; +const MIN_STILL_INTERVALS = IDLE_MIN_STILL_INTERVALS; /** * The smallest budget a poll round is allowed to start with. A round begun @@ -1470,6 +1458,23 @@ async function waitForIdle( reason: `the UI tree stayed empty for ${timeoutMs}ms — the screen never rendered content`, }; } + // Too few reads to have judged anything. A settle needs three of them + // spanning two intervals, so a step that got fewer has no evidence either + // way — and both verdicts below would be claims about an app that was never + // observed for long enough to make one. The parser rejects a `timeout:` too + // short to fit a settle, so what reaches here is a source slow enough to eat + // the wait, which is worth saying rather than dressing up as motion. + if (readsSucceeded <= MIN_STILL_INTERVALS) { + return { + ok: true, + warning: + `the screen was read ${readsSucceeded} time${readsSucceeded === 1 ? "" : "s"} in ` + + `${timeoutMs}ms, and a settle takes ${MIN_STILL_INTERVALS + 1} reads spanning ` + + `${MIN_STILL_INTERVALS} ${IDLE_POLL_MS}ms polls — so this step ended without ever being ` + + `able to tell whether the screen was moving. Raise its \`timeout:\`, and gate the next ` + + `action on a stable element rather than on stillness.`, + }; + } // The tree was settled as of the last read and no pair of captures ever // showed motion, yet the combined hold never completed. With captures // arriving this is unreachable: a pair either agrees — and the tree was diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index f85fbeaaf..18fb32f56 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -1691,13 +1691,48 @@ function parseWaitFields(raw: unknown, kind: "await" | "assert" | "when"): WaitF const IDLE_CONDITION = "idle"; /** - * `idle`'s defaults, spelled here rather than beside the runner because the - * parser needs the timeout one: a hold that cannot fit inside the wait can - * never be satisfied, and this file rejects unsatisfiable gates. + * `idle`'s defaults and cadence, spelled here rather than beside the runner + * because the parser needs all of them: a wait that cannot contain the settle + * it asks for can never be satisfied, and this file rejects unsatisfiable + * gates. The runner imports them back. */ export const IDLE_DEFAULT_TIMEOUT_MS = 7500; export const IDLE_DEFAULT_MIN_STABLE_MS = 250; +/** `idle` poll cadence, matching `await-screen-idle`'s own. */ +export const IDLE_POLL_MS = 200; + +/** + * How many consecutive intervals must read as still before the screen is + * called settled. Two, not one, because a single agreeing pair of captures is + * not evidence of stillness: any animation that reverses — a cross-fade, a + * pulse, a bounce — has a turning point, and two samples straddling it come + * back identical while the screen is very much moving. Observed on a 3s + * white/indigo cross-fade, where a default-shaped step passed on roughly one + * run in three. A second agreeing interval needs a third sample, which the + * same phase symmetry cannot supply unless the animation's period happens to + * match the poll — so the aliasing that survives one comparison does not + * survive two. + */ +export const IDLE_MIN_STILL_INTERVALS = 2; + +/** + * What a settle costs before any hold is counted: the intervals it is measured + * over, plus the budget the round that closes it needs to be allowed to start + * ({@link IDLE_POLL_MS} again — see the runner's MIN_ROUND_BUDGET_MS). A + * `timeout:` under this cannot produce a clean settle however still the screen + * is, so the step would report on a screen it never had the chance to judge. + */ +export const IDLE_SETTLE_OVERHEAD_MS = (IDLE_MIN_STILL_INTERVALS + 1) * IDLE_POLL_MS; + +/** + * Absolute ceiling on the hold, so an obviously wrong unit (seconds, or a + * pasted timestamp) is rejected as a number rather than silently becoming a + * gate no run can pass. The relationship that actually matters is with + * `timeout`, checked separately. + */ +const IDLE_MAX_MIN_STABLE_MS = 600_000; + /** * The `timeout` sibling key an `await` may carry, spelled once for both the * selector conditions and `idle`. @@ -1718,18 +1753,9 @@ function parseAwaitTimeout(entry: unknown, value: unknown): number { } /** Bounded non-negative integer option, in milliseconds. */ -function parseBoundedMs( - entry: unknown, - value: unknown, - where: string, - max: number, - why?: string -): number { +function parseBoundedMs(entry: unknown, value: unknown, where: string, max: number): number { if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > max) { - badEntry( - entry, - `${where} needs an integer between 0 and ${max} (milliseconds)${why ? ` — ${why}` : ""}` - ); + badEntry(entry, `${where} needs an integer between 0 and ${max} (milliseconds)`); } return value as number; } @@ -1761,17 +1787,37 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") const step: Extract = { kind: "idle" }; if ("timeout" in raw) step.timeout = parseAwaitTimeout(entry, raw.timeout); if (raw.minStableMs !== undefined) { - // A hold as long as the whole wait can never be observed within it, so the - // step would spend its full timeout and warn on every run, however still - // the screen was — a gate that cannot pass, reported as if the app were at - // fault. Caught here, deviceless. - const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; step.minStableMs = parseBoundedMs( entry, raw.minStableMs, "idle.minStableMs", - timeoutMs - 1, - `the hold has to fit inside the ${step.timeout === undefined ? "default " : ""}${timeoutMs}ms timeout` + IDLE_MAX_MIN_STABLE_MS + ); + } + + // A wait that cannot contain the settle it asks for is a gate that never + // passes, however still the screen is — and it does not fail quietly: the + // step spends its whole timeout and then reports either that the screen + // never stopped moving or that it could not be screenshotted, both of them + // claims about an app that did nothing. Which one it picks depends on where + // the budget ran out, so the same file yields different verdicts run to run. + // Caught here, deviceless. + // + // Checked against the EFFECTIVE hold, not just a written-out one: the + // default is what most steps run with, so leaving it out was the way to get + // an unsatisfiable step past the parser (`timeout: 100` was accepted while + // the identical `timeout: 100, minStableMs: 250` was rejected). + const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; + const minStableMs = step.minStableMs ?? IDLE_DEFAULT_MIN_STABLE_MS; + const needed = minStableMs + IDLE_SETTLE_OVERHEAD_MS; + if (timeoutMs < needed) { + badEntry( + entry, + `idle needs a timeout of at least ${needed}ms to hold still for ` + + `${step.minStableMs === undefined ? `the default ` : ``}${minStableMs}ms: a settle is ` + + `${IDLE_MIN_STILL_INTERVALS + 1} reads spanning ${IDLE_MIN_STILL_INTERVALS} ` + + `${IDLE_POLL_MS}ms polls, and the wait has to contain them as well as the hold. Raise ` + + `\`timeout\`${step.minStableMs === undefined ? "" : " or lower `minStableMs`"}` ); } return step; diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index 91aa8085a..bfbc7726d 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -53,26 +53,37 @@ describe("await { idle }", () => { ); }); - // A hold that cannot fit inside the wait is a gate that fails on every run — - // and fails blaming the app, which is the one thing it is not evidence - // about. Caught at parse, deviceless, rather than against a live screen. - it("rejects a hold that could never fit inside the timeout", () => { + // A wait that cannot contain the settle it asks for is a gate that fails on + // every run — and fails blaming the app, which is the one thing it is not + // evidence about. Caught at parse, deviceless, rather than against a live + // screen. The settle costs 600ms before any hold is counted: three reads + // spanning two 200ms polls, plus the budget the closing round has to start + // with. + it("rejects a wait that could never contain the settle it asks for", () => { expect(() => parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 1000 }\n`) - ).toThrow( - /idle.minStableMs needs an integer between 0 and 499 .* fit inside the 500ms timeout/ + ).toThrow(/idle needs a timeout of at least 1600ms to hold still for 1000ms/); + // With no explicit hold the DEFAULT is what has to fit — the spelling that + // slipped through, since leaving `minStableMs` out was the way past the + // check that only looked at a written-out one. + expect(() => parseSteps(` - await: { idle: true, timeout: 100 }\n`)).toThrow( + /idle needs a timeout of at least 850ms to hold still for the default 250ms/ ); - // With no explicit timeout the default is what it has to fit inside, and - // the message says which number it is measuring against. + // Which is the same step as writing the default out, so it is rejected the + // same way. + expect(() => parseSteps(` - await: { idle: true, timeout: 100, minStableMs: 250 }\n`)).toThrow( + /idle needs a timeout of at least 850ms/ + ); + // With no explicit timeout the default is what the hold has to fit inside. expect(() => parseSteps(` - await: { idle: true, minStableMs: 9000 }\n`)).toThrow( - /fit inside the default 7500ms timeout/ + /idle needs a timeout of at least 9600ms/ ); // The boundary itself is legal on both sides. - expect(parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 499 }\n`)).toEqual([ - { kind: "idle", timeout: 500, minStableMs: 499 }, + expect(parseSteps(` - await: { idle: true, timeout: 900, minStableMs: 300 }\n`)).toEqual([ + { kind: "idle", timeout: 900, minStableMs: 300 }, ]); - expect(() => parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 500 }\n`)).toThrow( - /idle.minStableMs/ + expect(() => parseSteps(` - await: { idle: true, timeout: 899, minStableMs: 300 }\n`)).toThrow( + /idle needs a timeout of at least 900ms/ ); }); diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index f8ead9f2f..19ccda56f 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -167,7 +167,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 600, minStableMs: 300 } + - await: { idle: true, timeout: 900, minStableMs: 300 } ` ); const r = await run("ready"); @@ -190,7 +190,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 600, minStableMs: 300 } + - await: { idle: true, timeout: 900, minStableMs: 300 } - echo: reached ` ); @@ -359,6 +359,28 @@ steps: expect(step.reason).not.toContain("never held still"); }); + // A settle is three reads spanning two intervals. A step that got fewer has + // no evidence either way, and both of the verdicts it used to reach for — + // "the screen never stopped moving", "no screenshot could be read" — are + // claims about an app nobody observed for long enough to make one. + it("says it ran out of looks rather than judging a screen it barely read", async () => { + treeDelayMs = 700; // two of these do not fit in the wait + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1200 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("read 1 time in 1200ms"); + expect(step.warning).toContain("`timeout:`"); + // The screen was static the whole time; it must not be described as moving. + expect(step.warning).not.toContain("never held still"); + expect(step.warning).not.toContain("no pair of screenshots"); + }); + it("settles on the tree alone when no screenshot can be captured, and says so", async () => { currentFrame = () => undefined; await writeFlow( @@ -479,7 +501,9 @@ steps: it("still warns, rather than erroring, when a slow tree source keeps changing", async () => { // 300ms per read against a 200ms tail budget: the LAST read runs out of // step budget, which is the step ending, not the source failing. Earlier - // reads landed and saw a moving screen, so the verdict is theirs. + // reads landed and saw a moving screen, so the verdict is theirs — and the + // budget the last read was given is what separates this from a source that + // wedged (see the case above). treeDelayMs = 300; let tick = 0; currentTree = () => screenWith(`frame ${tick++}`); @@ -487,7 +511,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1200, minStableMs: 0 } + - await: { idle: true, timeout: 2000, minStableMs: 0 } ` ); const r = await run("ready"); @@ -495,21 +519,24 @@ steps: expect(r.steps.at(-1)!.warning).toContain("never held still"); }); - // M1: the final round used to begin with no budget left, so its capture was - // skipped — and that skip was recorded as "this device cannot be - // screenshotted", warning about a capture path that had worked every round. - it("does not blame the capture when only the step's budget ran out", async () => { + // A step whose budget is spent mid-settle must not invent a verdict out of + // what it did not manage to observe. It has three ways to do that — blaming + // the capture, blaming motion, or claiming a settle — so this pins the + // outcome exactly rather than ruling one wording out. + it("reaches a real settle rather than a verdict about the budget that ran out", async () => { await writeFlow( "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1000, minStableMs: 900 } + - await: { idle: true, timeout: 1600, minStableMs: 900 } ` ); const r = await run("ready"); const step = r.steps.at(-1)!; - // Whatever the verdict, it must not claim the screen could not be captured. - expect(step.warning ?? "").not.toContain("no screenshot"); + // A still screen and a working capture path: the only honest outcome is a + // clean settle, reached before the hold could exhaust the wait. + expect(step.status).toBe("pass"); + expect(step.warning).toBeUndefined(); }); it("distinguishes a screen that never rendered from one that never settled", async () => { @@ -518,7 +545,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 500 } + - await: { idle: true, timeout: 900 } ` ); const r = await run("ready"); From ce97918da90bbe368736751dfef0e4246055aa1e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:15:04 +0200 Subject: [PATCH 67/98] fix(flow): let a flow past a screen whose tree reads back empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An `idle` step on a screen that rendered no tree content returned an error, which set the run not-ok and skipped every step after it — including the element check that would have named what was actually wrong. The tree read back fine; it was simply empty, which is an observation about the app and not a window that could not be read, and readiness is never this step's to fail a run over. It is also not always a fault: a canvas, a video surface or a splash image renders no accessible content by design. It now warns and goes on, saying which of the two it might be. Verified on a Chromium app pointed at a page with no accessible content: the step passes with the warning and the following step runs, 2 runs of 2, where both used to report `errored: 1` and skip it. --- .../tool-server/src/tools/flows/flow-actions.ts | 16 +++++++++++++--- .../tool-server/test/flows/flow-idle-run.test.ts | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 59a1299e4..5960ee43b 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1451,11 +1451,21 @@ async function waitForIdle( } // Readable throughout and never once carrying content: the screen rendered // nothing, which is not the same claim as "it never stopped moving". + // + // A warning, not a stop. The tree read back fine — this is an observation + // about the app, and readiness is never this step's to fail a run over. It + // also is not always the app's fault: a screen legitimately renders no + // accessible content (a bare canvas, a video surface, a splash image), and + // stopping the flow there took every later step with it, including the + // element check that would have said what was actually wrong. if (!sawContent) { return { - ok: false, - indeterminate: true, - reason: `the UI tree stayed empty for ${timeoutMs}ms — the screen never rendered content`, + ok: true, + warning: + `the UI tree stayed empty for ${timeoutMs}ms — the screen never rendered content, so ` + + `there was nothing to settle. If the screen is meant to render accessible content, this ` + + `is where it did not; if it is a canvas or a video surface, it has none to read. Gate ` + + `the next action on an element check either way.`, }; } // Too few reads to have judged anything. A settle needs three of them diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index 19ccda56f..cd0700d16 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -539,6 +539,11 @@ steps: expect(step.warning).toBeUndefined(); }); + // A tree that reads back fine and is empty is an observation about the app, + // not a window that could not be read — so it warns like any other screen + // that did not settle, and the flow goes on to the checks that carry its + // verdict. Stopping there used to take every later step with it, including + // the element check that would have named what was actually wrong. it("distinguishes a screen that never rendered from one that never settled", async () => { currentTree = () => n({ role: "AXWindow", frame: FULL, children: [] }); await writeFlow( @@ -546,10 +551,16 @@ steps: `executionPrerequisite: "" steps: - await: { idle: true, timeout: 900 } + - echo: reached ` ); const r = await run("ready"); - expect(r.steps.at(-1)!.status).toBe("error"); - expect(r.steps.at(-1)!.reason).toContain("never rendered content"); + expect(r.ok).toBe(true); + expect(r.errored).toBe(0); + const step = r.steps.find((s) => s.kind === "idle")!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("never rendered content"); + expect(step.warning).not.toContain("never held still"); + expect(r.steps.at(-1)).toMatchObject({ kind: "echo", status: "pass" }); }); }); From 2808f8269c3817e2490b03c9be33ab68771eeab2 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:18:11 +0200 Subject: [PATCH 68/98] fix(flow): tell a `when: { idle: true }` guard what it should have been `assert: { idle: true }` gets a tailored explanation of why there is no assert form. The guard spelling of the same mistake got the generic "when needs exactly one condition key (exists, visible, hidden, text, platform)", which never mentions the key the author actually wrote and leaves them to infer that it is missing from the list on purpose. It now says the same thing the assert form does: stillness is a wait, a guard asks what is on the screen now, put `await: { idle: true }` before the block. --- packages/tool-server/src/tools/flows/flow-utils.ts | 11 +++++++++++ .../test/flows/flow-idle-condition.test.ts | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 18fb32f56..df5bca16e 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2176,6 +2176,17 @@ function parseWhenCondition(raw: unknown): WhenCondition { badEntry({ when: raw }, `when needs exactly one condition key (${conditionKeys})`); } const b = raw as Record; + // A guard asks what is on the screen NOW, so "has it stopped moving yet" is + // not a question it can ask. Say that outright, the way the assert form does, + // rather than listing the keys the author could have written and leaving them + // to infer that the one they did write is not among them. + if (IDLE_CONDITION in b) { + badEntry( + { when: raw }, + "when has no idle form — stillness is a wait, and a guard asks what is on the screen now. " + + "Put `await: { idle: true }` before the block instead" + ); + } const present = [...WAIT_CONDITIONS, "platform"].filter((c) => c in b); if (present.length !== 1) { badEntry({ when: raw }, `when needs exactly one condition key (${conditionKeys})`); diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index bfbc7726d..538857de6 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -137,8 +137,12 @@ describe("condition families are mutually exclusive", () => { ); expect(stray).not.toThrow(/idle/); - // And `idle` itself is not a guard. - expect(guard("{ idle: true }")).toThrow(/when needs exactly one condition key/); + // And `idle` itself is not a guard — which is said outright, the way the + // assert form is, rather than left to be inferred from a list it is + // missing from. + expect(guard("{ idle: true }")).toThrow( + /when has no idle form .* Put `await: \{ idle: true \}` before the block/ + ); }); it("leaves the selector conditions untouched", () => { From 2884d7c8e2e4bb1c0ff6aedef073f4b6b29462e4 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:18:43 +0200 Subject: [PATCH 69/98] fix(flow): name the spelling when `idle` is written as a step of its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `- idle: true` at the top level of a step returned "unrecognized step kind" with no hint, and the near-miss hint could not find one either — `idle` is not a step kind to be close to. It is also the one near-miss the docs actively produce: every other condition is written with a selector beside it, so `await:` comes along for free, while this one reads like a directive. Say what it should have been. --- packages/tool-server/src/tools/flows/flow-utils.ts | 8 ++++++++ .../tool-server/test/flows/flow-idle-condition.test.ts | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index df5bca16e..58709d70e 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -2400,6 +2400,14 @@ function fromYamlStep(raw: YamlStep, whenDepth = 0): FlowStep { } const kinds = STEP_DIRECTIVE_KEYS.filter((k) => k in entry); if (kinds.length === 0) { + // `idle` is a condition, not a step kind, and it is the one near-miss the + // docs actively produce: every other condition is written with a selector + // beside it, so `await:` comes along for free, while this one reads like a + // directive of its own. Spell the answer rather than reporting that a step + // kind nobody wrote is unrecognized. + if (IDLE_CONDITION in entry) { + badEntry(raw, `idle is a condition, not a step kind — write it as \`await: { idle: true }\``); + } const hint = Object.keys(entry) .map((k) => closestKey(k, STEP_DIRECTIVE_KEYS)) .find((h) => h !== null); diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index 538857de6..6a4f795a9 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -87,6 +87,15 @@ describe("await { idle }", () => { ); }); + // The one near-miss the docs actively produce: every other condition is + // written with a selector beside it, so `await:` comes along for free, while + // this one reads like a directive of its own. + it("names the spelling when `idle` is written as a step of its own", () => { + expect(() => parseSteps(` - idle: true\n`)).toThrow( + /idle is a condition, not a step kind — write it as `await: \{ idle: true \}`/ + ); + }); + it("rejects a non-positive timeout", () => { expect(() => parseSteps(` - await: { idle: true, timeout: 0 }\n`)).toThrow(/await.timeout/); expect(() => parseSteps(` - await: { idle: true, timeout: "soon" }\n`)).toThrow( From e93cc42466e452301106db21b4371d1e6e74060e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:19:15 +0200 Subject: [PATCH 70/98] test(flow): pin that a fractional idle timeout gets a real answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `timeout: 0.5` was turned into the upper bound for the hold, which asked the author for "an integer between 0 and -0.5" — a range no value satisfies. The two are now checked against each other as a sum, which has an answer for any timeout; this holds that, and the impossible range, in place. --- .../test/flows/flow-idle-condition.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index 6a4f795a9..dbccfa8a0 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -96,6 +96,18 @@ describe("await { idle }", () => { ); }); + // The hold is validated as an integer while the timeout is not, so a + // fractional timeout used to be turned into a bound for the hold and asked + // for "an integer between 0 and -0.5". Nothing is derived from it now: the + // two are checked against each other as a sum, which has an answer whatever + // the timeout is. + it("rejects a timeout too small to settle in without inventing a range", () => { + const parse = (): FlowStep[] => + parseSteps(` - await: { idle: true, timeout: 0.5, minStableMs: 0 }\n`); + expect(parse).toThrow(/idle needs a timeout of at least 600ms/); + expect(parse).not.toThrow(/between 0 and -/); + }); + it("rejects a non-positive timeout", () => { expect(() => parseSteps(` - await: { idle: true, timeout: 0 }\n`)).toThrow(/await.timeout/); expect(() => parseSteps(` - await: { idle: true, timeout: "soon" }\n`)).toThrow( From b9b5b586c3fca2ffe92bc9b826c764825898f3f3 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:21:40 +0200 Subject: [PATCH 71/98] fix(flow): stop one missed capture from blinding two intervals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capture that went missing was also stored as the previous frame, so the round after it had nothing to compare against either — one slow capture cost the settle two intervals, and a backend that was merely intermittently slow ended up reported as one where no screenshot could be read at all. Keep the last frame that actually arrived and compare across the gap instead: the same question, asked over a longer interval. The degraded warning no longer overclaims either — it says the screen could not be screenshotted on enough polls to compare a pair, which is what it knows. Pinned by count rather than wording: with one capture missing out of a still screen's run, the settle now takes five captures where it took six. --- .../src/tools/flows/flow-actions.ts | 29 ++++++++++++------- .../test/flows/flow-idle-run.test.ts | 27 +++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 5960ee43b..ca951fdd2 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1368,15 +1368,23 @@ async function waitForIdle( let localizedThisInterval = false; if (frame === undefined) { captureFailed = true; - } else if (previousFrame !== undefined) { - const change = comparePixels(previousFrame, frame); - if (change === "moving") pixelsEverMoved = true; - else { - pixelsHeld = true; - localizedThisInterval = change === "localized"; + } else { + if (previousFrame !== undefined) { + const change = comparePixels(previousFrame, frame); + if (change === "moving") pixelsEverMoved = true; + else { + pixelsHeld = true; + localizedThisInterval = change === "localized"; + } } + // Only a frame that arrived replaces the reference. A missed capture + // used to overwrite it with `undefined`, which cost the NEXT round its + // comparison too — one slow capture blinded two intervals, so a + // backend that is merely intermittently slow ended up reported as one + // that could not be screenshotted at all. Holding the last good frame + // asks the same question across the gap, over a longer interval. + previousFrame = frame; } - previousFrame = frame; if (treeHeld && pixelsHeld) { stillIntervals += 1; @@ -1496,9 +1504,10 @@ async function waitForIdle( return { ok: true, warning: - `settled on the UI tree alone — no screenshot of this screen could be read, so animation ` + - `that moves pixels without moving nodes (a push, a fade, a dismissing modal) was not ` + - `waited out. Follow this with the element check the next step actually needs.`, + `settled on the UI tree alone — this screen could not be screenshotted on enough polls ` + + `to compare a pair of them, so animation that moves pixels without moving nodes (a push, ` + + `a fade, a dismissing modal) was not waited out. Follow this with the element check the ` + + `next step actually needs.`, }; } return { diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index cd0700d16..fc6f413e8 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -401,6 +401,33 @@ steps: expect(step.warning).not.toContain("could not be captured on"); }); + // A capture that goes missing used to cost the settle TWO intervals, not + // one: the missing frame was also stored as the previous frame, so the next + // round had nothing to compare against either. Holding the last good frame + // asks the same question across the gap. The witness is the number of + // captures the settle takes — rounds run in lockstep, so it is exact. + it("loses only the interval a capture went missing in, not the one after it", async () => { + let captures = 0; + currentFrame = () => { + captures += 1; + return captures === 3 ? undefined : frameAt(120); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + // 1,2 hold; 3 is missing; 4 and 5 hold across the gap and settle. Six would + // mean round 4 was blinded by round 3's absence. + expect(captures).toBe(5); + // And one missed capture out of five is not "no screenshot could be read". + expect(step.warning).toBeUndefined(); + }); + // A capture that goes missing is the ABSENCE of visual evidence. Treating it // as evidence of stillness is how a moving screen used to pass: the round // that outran the deadline skipped its capture, and the skip stood in for From f6601eb7516736bdf96f1628eaa41439d5ae57d2 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:26:36 +0200 Subject: [PATCH 72/98] perf(flow): take Chromium's settle capture at the scale it asks for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The settle asks every capture route for a quarter-scale frame. Chromium's answered through the screenshot path that resizes with `sharp` — an optional dependency nothing in this repo installs — so it silently returned the full-resolution PNG, and a settle decoded a 1800x1226 image into an 8.8MB buffer twice a second on the shared tool-server's only thread. `Page.captureScreenshot`'s own `clip.scale` is applied while rasterizing, so the small frame is the only one that ever exists. Measured on a 900x613 viewport at dpr 2: 1800x1226 and 21-43ms of blocking decode per poll became 450x307 and 2-3ms. It also drops the temp file this route wrote and deleted on every poll. The behaviours the smaller frame has to preserve were re-run against the live app: a static page settles clean, a spinner still reports its localized motion, and a full-screen pulse still reports that the screen never held still. --- .../src/tools/flows/flow-pixels.ts | 81 ++++++++++++++----- .../test/flows/flow-pixels.test.ts | 34 ++++++++ 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index e3bbd0293..b5af25f94 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -22,9 +22,10 @@ export interface PixelFrame { /** What {@link comparePixels} saw between two captures. */ export type PixelChange = "still" | "localized" | "moving"; -// Hard downscale: motion detection only needs to see a large region moving, -// and a quarter-scale frame decodes ~16x faster. (Chromium without `sharp` -// ignores the scale and returns full-res — the comparison is scale-agnostic.) +// Hard downscale: motion detection only needs to see a region of the screen +// change, and a quarter-scale frame decodes ~16x faster. Every route honours +// it, including Chromium — which reaches the scale a different way, see +// captureChromiumPng. const CAPTURE_SCALE = 0.25; // Per-pixel RGB tolerance, owned by this comparison and deliberately NOT @@ -108,22 +109,54 @@ export function pixelCaptureTimeoutMs(device: ActionEnv["device"], firstCapture: : PIXEL_CAPTURE_TIMEOUT_MS; } +/** + * Chromium's downscale, taken from the compositor rather than from `sharp`. + * + * The `screenshot` tool's Chromium route resizes the captured PNG with + * `sharp`, which is an optional dependency nothing in this repo installs — so + * asking it for a quarter-scale frame returned a full-resolution one, and a + * settle decoded a 2400x2558 PNG into a 23MB buffer twice a second, blocking + * the shared tool-server's event loop for ~79ms of every 200ms round. + * + * `Page.captureScreenshot`'s own `clip.scale` is applied while rasterizing, so + * the small frame is the only one that ever exists: no resize step, no + * dependency, and nothing to decode but the quarter-scale image. `clip` is + * measured in CSS pixels but its scale composes with the page's device scale + * factor, so passing CAPTURE_SCALE straight through lands on a quarter of the + * frame this route used to return — the same reduction every other route + * applies. Measured on a 900x613 viewport at dpr 2: 1800x1226 and ~25ms of + * blocking decode per poll became 450x307 and ~3ms. + * + * The viewport is the cached one rather than a fresh read: refreshing it costs + * a `Runtime.evaluate` per poll, and that call fails outright on a renderer + * mid-navigation — which is exactly when this check runs. A window resized + * mid-step therefore clips against the previous size for the rest of it, and + * registers as content change like anything else that moves. + */ +async function captureChromiumPng(env: ActionEnv): Promise { + const ref = chromiumCdpRef(env.device); + const api = (await env.registry.resolveService(ref.urn, ref.options)) as ChromiumCdpApi; + const { width, height } = api.getViewport(); + const shot = (await api.cdp.send("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: false, + clip: { x: 0, y: 0, width, height, scale: CAPTURE_SCALE }, + })) as { data?: string }; + if (!shot.data) throw new Error("Page.captureScreenshot returned no data"); + return Buffer.from(shot.data, "base64"); +} + /** * Capture one downscaled screenshot to a temp file, routed exactly as the - * `screenshot` tool routes it: Chromium over CDP, tvOS and Vega through their - * own shells (neither has a simulator-server backend), everything else through - * the simulator-server both iOS and Android share. + * `screenshot` tool routes it: tvOS and Vega through their own shells (neither + * has a simulator-server backend), everything else through the simulator-server + * both iOS and Android share. Chromium does not appear here — it answers with + * bytes, never a file (see captureChromiumPng). * * The `screenshot` tool itself is deliberately not reused: it registers every * capture as an artifact, and a settle takes tens of them per step. */ async function captureFile(env: ActionEnv): Promise { - if (env.device.platform === "chromium") { - const ref = chromiumCdpRef(env.device); - const api = (await env.registry.resolveService(ref.urn, ref.options)) as ChromiumCdpApi; - const { path } = await api.captureScreenshot({ scale: CAPTURE_SCALE }); - return path; - } if (env.device.platform === "vega") { return captureVegaScreenshotPng({ scale: CAPTURE_SCALE }); } @@ -146,6 +179,21 @@ async function captureFile(env: ActionEnv): Promise { return path; } +/** + * One capture as PNG bytes. Every route but Chromium's writes a temp file, + * which is scratch and never an artifact — it is removed as soon as it has + * been read, whether or not the read worked. + */ +async function capturePng(env: ActionEnv): Promise { + if (env.device.platform === "chromium") return captureChromiumPng(env); + const file = await captureFile(env); + try { + return await fs.readFile(file); + } finally { + await fs.rm(file, { force: true }).catch(() => {}); + } +} + /** * One capture as decoded pixels, or `undefined` when the pixels could not be * read (any capture or decode failure). Soft by design — the caller treats it @@ -153,13 +201,8 @@ async function captureFile(env: ActionEnv): Promise { */ async function capturePixels(env: ActionEnv): Promise { try { - const file = await captureFile(env); - try { - const png = PNG.sync.read(await fs.readFile(file)); - return { width: png.width, height: png.height, data: png.data }; - } finally { - await fs.rm(file, { force: true }).catch(() => {}); - } + const png = PNG.sync.read(await capturePng(env)); + return { width: png.width, height: png.height, data: png.data }; } catch { return undefined; } diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index 6f50d8b9d..2a5d26fd8 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -278,6 +278,40 @@ describe("capturePixels routing", () => { expect(resolveService).not.toHaveBeenCalled(); }); + // Chromium is the one route that never touches the filesystem, and the one + // that cannot use the `screenshot` tool's scaling: that resizes with `sharp`, + // an optional dependency nothing here installs, so asking it for a quarter + // scale returned a full-resolution PNG and a settle decoded a 23MB buffer + // twice a second. The compositor applies `clip.scale` while rasterizing, so + // the small frame is the only one that exists. + it("captures Chromium through the compositor's own scale, and never via a file", async () => { + const png = new PNG({ width: 2, height: 1 }); + png.data.set([10, 20, 30, 255, 40, 50, 60, 255]); + const send = vi.fn(async () => ({ data: PNG.sync.write(png).toString("base64") })); + const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; + const resolveService = vi.fn(async () => ({ + cdp: { send }, + getViewport: () => ({ width: 900, height: 700, devicePixelRatio: 2 }), + // The route that WOULD go through sharp. Reaching for it is the bug. + captureScreenshot: vi.fn(() => { + throw new Error("the sharp-backed capture must not be used for a settle"); + }), + })); + + const pixels = await capture(envFor(device, resolveService)); + + expect(pixels).toMatchObject({ width: 2, height: 1 }); + expect(resolveService).toHaveBeenCalledWith(`ChromiumCdp:${device.id}`, { device }); + // The clip is the viewport in CSS pixels; its scale composes with the + // page's device scale factor, so the plain capture scale lands on a quarter + // of the frame this route used to return. + expect(send).toHaveBeenCalledWith("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: false, + clip: { x: 0, y: 0, width: 900, height: 700, scale: 0.25 }, + }); + }); + it("leaves Android on the simulator-server route without an iOS runtime probe", async () => { const file = await pngAt(tmpDir, "android.png"); const resolveService = vi.fn(async () => ({ From 9c94ce1ecddc6e166f6dafab36ca02cb1247cfac Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:34:06 +0200 Subject: [PATCH 73/98] test(flow): hold the idle check's constants and unexercised branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite passed with MIN_STILL_INTERVALS raised to 3 and with the hold clock replaced by `>= 0` — the two constants the step's design argues hardest for were pinned from one side or not at all. Both mutations now fail. Also covered: the default 250ms hold, where the number is actually used rather than only in a parser message; the branch where no tree read ever succeeds, which produces the foreground advice from a standing start; the hold restarting after a failed read and after the screen goes blank; and abort, which had no test anywhere. The capture stub now carries the same three inputs the real one takes — it was dropping the abort signal and the first-capture flag, so neither was exercised inside the loop. --- .../test/flows/flow-idle-run.test.ts | 184 ++++++++++++++++-- 1 file changed, 173 insertions(+), 11 deletions(-) diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index fc6f413e8..e63bf775f 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; -import type { Registry } from "@argent/registry"; +import type { Registry, ToolContext } from "@argent/registry"; import type { DescribeNode, DescribeTreeData } from "../../src/tools/describe/contract"; import type { PixelFrame } from "../../src/tools/flows/flow-pixels"; @@ -21,20 +21,34 @@ vi.mock("../../src/tools/flows/flow-tree", () => ({ }), })); -// Stub only the capture; the real `pixelsDiffer` decides whether two frames +// Stub only the capture; the real `comparePixels` decides whether two frames // moved, so the comparison the check depends on is the one under test. let currentFrame: () => PixelFrame | undefined; +/** Every `firstCapture` flag the runner passed, in order. */ +const captureFirstFlags: boolean[] = []; vi.mock("../../src/tools/flows/flow-pixels", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - // Honours `deadline` exactly as the real one does — it returns undefined - // without capturing once the budget is gone. A stub that ignored it hid - // the case where the runner judges a screen on a round it had no time to - // observe, which is where a missing frame used to read as stillness. + // Carries the same three inputs the real one has, so nothing the runner + // hands it can go unexercised: + // - `deadline`, which it honours by returning undefined without capturing + // once the budget is gone. A stub that ignored it hid the case where the + // runner judges a screen on a round it had no time to observe, which is + // where a missing frame used to read as stillness. + // - the abort signal, which the real capture is abandoned on. + // - `firstCapture`, which buys a cold stream's first frame a wider bound + // and must be true exactly once per step. capturePixelsWithin: vi.fn( - async (_env: unknown, deadline: number): Promise => - Date.now() >= deadline ? undefined : currentFrame() + async ( + env: { signal?: AbortSignal }, + deadline: number, + firstCapture: boolean + ): Promise => { + captureFirstFlags.push(firstCapture); + if (env.signal?.aborted) return undefined; + return Date.now() >= deadline ? undefined : currentFrame(); + } ), }; }); @@ -99,9 +113,14 @@ async function writeFlow(name: string, yaml: string): Promise { await fs.writeFile(path.join(dir, `${name}.yaml`), yaml, "utf8"); } -async function run(name: string): Promise { +async function run(name: string, signal?: AbortSignal): Promise { const tool = createRunFlowTool(mockRegistry()); - const result = await tool.execute({}, { name, project_root: tmpDir, device: DEVICE }, undefined); + const result = await tool.execute( + {}, + { name, project_root: tmpDir, device: DEVICE }, + // Only the signal matters here; the runner does not touch the rest. + signal ? ({ signal } as unknown as ToolContext) : undefined + ); if (!("steps" in result)) throw new Error("expected a run result"); return result; } @@ -111,6 +130,7 @@ beforeEach(async () => { currentTree = () => screenWith("Home"); currentFrame = () => frameAt(120); treeDelayMs = 0; + captureFirstFlags.length = 0; }); afterEach(async () => { @@ -157,7 +177,33 @@ steps: ` ); expect((await run("ready")).ok).toBe(true); - expect(reads).toBeGreaterThanOrEqual(3); + // Exactly three, pinned from both sides: two would settle on a single + // agreeing pair (the aliasing case below), four would make every settle a + // poll slower than it needs to be. + expect(reads).toBe(3); + // And only the first capture of the step may claim the cold-stream bound. + expect(captureFirstFlags).toEqual([true, false, false]); + }); + + // `minStableMs` is a clock, not a label: a screen that is still from the + // first read must still be held for it before the step returns. Without that + // the option means nothing, since three reads take about 400ms whatever it + // is set to. + it("holds a still screen for the whole requested hold before passing", async () => { + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 2500, minStableMs: 800 } +` + ); + const started = Date.now(); + const r = await run("ready"); + const elapsed = Date.now() - started; + expect(r.steps.at(-1)).toMatchObject({ kind: "idle", status: "pass" }); + expect(r.steps.at(-1)!.warning).toBeUndefined(); + expect(elapsed).toBeGreaterThanOrEqual(750); + expect(elapsed).toBeLessThan(2_400); }); it("warns, and does not fail, when the tree never stops changing", async () => { @@ -451,6 +497,122 @@ steps: expect(r.steps.at(-1)!.warning).not.toContain("UI tree alone"); }); + // The default hold is what nearly every step runs with, so it is worth + // pinning somewhere the number is actually used rather than only in a parser + // message. + it("holds for 250ms by default", async () => { + let tick = 0; + currentTree = () => screenWith(`frame ${tick++}`); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 900 } +` + ); + expect((await run("ready")).steps.at(-1)!.warning).toContain( + "never held still for 250ms within 900ms" + ); + }); + + // A tree source that never answers at all: no read succeeded, so there is + // nothing to reason from and the advice is about the window, not the app. + // The existing mid-wait case lets the first read land, so this branch — the + // one that produces the foreground advice from a standing start — was never + // taken. + it("reports a tree source that never answered as unreadable, naming the underlying error", async () => { + currentTree = () => { + throw new Error("native-devtools is not connected"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 900, minStableMs: 0 } + - echo: unreachable +` + ); + const r = await run("ready"); + expect(r.ok).toBe(false); + const step = r.steps.find((s) => s.kind === "idle")!; + expect(step.status).toBe("error"); + expect(step.reason).toContain("could not read the UI tree"); + expect(step.reason).toContain("foreground"); + expect(step.reason).toContain("native-devtools is not connected"); + // An indeterminate readiness check stops the run rather than recording a + // regression the app never had. + expect(r.steps.at(-1)!.status).toBe("skip"); + }); + + // A blip mid-settle is expected — the hold restarts from the next good read + // rather than the step giving up or carrying its pre-blip state across. + it("restarts the hold after a failed read, and still settles", async () => { + let reads = 0; + currentTree = () => { + reads += 1; + if (reads === 2) throw new Error("transient describe failure"); + return screenWith("Home"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toBeUndefined(); + // 1 ok, 2 failed, 3 is a fresh start (nothing to compare against), 4 and 5 + // are the two agreeing intervals. Three would mean the blip was ignored. + expect(reads).toBe(5); + }); + + // The same for a screen that goes blank in the middle: an observation that + // resets both holds, not a gap and not a reason to give up. + it("restarts the hold after the screen goes blank, and still settles", async () => { + let reads = 0; + currentTree = () => { + reads += 1; + return reads === 2 ? n({ role: "AXWindow", frame: FULL, children: [] }) : screenWith("Home"); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toBeUndefined(); + expect(reads).toBe(5); + }); + + // Cancelling a run is not a verdict about the screen. The check has to stop + // promptly and report a skip, never a pass, a warning or an error. + it("stops on abort without judging the screen", async () => { + let tick = 0; + currentTree = () => screenWith(`frame ${tick++}`); // never settles + const controller = new AbortController(); + setTimeout(() => controller.abort(), 300); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 7500 } + - echo: unreachable +` + ); + const started = Date.now(); + const r = await run("ready", controller.signal); + expect(Date.now() - started).toBeLessThan(3_000); + const step = r.steps.find((s) => s.kind === "idle")!; + expect(step.status).toBe("skip"); + expect(step.reason).toContain("aborted"); + expect(step.warning).toBeUndefined(); + }); + // One good read early does not license an app verdict drawn from a window // that went dark afterwards: a backgrounded app or a dropped instrumentation // session reads as "unknown", never as "still animating". From 7c032694dc9bce6b25a4297daa25f1f05f7060ba Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:38:03 +0200 Subject: [PATCH 74/98] docs(flow): make every description of the idle step match what it does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commit that turned the idle timeout from a failure into a warning updated the skill, the section comments and the tests, and missed the `flow-execute` tool description — the surface an authoring agent actually reads — which still said it FAILS on timeout, with "so it is safe to persist" hung off the claim that was now backwards. It is safe to persist BECAUSE it cannot fail. A test pins both descriptions to that. Also corrected: - the skill now lists all four warnings a passing step can carry, rather than sending the author to look for one they may never get, and says what actually stops a run: a tree source that cannot be read, not merely an empty tree. - it documents that there is no `assert` or `when:` form, that the recorder cannot emit one, and — restoring a line the fail-to-warn commit deleted — that sprinkling it after every step buys nothing now that it cannot fail. - the timeout floor the parser enforces. - MIN_ROUND_BUDGET_MS described the guarantee it does not make: the check runs before the poll sleep, so it is a floor on the wait, not on the round. - both renderers described `warning` as legacy wire-compat for old tool-servers, which now reads as an invitation to delete the field this feature reports through. --- packages/argent-cli/src/flow.ts | 10 ++++--- packages/argent-mcp/src/content.ts | 10 ++++--- .../skills/skills/argent-create-flow/SKILL.md | 13 ++++++--- .../src/tools/flows/flow-actions.ts | 20 ++++++++------ .../tool-server/src/tools/flows/flow-run.ts | 27 ++++++++++--------- .../test/flows/flow-skill-docs.test.ts | 20 ++++++++++++++ 6 files changed, 71 insertions(+), 29 deletions(-) diff --git a/packages/argent-cli/src/flow.ts b/packages/argent-cli/src/flow.ts index 92e135b78..ac97a78ca 100644 --- a/packages/argent-cli/src/flow.ts +++ b/packages/argent-cli/src/flow.ts @@ -25,9 +25,13 @@ export interface StepReport { status: "pass" | "fail" | "skip" | "error"; reason?: string; /** - * Legacy: older tool-servers passed a snapshot that adopted a missing - * baseline and annotated it with this caveat (a missing baseline now fails - * the step). Rendered for wire compat with a not-yet-updated server. + * A step that passed in a way that weakens it as proof — raised today by + * `await: { idle: true }`, which never fails a run and says here what its + * green actually bought (see StepReport.warning in the tool-server's + * flow-run). Also carries the caveat older tool-servers put on a snapshot + * that adopted a missing baseline, which now fails the step instead. Live + * either way: dropping the field would silently delete the only thing the + * readiness check reports. */ warning?: string; tool?: string; diff --git a/packages/argent-mcp/src/content.ts b/packages/argent-mcp/src/content.ts index 799f9efc6..59c6431a0 100644 --- a/packages/argent-mcp/src/content.ts +++ b/packages/argent-mcp/src/content.ts @@ -211,9 +211,13 @@ export type FlowStepResult = { status?: "pass" | "fail" | "skip" | "error"; reason?: string; /** - * Legacy: older tool-servers passed a snapshot that adopted a missing - * baseline and annotated it with this caveat (a missing baseline now fails - * the step). Rendered for wire compat with a not-yet-updated server. + * A step that passed in a way that weakens it as proof — raised today by + * `await: { idle: true }`, which never fails a run and says here what its + * green actually bought (see StepReport.warning in the tool-server's + * flow-run). Also carries the caveat older tool-servers put on a snapshot + * that adopted a missing baseline, which now fails the step instead. Live + * either way: dropping the field would silently delete the only thing the + * readiness check reports. */ warning?: string; tool?: string; diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 97e3f2fd3..92a342cfd 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -85,11 +85,18 @@ This condition-as-key form is the only spelling. `await` also accepts an optiona For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. -**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250 — it has to be shorter than the timeout, or the gate could never pass) and `timeout` (default 7500). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. +**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. -It **never fails a run.** A screen that never settles spends the timeout, then passes with a `warning` on the step: readiness is not an acceptance criterion, and plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text). Read that warning rather than stepping over it — a stuck spinner looks exactly the same, and it means the screen never finished loading. Only an unreadable tree stops the run, as an `error`. +It **never fails a run.** Readiness is not an acceptance criterion, so every outcome short of a clean settle passes carrying a `warning` on the step — read it rather than stepping over it: -It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. +- **the screen never held still** — it spent the timeout and went ahead. Plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text); a screen that never finished loading looks the same from here. +- **a small part of it kept changing** — a spinner, a caret, a progress dot. Too small to be the screen moving, so the settle completed anyway; if it is a loading spinner, the screen was still loading when this step returned. +- **the tree stayed empty** — the screen rendered no accessible content. Sometimes the app (a canvas, a video surface), sometimes a screen that never arrived. +- **settled on the UI tree alone** — the screen could not be screenshotted often enough to compare a pair, so presentation-layer motion (a push, a fade, a dismissing modal) was not waited out. + +Only a tree source that cannot be read stops the run, as an `error` — one that fails outright, or one that answers and then wedges. That is a broken window, not a verdict about the app. + +It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. There is no `assert` form (waiting is the whole point), no `when:` form, and the recorder cannot emit one — every `idle` step is hand-written. Do not sprinkle it after every step: each one costs a settle, and it cannot fail, so a flow full of them is slower without being stricter. ### `type` and `scroll-to` diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index ca951fdd2..21bf4853e 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1171,11 +1171,15 @@ async function waitForCondition( const MIN_STILL_INTERVALS = IDLE_MIN_STILL_INTERVALS; /** - * The smallest budget a poll round is allowed to start with. A round begun - * with nothing left cannot capture and cannot read, and both absences were - * being recorded as facts about the device: the skipped capture latched - * "captures do not work here", and the abandoned read latched "the tree source - * is not answering". Neither was true — the step had simply run out of time. + * How much budget must be left for another round to be worth STARTING — which + * is checked before the poll sleep, so it is spent on the sleep and the round + * that follows begins with whatever is left. It is a floor on the wait, not on + * the round: what it rules out is starting a round in the last few + * milliseconds of the step, where the capture is skipped and the read is + * abandoned, and both absences used to be recorded as facts about the device — + * "captures do not work here", "the tree source is not answering" — when the + * step had simply run out of time. + * * The first round always runs, so an unusually short `timeout:` still buys one * honest look, and ending up to one round early is strictly better than * judging a screen nobody managed to observe. @@ -1515,9 +1519,9 @@ async function waitForIdle( warning: `the screen never held still for ${minStableMs}ms within ${timeoutMs}ms, so this step went ` + `ahead without waiting it out. Either something on it never stops (a video, a looping ` + - `animation, a carousel, live-updating text) or the screen never finished loading — a stuck ` + - `spinner looks like both. Look at what is moving, and make sure the next action is gated on ` + - `a stable element rather than on stillness.`, + `animation, a carousel, live-updating text) or the screen never finished loading. Look at ` + + `what is moving, and make sure the next action is gated on a stable element rather than on ` + + `stillness.`, }; } diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 17b8cc635..793ba4761 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -190,11 +190,13 @@ export interface StepReport { reason?: string; /** * The step passed, but the WAY it passed weakens it as proof. Rendered as a - * "⚠" suffix by the MCP client. Raised by `await: { idle: true }`, either - * because the screen never settled at all — it waits, then goes ahead — or - * because its captures never produced a comparable pair, leaving stillness - * proved on the UI tree alone without the presentation-layer motion the - * pixel half exists to catch. + * "⚠" suffix by the MCP client. Raised by `await: { idle: true }`: the screen + * never settled at all (it waits, then goes ahead); something small on it + * never stopped, which is what a spinner looks like; it rendered no content + * to settle; the step ran out of reads before it could judge anything; or its + * captures never produced a comparable pair, leaving stillness proved on the + * UI tree alone without the presentation-layer motion the pixel half exists + * to catch. */ warning?: string; /** Underlying tool id for `tool` steps. */ @@ -925,10 +927,10 @@ reads "inside card inside list", each container's frame inside the next); two-finger rotation gesture (\`rotate: { on?, by }\` — degrees, + clockwise, within ±3000°; screen center when \`on\` is omitted; distinct from the \`rotate\` tool, which changes device orientation); \`await\` waits for a UI condition, and additionally takes the one condition that has no selector: \`idle: true\` waits -until the screen has content and stops moving in BOTH the UI tree and the rendered pixels (unlike the -\`await-screen-idle\` tool it FAILS on timeout, so it is safe to persist; it says nothing about WHICH -screen settled — a dropped tap leaves the source screen perfectly idle — so pair it with the element -check that names the destination); \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` +until the screen has content and stops moving in BOTH the UI tree and the rendered pixels (it never +fails a run — a screen that never settles passes carrying a \`warning\`, which is what makes it safe to +persist; it says nothing about WHICH screen settled — a dropped tap leaves the source screen perfectly +idle — so pair it with the element check that names the destination); \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` diffs a screenshot — or, with \`cropOn: \`, one element's cropped region — against a stored baseline (a missing baseline fails the step — set updateBaselines to adopt the current screen; a cropped element whose size drifted fails on dimensions); \`echo\` annotates; \`run\` executes another flow @@ -2128,9 +2130,10 @@ async function execLeafStep( // and `wait`), never a step failure — the app did nothing wrong. if (r.aborted) return { ...base, status: "skip", reason: r.reason }; // `indeterminate` is `idle`'s only non-passing outcome: a screen that - // merely kept moving passes with a warning, so what is left here is a - // wait that could not run at all (an unreadable or degraded tree, a - // screen nobody managed to observe). Scoring that `fail` would make CI + // merely kept moving passes with a warning, and so does one that + // rendered nothing, so what is left here is a wait that could not run + // at all — a tree source that failed, or one that answered and then + // wedged. Scoring that `fail` would make CI // read an environment problem as a regression and a QA author reset a // pass streak over it. `error` keeps the run non-ok while saying // plainly that the app was never judged. Scoped to `idle`, whose whole diff --git a/packages/tool-server/test/flows/flow-skill-docs.test.ts b/packages/tool-server/test/flows/flow-skill-docs.test.ts index 880570dcc..5d8247337 100644 --- a/packages/tool-server/test/flows/flow-skill-docs.test.ts +++ b/packages/tool-server/test/flows/flow-skill-docs.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; import * as path from "node:path"; +import type { Registry } from "@argent/registry"; import { parseFlow } from "../../src/tools/flows/flow-utils"; +import { createRunFlowTool } from "../../src/tools/flows/flow-run"; /** * The create-flow skill is the agent-facing reference for selector scopes, so @@ -42,6 +44,24 @@ describe("create-flow SKILL.md scope snippets", () => { } }); + // The two agent-facing descriptions of `idle` have to agree with what it + // does. The commit that turned its timeout from a failure into a warning + // updated the skill, the comments and the tests, and left the tool + // description — the surface an authoring agent actually reads — saying the + // opposite, with "so it is safe to persist" hung off the claim that was now + // backwards. + it("the flow-execute description and the skill agree that idle warns rather than fails", () => { + const description = createRunFlowTool({} as unknown as Registry).description; + expect(description).toContain("idle: true"); + expect(description).toMatch(/never\s+fails a run/); + expect(description).not.toMatch(/FAILS on timeout/i); + + const skill = readFileSync(SKILL, "utf8"); + expect(skill).toContain("It **never fails a run.**"); + // The one outcome that does stop a run is the window, never the app. + expect(skill).toMatch(/Only a tree source that cannot be read stops the run/); + }); + it("the paragraph's rejected `any` spelling really is rejected", () => { // The docs tell agents to write `{ role: Switch, next: … }` and NOT // `{ any: true, role: Switch, next: … }`. If the parser ever started From 1f98b232a2bc8cb4edd97b461daa5f7daec5824e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 15:44:31 +0200 Subject: [PATCH 75/98] docs(flow): cite the measured numbers behind the localized-motion floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified on real devices at the capture scale the check uses: a static iOS screen changes 0 pixels between two captures, a live spinner on an iPhone 16 Pro changes 50 of 198k, and a blinking caret on a Pixel 7 changes 50 of 162k. The floor sits an order of magnitude below the movers and clear of the noise, and the residual limit — an indicator smaller than that — is now stated rather than left to be discovered. --- packages/tool-server/src/tools/flows/flow-pixels.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index b5af25f94..0e393e282 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -79,8 +79,12 @@ const MOTION_FRACTION = 0.002; // capture and a full-resolution desktop one. // // The floor is not zero because a capture pair is not guaranteed byte-identical -// on every backend; it is two orders of magnitude below the smallest spinner -// and one below a caret, which is the widest margin that still sees them. +// on every backend. Measured at CAPTURE_SCALE: two captures of a static iOS +// screen changed 0 pixels of 237k, while a live spinner on an iPhone 16 Pro +// changed 50 of 198k and a blinking caret on a Pixel 7 changed 50 of 162k. A +// floor of ~10 pixels on a phone-sized frame therefore sits well clear of both +// ends. An indicator smaller than that stays invisible, which is the residual +// limit of comparing whole frames. const LOCALIZED_MOTION_FRACTION = 0.00005; // `httpScreenshot` may spend its full first-frame wait before it even returns From 5c38fac4cccc2538b8563485d694067672f31069 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:49:35 +0200 Subject: [PATCH 76/98] fix(flow): aim the Chromium settle capture at the window, not the page top MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Page.captureScreenshot`'s `clip` is measured from the top of the document, so the fixed `{ x: 0, y: 0 }` origin pointed at the top of the page rather than at the window. On a scrolled document that rectangle is off-screen and Chrome rasterizes it blank; two blank captures compare as identical, so the pixel half of the settle voted "still" on every interval of a visibly animating screen — and did it silently, because the capture succeeded and the tree-only warning could not fire. Measured on a scrolled page carrying a full-window black-to-red cross-fade: 5 clean passes out of 5. Take the origin from `Page.getLayoutMetrics`, a browser-side layout read that does not need the live main world the cached viewport's `Runtime.evaluate` does, so it survives the mid-navigation renderer this check runs against. Same page, same scroll offset: motion detected 5 runs of 5, while an unscrolled page and one scrolled to a static band both still settle cleanly 3/3, and a page scrolling inside an inner element is unaffected 3/3. The Chromium capture test asserted the clip arguments against a mock with no concept of scroll, which pinned the buggy shape. It now answers the way the compositor does — content for a rectangle inside the rendered window, blank white for one outside it — so a wrong origin fails on the colour it returns. --- .../src/tools/flows/flow-pixels.ts | 59 ++++++- .../test/flows/flow-pixels.test.ts | 158 ++++++++++++++++-- 2 files changed, 195 insertions(+), 22 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index 0e393e282..7993bfaea 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -131,25 +131,72 @@ export function pixelCaptureTimeoutMs(device: ActionEnv["device"], firstCapture: * applies. Measured on a 900x613 viewport at dpr 2: 1800x1226 and ~25ms of * blocking decode per poll became 450x307 and ~3ms. * - * The viewport is the cached one rather than a fresh read: refreshing it costs - * a `Runtime.evaluate` per poll, and that call fails outright on a renderer - * mid-navigation — which is exactly when this check runs. A window resized - * mid-step therefore clips against the previous size for the rest of it, and - * registers as content change like anything else that moves. + * A `clip` is measured from the top of the DOCUMENT, not of the window, so its + * origin has to follow the scroll — `{ x: 0, y: 0 }` names the top of the page, + * which on a scrolled document is off-screen and rasterizes as a blank + * rectangle. Two blank rectangles compare as identical, so that origin made the + * pixel half of the check vote "still" on every interval of a visibly animating + * screen, and vote it silently: the capture succeeded, so nothing warned. The + * offset comes from `Page.getLayoutMetrics`, which is a browser-side read of + * the frame's own layout — unlike the `Runtime.evaluate` behind the cached + * viewport, it does not depend on a live main world, so it survives the + * mid-navigation renderer this check runs against. A page whose scrolling lives + * in an inner element reports no document scroll and clips at the origin, which + * is already the right rectangle for it. + * + * The viewport SIZE is still the cached one: refreshing it does cost a + * `Runtime.evaluate` per poll. A window resized mid-step therefore clips + * against the previous size for the rest of it, and registers as content + * change like anything else that moves. */ async function captureChromiumPng(env: ActionEnv): Promise { const ref = chromiumCdpRef(env.device); const api = (await env.registry.resolveService(ref.urn, ref.options)) as ChromiumCdpApi; const { width, height } = api.getViewport(); + const { x, y } = await chromiumScrollOffset(api); const shot = (await api.cdp.send("Page.captureScreenshot", { format: "png", captureBeyondViewport: false, - clip: { x: 0, y: 0, width, height, scale: CAPTURE_SCALE }, + clip: { x, y, width, height, scale: CAPTURE_SCALE }, })) as { data?: string }; if (!shot.data) throw new Error("Page.captureScreenshot returned no data"); return Buffer.from(shot.data, "base64"); } +/** What `Page.getLayoutMetrics` reports about where the window sits in the page. */ +interface CssViewportMetrics { + pageX?: number; + pageY?: number; +} + +/** + * Where the window's top-left corner sits in document coordinates, in CSS + * pixels — the origin {@link captureChromiumPng} must clip from. + * + * `cssVisualViewport` is preferred because it also carries a pinch-zoom offset; + * `cssLayoutViewport` is the fallback for a protocol that predates it. A read + * that fails or answers with nothing usable falls back to the document origin, + * which is the unscrolled answer and no worse than not asking. + */ +async function chromiumScrollOffset(api: ChromiumCdpApi): Promise<{ x: number; y: number }> { + try { + const metrics = (await api.cdp.send("Page.getLayoutMetrics")) as { + cssVisualViewport?: CssViewportMetrics; + cssLayoutViewport?: CssViewportMetrics; + layoutViewport?: CssViewportMetrics; + }; + const vp = metrics.cssVisualViewport ?? metrics.cssLayoutViewport ?? metrics.layoutViewport; + const x = vp?.pageX; + const y = vp?.pageY; + return { + x: typeof x === "number" && Number.isFinite(x) ? x : 0, + y: typeof y === "number" && Number.isFinite(y) ? y : 0, + }; + } catch { + return { x: 0, y: 0 }; + } +} + /** * Capture one downscaled screenshot to a temp file, routed exactly as the * `screenshot` tool routes it: tvOS and Vega through their own shells (neither diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index 2a5d26fd8..6c5f73cc7 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -223,6 +223,82 @@ function capture(env: ActionEnv): Promise { return capturePixelsWithin(env, Date.now() + 30_000, false); } +/** What the fake page below paints inside the rendered window. */ +const VISIBLE_BAND_RGB: [number, number, number] = [200, 30, 10]; +/** What Chrome hands back for a clip rectangle outside the rendered window. */ +const OFF_SCREEN_RGB: [number, number, number] = [255, 255, 255]; + +/** The RGB triple of one pixel of a decoded frame. */ +function pixelAt(frame: PixelFrame | undefined, index: number): [number, number, number] { + if (!frame) throw new Error("expected a decoded frame"); + const o = index * 4; + return [frame.data[o], frame.data[o + 1], frame.data[o + 2]]; +} + +interface Clip { + x: number; + y: number; + width: number; + height: number; + scale: number; +} + +/** + * A Chromium page that answers `Page.captureScreenshot` the way the compositor + * does: `clip` is in DOCUMENT coordinates, and with `captureBeyondViewport` + * false only the rendered window has pixels — a rectangle outside it comes back + * blank white. That is what makes a wrong clip origin visible as a colour here + * rather than only as an argument. + */ +function fakeChromiumPage(opts: { metricsError?: Error } = {}): { + api: unknown; + scrollTo(y: number): void; + lastClip(): Clip | undefined; +} { + const viewport = { width: 900, height: 700, devicePixelRatio: 2 }; + let scrollY = 0; + let lastClip: Clip | undefined; + + const send = vi.fn(async (method: string, params?: Record) => { + if (method === "Page.getLayoutMetrics") { + if (opts.metricsError) throw opts.metricsError; + return { + cssVisualViewport: { + pageX: 0, + pageY: scrollY, + clientWidth: viewport.width, + clientHeight: viewport.height, + }, + }; + } + if (method !== "Page.captureScreenshot") throw new Error(`unexpected CDP call ${method}`); + lastClip = params?.clip as Clip; + const insideWindow = + lastClip !== undefined && + lastClip.y >= scrollY && + lastClip.y + lastClip.height <= scrollY + viewport.height; + const [r, g, b] = insideWindow ? VISIBLE_BAND_RGB : OFF_SCREEN_RGB; + const png = new PNG({ width: 2, height: 1 }); + png.data.set([r, g, b, 255, r, g, b, 255]); + return { data: PNG.sync.write(png).toString("base64") }; + }); + + return { + api: { + cdp: { send }, + getViewport: () => viewport, + // The route that WOULD go through sharp. Reaching for it is the bug. + captureScreenshot: vi.fn(() => { + throw new Error("the sharp-backed capture must not be used for a settle"); + }), + }, + scrollTo(y: number) { + scrollY = y; + }, + lastClip: () => lastClip, + }; +} + describe("capturePixels routing", () => { // Every platform argent can screenshot has a route here, and each one is a // different backend — sending a device down the wrong one silently costs the @@ -285,18 +361,9 @@ describe("capturePixels routing", () => { // twice a second. The compositor applies `clip.scale` while rasterizing, so // the small frame is the only one that exists. it("captures Chromium through the compositor's own scale, and never via a file", async () => { - const png = new PNG({ width: 2, height: 1 }); - png.data.set([10, 20, 30, 255, 40, 50, 60, 255]); - const send = vi.fn(async () => ({ data: PNG.sync.write(png).toString("base64") })); const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; - const resolveService = vi.fn(async () => ({ - cdp: { send }, - getViewport: () => ({ width: 900, height: 700, devicePixelRatio: 2 }), - // The route that WOULD go through sharp. Reaching for it is the bug. - captureScreenshot: vi.fn(() => { - throw new Error("the sharp-backed capture must not be used for a settle"); - }), - })); + const page = fakeChromiumPage(); + const resolveService = vi.fn(async () => page.api); const pixels = await capture(envFor(device, resolveService)); @@ -305,11 +372,70 @@ describe("capturePixels routing", () => { // The clip is the viewport in CSS pixels; its scale composes with the // page's device scale factor, so the plain capture scale lands on a quarter // of the frame this route used to return. - expect(send).toHaveBeenCalledWith("Page.captureScreenshot", { - format: "png", - captureBeyondViewport: false, - clip: { x: 0, y: 0, width: 900, height: 700, scale: 0.25 }, - }); + expect(page.lastClip()).toMatchObject({ width: 900, height: 700, scale: 0.25 }); + }); + + // `clip` is measured from the top of the DOCUMENT. Pinning its origin at + // (0, 0) therefore aimed the capture at the top of the page rather than at + // the window, and on a scrolled document Chrome rasterizes that off-screen + // rectangle as blank white. Two blank captures compare as identical, so the + // pixel half of the settle voted "still" on every interval of a visibly + // animating screen — and voted it silently, because the capture succeeded. + // + // The mock below is the compositor's actual behaviour rather than an + // argument matcher: it serves whatever the clip rectangle overlaps in the + // rendered window, and white for anything outside it. A capture aimed at the + // wrong origin therefore comes back the wrong COLOR, which is the thing the + // comparison acts on. + it("captures the scrolled window, not the top of the document", async () => { + const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; + const page = fakeChromiumPage(); + page.scrollTo(1839); + + const pixels = await capture( + envFor( + device, + vi.fn(async () => page.api) + ) + ); + + expect(page.lastClip()).toMatchObject({ x: 0, y: 1839 }); + // The visible band, not the blank rectangle above the fold. + expect(pixelAt(pixels, 0)).toEqual(VISIBLE_BAND_RGB); + expect(pixelAt(pixels, 0)).not.toEqual(OFF_SCREEN_RGB); + }); + + it("still clips at the document origin for an unscrolled page", async () => { + const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; + const page = fakeChromiumPage(); + + const pixels = await capture( + envFor( + device, + vi.fn(async () => page.api) + ) + ); + + expect(page.lastClip()).toMatchObject({ x: 0, y: 0 }); + expect(pixelAt(pixels, 0)).toEqual(VISIBLE_BAND_RGB); + }); + + it("falls back to the document origin when the layout metrics cannot be read", async () => { + // A renderer that will not answer the metrics read leaves the capture no + // worse off than never asking — an unscrolled clip, not a failed settle. + const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; + const page = fakeChromiumPage({ metricsError: new Error("renderer is navigating") }); + page.scrollTo(1839); + + const pixels = await capture( + envFor( + device, + vi.fn(async () => page.api) + ) + ); + + expect(page.lastClip()).toMatchObject({ x: 0, y: 0 }); + expect(pixels).toMatchObject({ width: 2, height: 1 }); }); it("leaves Android on the simulator-server route without an iOS runtime probe", async () => { From 87bde5efb902a43106757ad2c2a971bb2c1e1555 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 17:57:49 +0200 Subject: [PATCH 77/98] fix(flow): keep the system status bar out of the settle's comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was the one comparison in the repo that looked at system chrome: `screenshot-diff` ignores the top 6% of the frame, the `snapshot` step opts into that, and the Android flow tree strips com.android.systemui outright, while the settle's comparator masked nothing and leaned entirely on the run-level `pinStatusBar`. That pin has two holes. It lands AFTER the run starts — the simulator repaints the clock and animates the battery fill 100-450ms in — so a fragment beginning with this step compares a real clock against a pinned one; and a nested `tool: flow-execute` clears the pin on its way out without the outer run re-pinning, leaving every later step comparing against a ticking clock. Either way a static, fully-loaded screen was reported as moving, or as carrying "a spinner ... the screen had not finished loading", which is the step's entire product. Mask the same 6% band, on the two platforms that have one: a Chromium window's top band is page content, and Vega and tvOS render with no system chrome, so those keep the whole frame. The masked rows leave the denominator too, so the motion and localized fractions still mean what they say. The two frames the runner was caught comparing on a static iPhone 16 Pro are now cases: 408 changed pixels at y[19..29] (over the 396-pixel motion budget) and the 13-pixel tail at y[19..25] that became the spinner warning. Both read as motion unmasked and still masked, while the same changes one row below the band are still seen. Verified on a live Chromium target that a window whose top 4% is the only thing animating is still detected as moving, 3 runs of 3. --- .../src/tools/flows/flow-actions.ts | 12 ++- .../src/tools/flows/flow-pixels.ts | 55 +++++++++-- .../test/flows/flow-idle-run.test.ts | 58 ++++++++++- .../test/flows/flow-pixels.test.ts | 98 +++++++++++++++++++ 4 files changed, 210 insertions(+), 13 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 21bf4853e..3b93ba208 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -23,7 +23,12 @@ import { settleWithin, sleepOrAbort } from "../../utils/timing"; import { invokeSubTool } from "../../utils/sub-invoke"; import { bindDeviceArgs } from "./flow-device"; import { fetchFlowTree } from "./flow-tree"; -import { capturePixelsWithin, comparePixels, type PixelFrame } from "./flow-pixels"; +import { + capturePixelsWithin, + comparePixels, + statusBarMaskFraction, + type PixelFrame, +} from "./flow-pixels"; import { buildAxisCandidate, decomposePinch, @@ -1251,6 +1256,9 @@ async function waitForIdle( ): Promise { const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; const minStableMs = step.minStableMs ?? IDLE_DEFAULT_MIN_STABLE_MS; + // Resolved once: it depends only on the device, and on iOS it costs a + // runtime probe the capture path memoizes anyway. + const maskTopFraction = await statusBarMaskFraction(env.device); const deadline = Date.now() + timeoutMs; // Two hold clocks, because the tree can settle while the pixels have not. @@ -1374,7 +1382,7 @@ async function waitForIdle( captureFailed = true; } else { if (previousFrame !== undefined) { - const change = comparePixels(previousFrame, frame); + const change = comparePixels(previousFrame, frame, maskTopFraction); if (change === "moving") pixelsEverMoved = true; else { pixelsHeld = true; diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index 7993bfaea..995622e0b 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -87,6 +87,39 @@ const MOTION_FRACTION = 0.002; // limit of comparing whole frames. const LOCALIZED_MOTION_FRACTION = 0.00005; +// Top band excluded from the comparison on a device with a system status bar, +// as a fraction of frame height. The same 6% screenshot-diff ignores +// (DEFAULT_IGNORE_TOP_NORMALIZED_Y), which the `snapshot` step opts into and +// the Android flow tree matches by stripping com.android.systemui outright — +// this comparison was the one place in the repo that looked at system chrome. +// +// It has to be masked here rather than left to the run-level `pinStatusBar`, +// which is not enough on its own for two reasons, both measured. The pin lands +// AFTER the run starts — the simulator repaints the clock and animates the +// battery fill 100-450ms in — so a fragment whose first step is this one +// compares a real clock against a pinned one and calls a static screen moving +// (23 false spinner warnings in 43 runs on an iPhone 16 Pro; 0 in 10 when the +// same step followed a `wait: 2000`). And a nested `tool: flow-execute` clears +// the pin on its way out without the outer run ever re-pinning, leaving every +// later step of that run comparing against a live, ticking clock. +const STATUS_BAR_MASK_FRACTION = 0.06; + +/** + * The fraction of the frame {@link comparePixels} must ignore for this device, + * because the system paints it and the app does not. + * + * Only iOS and Android put a status bar in the capture. A Chromium window's top + * band is page content, and Vega and tvOS render full-screen with no system + * chrome, so masking any of those would blind the check to real motion for + * nothing. tvOS shares iOS's platform tag and is only distinguishable by the + * runtime probe, which is memoized per UDID and already paid by the capture. + */ +export async function statusBarMaskFraction(device: ActionEnv["device"]): Promise { + if (device.platform === "android") return STATUS_BAR_MASK_FRACTION; + if (device.platform !== "ios") return 0; + return (await isTvOsSimulator(device.id)) ? 0 : STATUS_BAR_MASK_FRACTION; +} + // `httpScreenshot` may spend its full first-frame wait before it even returns // a file path. Leave a separate completion margin for reading, decoding, and // removing that PNG. Warm captures get the tighter bound below. @@ -290,18 +323,24 @@ export async function capturePixelsWithin( * * Alpha is ignored — a screen capture is opaque. * - * Different dimensions count as motion. That branch covers a resized window - * (Chromium). It is NOT how a device rotation is caught: the Android capture - * keeps its portrait shape across one, so rotation registers through content - * change like anything else. + * `maskTopFraction` excludes that fraction of rows at the top of the frame, + * both from the count and from the total the fractions are taken against — see + * {@link statusBarMaskFraction} for which devices need it and why. + * + * Different dimensions count as motion. It is NOT how a device rotation is + * caught: the Android capture keeps its portrait shape across one, so rotation + * registers through content change like anything else. On Chromium the branch + * is effectively unreachable — the clip is built from the cached viewport, so a + * resize does not change the captured dimensions until something refreshes it. */ -export function comparePixels(a: PixelFrame, b: PixelFrame): PixelChange { +export function comparePixels(a: PixelFrame, b: PixelFrame, maskTopFraction = 0): PixelChange { if (a.width !== b.width || a.height !== b.height) return "moving"; - const total = a.width * a.height; - if (total === 0) return "still"; + const maskedRows = Math.min(a.height, Math.floor(a.height * maskTopFraction)); + const total = a.width * (a.height - maskedRows); + if (total <= 0) return "still"; const limit = Math.min(a.data.length, b.data.length); let changed = 0; - for (let o = 0; o + 2 < limit; o += 4) { + for (let o = maskedRows * a.width * 4; o + 2 < limit; o += 4) { const dr = a.data[o] - b.data[o]; const dg = a.data[o + 1] - b.data[o + 1]; const db = a.data[o + 2] - b.data[o + 2]; diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index e63bf775f..c12a88706 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -6,6 +6,15 @@ import type { Registry, ToolContext } from "@argent/registry"; import type { DescribeNode, DescribeTreeData } from "../../src/tools/describe/contract"; import type { PixelFrame } from "../../src/tools/flows/flow-pixels"; +// The status-bar mask asks the iOS runtime whether this UDID is a tvOS +// simulator. These UDIDs are fabricated, so a real probe would shell out to +// `xcrun simctl list` on every step and answer "unknown" each time — pin it to +// the mobile answer the cases are written against. +vi.mock("../../src/utils/ios-devices", async (importOriginal) => ({ + ...(await importOriginal()), + isTvOsSimulator: vi.fn(async () => false), +})); + // Serve the flow tree directly (see flow-when.test.ts) — `idle` polls it. let currentTree: () => DescribeNode; /** Simulates a tree source that is slow, or wedged when it exceeds the step. */ @@ -88,13 +97,23 @@ function frameWithMovingPixels(movingPixels: number, level: number): PixelFrame const [width, height] = [300, 600]; const data = Buffer.alloc(width * height * 4, 255); for (let i = 0; i < movingPixels; i++) { - data[i * 4] = level; - data[i * 4 + 1] = level; - data[i * 4 + 2] = level; + const o = (STATUS_BAR_ROWS * width + i) * 4; + data[o] = level; + data[o + 1] = level; + data[o + 2] = level; } return { width, height, data }; } +/** + * The first row of a 600-row frame the comparison actually looks at: the + * comparator masks the top 6% on a device with a system status bar, so motion + * a case means to be SEEN has to be placed below it. Frames that move + * everywhere (frameAt) are unaffected, as is a 10-row one, where 6% floors to + * no rows at all. + */ +const STATUS_BAR_ROWS = Math.floor(600 * 0.06); + function mockRegistry(): Registry { return { invokeTool: vi.fn(async (id: string) => { @@ -296,6 +315,39 @@ steps: expect(step.warning).not.toContain("never held still"); }); + // The runner pins the status bar for the whole run, but the pin lands a few + // hundred milliseconds AFTER the run starts and a nested `tool: flow-execute` + // clears it for the rest of the outer run — so this step regularly compared a + // real clock against a pinned one, or against a ticking one, and reported a + // static screen as moving or as carrying a spinner. The band is masked. + it("ignores the system status bar the run's own pin repaints", async () => { + let tick = 0; + // 400 pixels — over the motion budget for this frame — confined to the + // masked band, which is where the measured clock repaint landed. + currentFrame = () => { + const level = tick++ % 2 === 0 ? 0 : 255; + const [width, height] = [300, 600]; + const data = Buffer.alloc(width * height * 4, 255); + for (let i = 0; i < 400; i++) { + const o = (width + i) * 4; // row 1, inside the status bar + data[o] = level; + data[o + 1] = level; + data[o + 2] = level; + } + return { width, height, data }; + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step).toMatchObject({ kind: "idle", status: "pass" }); + expect(step.warning).toBeUndefined(); + }); + it("says nothing about small motion when the screen is genuinely still", async () => { currentFrame = () => frameWithMovingPixels(0, 0); await writeFlow( diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index 6c5f73cc7..7d049acfc 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -12,6 +12,7 @@ import { PIXEL_THRESHOLD, comparePixels, pixelCaptureTimeoutMs, + statusBarMaskFraction, type PixelFrame, } from "../../src/tools/flows/flow-pixels"; import { isTvOsSimulator } from "../../src/utils/ios-devices"; @@ -202,6 +203,103 @@ describe("comparePixels", () => { // pixels of 198k is an order of magnitude under a caret. expect(comparePixels(...changed(IPHONE, 3))).toBe("still"); }); + + // The two frames below are the ones the runner was actually caught + // comparing on a static iPhone 16 Pro screen: the run-level status-bar pin + // lands a few hundred milliseconds AFTER the run starts, so frame A holds + // the real clock and frame B the pinned one. Both changes sit inside the + // top band, which is why masking it is what fixes them. + describe("with the status bar masked", () => { + const MASK = 0.06; + + /** Change `count` pixels confined to rows [top, bottom] of an iPhone frame. */ + function changedInRows(count: number, top: number): [PixelFrame, PixelFrame] { + const before = solid(IPHONE[0], IPHONE[1], [255, 255, 255]); + const after = solid(IPHONE[0], IPHONE[1], [255, 255, 255]); + for (let i = 0; i < count; i++) { + const o = (top * IPHONE[0] + i) * 4; + after.data[o] = 0; + after.data[o + 1] = 0; + after.data[o + 2] = 0; + } + return [before, after]; + } + + it("stops the pin's own clock repaint from reading as a moving screen", () => { + // 408 changed pixels at y[19..29] — over the 396-pixel motion budget, + // so unmasked this static screen was judged to be in motion. + const frames = changedInRows(408, 19); + expect(comparePixels(...frames)).toBe("moving"); + expect(comparePixels(...frames, MASK)).toBe("still"); + }); + + it("stops the pin's battery-fill tail from reading as a spinner", () => { + // The same repaint a moment later: 13 pixels at y[19..25], which is + // over the localized floor and became "a spinner, a caret, a progress + // dot ... the screen had not finished loading" on a loaded screen. + const frames = changedInRows(13, 19); + expect(comparePixels(...frames)).toBe("localized"); + expect(comparePixels(...frames, MASK)).toBe("still"); + }); + + it("still sees a spinner just below the masked band", () => { + // The mask must cost the check only the system's own band. 39 rows of + // 656 are masked, so a spinner at row 40 is still fully visible. + expect(comparePixels(...changedInRows(66, 40), MASK)).toBe("localized"); + }); + + it("still sees a transition below the masked band", () => { + const frames = changedInRows(0, 0); + for (let i = 0; i < Math.round(IPHONE[0] * IPHONE[1] * 0.01); i++) { + const o = (39 * IPHONE[0] + i) * 4; + frames[1].data[o] = 0; + frames[1].data[o + 1] = 0; + frames[1].data[o + 2] = 0; + } + expect(comparePixels(...frames, MASK)).toBe("moving"); + }); + + it("takes its fractions against the unmasked area, not the whole frame", () => { + // 1% of the REMAINING rows must still read as motion; measuring against + // the full frame would quietly raise every threshold by the mask. + const visible = IPHONE[0] * (IPHONE[1] - 39); + expect(comparePixels(...changedInRows(Math.round(visible * 0.0025), 39), MASK)).toBe( + "moving" + ); + }); + }); + }); +}); + +describe("statusBarMaskFraction", () => { + // Only iOS and Android paint a status bar into the capture. Masking a + // Chromium window's top band would hide page content, and Vega / tvOS render + // full-screen with no system chrome at all. + it("masks the band on Android", async () => { + await expect( + statusBarMaskFraction({ platform: "android", kind: "emulator", id: "emulator-5554" }) + ).resolves.toBe(0.06); + }); + + it("masks the band on an iOS simulator", async () => { + vi.mocked(isTvOsSimulator).mockResolvedValue(false); + await expect( + statusBarMaskFraction({ platform: "ios", kind: "simulator", id: "ios-udid" }) + ).resolves.toBe(0.06); + }); + + it("masks nothing on a tvOS simulator, which shares the iOS platform tag", async () => { + vi.mocked(isTvOsSimulator).mockResolvedValue(true); + await expect( + statusBarMaskFraction({ platform: "ios", kind: "simulator", id: "tv-udid" }) + ).resolves.toBe(0); + }); + + it.each(["chromium", "vega"] as const)("masks nothing on %s", async (platform) => { + await expect( + statusBarMaskFraction({ platform, kind: "unknown", id: "some-device" }) + ).resolves.toBe(0); + expect(isTvOsSimulator).not.toHaveBeenCalled(); }); }); From cd1f7b58f382903019d84f5f4345077e5085d06e Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:00:53 +0200 Subject: [PATCH 78/98] fix(flow): bound the tvOS settle capture so a wedged xcrun is killed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tvOS arm passed no abort signal to `tvScreenshot`, which forwards it straight to `execFileAsync`. A wedged `xcrun simctl io screenshot` was therefore never killed: the round abandoned the promise, but nothing abandoned the process, and the next poll 200ms later spawned another — one stuck capture becoming a growing pile of subprocesses for the rest of the step. The `screenshot` tool's own tvOS route has always bounded it. Thread the capture's own budget down to the shell-out and combine it with the run's signal, so the subprocess dies with the round that stopped waiting for it. The simulator-server arm deliberately keeps its no-signal ownership (it learns the temp path only from the reply); this route's path is deterministic and internal, and an unbounded process is the worse trade. --- .../src/tools/flows/flow-pixels.ts | 33 ++++++++++--- .../test/flows/flow-pixels.test.ts | 47 ++++++++++++++++++- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index 995622e0b..ee9e213c4 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -240,14 +240,23 @@ async function chromiumScrollOffset(api: ChromiumCdpApi): Promise<{ x: number; y * The `screenshot` tool itself is deliberately not reused: it registers every * capture as an artifact, and a settle takes tens of them per step. */ -async function captureFile(env: ActionEnv): Promise { +async function captureFile(env: ActionEnv, budgetMs: number): Promise { if (env.device.platform === "vega") { return captureVegaScreenshotPng({ scale: CAPTURE_SCALE }); } // Shape alone cannot tell tvOS from iOS — both are 8-4-4-4-12 UUIDs tagged // `platform: "ios"` — so ask the runtime, which is memoized per UDID. if (env.device.platform === "ios" && (await isTvOsSimulator(env.device.id))) { - return tvScreenshot(env.device.id, CAPTURE_SCALE, undefined); + // This one DOES take the signal, unlike the simulator-server arm below. + // `tvScreenshot` forwards it to `execFileAsync`, and without it a wedged + // `xcrun simctl io screenshot` is never killed — the round abandons the + // promise and the next poll 200ms later spawns another, so a stuck + // subprocess becomes a growing pile of them. The `screenshot` tool's own + // tvOS route has always bounded it the same way. Nothing is orphaned that + // is not already: on a severed capture the temp path never comes back, but + // the file is a deterministic one under tmpdir and the alternative is an + // unbounded process. + return tvScreenshot(env.device.id, CAPTURE_SCALE, captureAbortSignal(env, budgetMs)); } const ref = simulatorServerRef(env.device); const api = (await env.registry.resolveService(ref.urn, ref.options)) as SimulatorServerApi; @@ -268,9 +277,9 @@ async function captureFile(env: ActionEnv): Promise { * which is scratch and never an artifact — it is removed as soon as it has * been read, whether or not the read worked. */ -async function capturePng(env: ActionEnv): Promise { +async function capturePng(env: ActionEnv, budgetMs: number): Promise { if (env.device.platform === "chromium") return captureChromiumPng(env); - const file = await captureFile(env); + const file = await captureFile(env, budgetMs); try { return await fs.readFile(file); } finally { @@ -283,9 +292,9 @@ async function capturePng(env: ActionEnv): Promise { * read (any capture or decode failure). Soft by design — the caller treats it * as the ABSENCE of visual evidence, never as evidence of stillness. */ -async function capturePixels(env: ActionEnv): Promise { +async function capturePixels(env: ActionEnv, budgetMs: number): Promise { try { - const png = PNG.sync.read(await capturePng(env)); + const png = PNG.sync.read(await capturePng(env, budgetMs)); return { width: png.width, height: png.height, data: png.data }; } catch { return undefined; @@ -307,10 +316,20 @@ export async function capturePixelsWithin( ): Promise { const budget = Math.min(deadline - Date.now(), pixelCaptureTimeoutMs(env.device, firstCapture)); if (budget <= 0) return undefined; - const result = await settleWithin(capturePixels(env), budget, env.signal); + const result = await settleWithin(capturePixels(env, budget), budget, env.signal); return result.type === "value" ? result.value : undefined; } +/** + * The signal a shell-out capture is bounded by: the run's own abort, plus this + * capture's budget, so the subprocess dies with the round that stopped waiting + * for it rather than outliving the whole step. + */ +function captureAbortSignal(env: ActionEnv, budgetMs: number): AbortSignal { + const bound = AbortSignal.timeout(budgetMs); + return env.signal ? AbortSignal.any([env.signal, bound]) : bound; +} + /** * How much of the screen changed between two captures. * diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index 7d049acfc..b5bfa6c9b 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -434,10 +434,55 @@ describe("capturePixels routing", () => { width: 2, height: 1, }); - expect(tvScreenshot).toHaveBeenCalledWith("tv-udid", 0.25, undefined); + expect(tvScreenshot).toHaveBeenCalledWith("tv-udid", 0.25, expect.any(AbortSignal)); expect(resolveService).not.toHaveBeenCalled(); }); + // `tvScreenshot` forwards its signal to `execFileAsync`. Without one, a + // wedged `xcrun simctl io screenshot` is never killed and the next poll + // 200ms later spawns another, so one stuck subprocess becomes a pile of + // them — the round abandons the promise, but nothing abandons the process. + it("kills a wedged tvOS capture when its budget runs out", async () => { + vi.mocked(isTvOsSimulator).mockResolvedValue(true); + let signal: AbortSignal | undefined; + vi.mocked(tvScreenshot).mockImplementation( + (_udid, _scale, sig) => + new Promise((_resolve, reject) => { + signal = sig; + sig?.addEventListener("abort", () => reject(new Error("aborted"))); + }) + ); + const env = envFor({ platform: "ios", kind: "simulator", id: "tv-udid" }); + + expect(await capturePixelsWithin(env, Date.now() + 50, false)).toBeUndefined(); + // The budget bounds the round and the subprocess with the same deadline, + // so which of the two timers lands first is not fixed — only that the + // capture does not outlive the round it belonged to. + await vi.waitFor(() => expect(signal?.aborted).toBe(true)); + }); + + it("kills a tvOS capture when the run itself is cancelled", async () => { + vi.mocked(isTvOsSimulator).mockResolvedValue(true); + const controller = new AbortController(); + let signal: AbortSignal | undefined; + vi.mocked(tvScreenshot).mockImplementation( + (_udid, _scale, sig) => + new Promise((_resolve, reject) => { + signal = sig; + sig?.addEventListener("abort", () => reject(new Error("aborted"))); + }) + ); + const env = { ...envFor({ platform: "ios", kind: "simulator", id: "tv-udid" }) } as ActionEnv; + (env as { signal?: AbortSignal }).signal = controller.signal; + + const pending = capturePixelsWithin(env, Date.now() + 30_000, false); + await vi.waitFor(() => expect(signal).toBeDefined()); + controller.abort(); + + expect(await pending).toBeUndefined(); + expect(signal?.aborted).toBe(true); + }); + it("routes Vega to the emulator console, and never probes the iOS runtime for it", async () => { vi.mocked(captureVegaScreenshotPng).mockImplementation(async () => pngAt(tmpDir, "vega.png")); const resolveService = vi.fn(() => { From a12d9183ac6273136e69f2c66f0da0cd8e32edcc Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:04:39 +0200 Subject: [PATCH 79/98] fix(flow): print a passing step's warning in a directory run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A directory run's step renderer skipped everything that was not `fail`/`error` before reaching the line that prints a warning, while the summary counts warnings whatever the status. `await: { idle: true }` only ever warns on a step that PASSED — so `argent flow run ` reported "1 warning" with the text nowhere on screen, and the warning is the whole of what the step produces. Single-flow runs and the MCP surface both printed it correctly. Verified end to end with the CLI against a live Chromium target: a directory holding a warning flow and a clean one now prints the idle warning under its step and still counts it in the summary. --- packages/argent-cli/src/flow.ts | 7 ++++++- packages/argent-cli/test/flow-render.test.ts | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/argent-cli/src/flow.ts b/packages/argent-cli/src/flow.ts index ac97a78ca..ee427653c 100644 --- a/packages/argent-cli/src/flow.ts +++ b/packages/argent-cli/src/flow.ts @@ -344,6 +344,11 @@ export function renderArtifactLines(report: FlowReport): string[] { * Batch mode prints only what needs attention: each fail/error step with its * under-lines, numbered by walking the full step list so the numbers match a * single-mode rerun of the same flow. + * + * A PASSING step carrying a warning needs attention too. `await: { idle: true }` + * only ever warns on a step that passed, and renderSummary counts every warning + * whatever its status — so skipping those here printed "1 warning" with the + * text nowhere on screen, which is the whole of what the step reports. */ export function renderFailedSteps(report: FlowReport): string[] { const lines: string[] = []; @@ -351,7 +356,7 @@ export function renderFailedSteps(report: FlowReport): string[] { for (const s of report.steps) { if (s.kind === "echo") continue; n++; - if (s.status !== "fail" && s.status !== "error") continue; + if (s.status !== "fail" && s.status !== "error" && !s.warning) continue; lines.push(renderStepLine(s, n, report.flow)); if (s.warning) lines.push(renderUnderStepLine(s, n, `⚠ ${s.warning}`)); if (s.artifacts && typeof s.artifacts === "object") { diff --git a/packages/argent-cli/test/flow-render.test.ts b/packages/argent-cli/test/flow-render.test.ts index 0f59a3d28..33662f101 100644 --- a/packages/argent-cli/test/flow-render.test.ts +++ b/packages/argent-cli/test/flow-render.test.ts @@ -296,6 +296,21 @@ describe("flow report rendering", () => { expect(renderFailedSteps(mkReport([{ index: 0, kind: "tap", status: "pass" }]))).toEqual([]); }); + it("renderFailedSteps prints a passing step's warning, which renderSummary counts", () => { + // `await: { idle: true }` only ever warns on a step that PASSED, and the + // summary counts warnings whatever the status — so a directory run used to + // report "1 warning" with the text nowhere on screen. + const report = mkReport([ + { index: 0, kind: "tap", status: "pass" }, + { index: 1, kind: "idle", status: "pass", warning: "the screen never held still" }, + ]); + expect(renderFailedSteps(report)).toEqual([ + " ⚠ 2 idle", + " ⚠ the screen never held still", + ]); + expect(renderSummary(report)).toContain("1 warning"); + }); + it("renderBatchSummary mirrors the step summary's verdict shape", () => { expect(renderBatchSummary({ total: 3, passed: 2, failed: 1, skipped: 0 })).toBe( "FAIL — 3 flows: 2 passed, 1 failed, 0 skipped" From f332de364c03e299540e29611d88226ecb16f32b Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:08:39 +0200 Subject: [PATCH 80/98] fix(flow): stop a blank read from counting as a look at the screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "too few reads to judge anything" guard counted every read that ANSWERED, including the blank ones — but a blank tree resets both holds and measures no interval, so it is not one of the three reads a settle takes. A window that was blank for most of its life therefore sailed past the guard and reached the motion verdict instead, telling the author that "the screen never held still ... something on it never stops" on the strength of a single measured interval. Count content-bearing reads for that guard, leaving the reads-that-answered count to the unreadable-source check above it, which is the question it actually asks. The warning now says how many reads came back with content, which is the number the advice ("raise its `timeout:`") acts on. --- .../src/tools/flows/flow-actions.ts | 28 ++++++++++++++----- .../test/flows/flow-idle-run.test.ts | 26 ++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 3b93ba208..82f17aa7a 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1277,6 +1277,11 @@ async function waitForIdle( let localizedMotionDuringHold = false; let readsSucceeded = 0; + // Reads that came back with a tree AND something in it. Only these can + // measure an interval, so this — not readsSucceeded — is what the + // "too few reads to judge" guard at the bottom counts. A blank read is an + // observation (it resets both holds) but never evidence about motion. + let contentReads = 0; // Definitely assigned: the loop below always completes at least one round, // and every arm of that round sets it. let lastRead!: TreeReadOutcome; @@ -1354,6 +1359,7 @@ async function waitForIdle( stillIntervals = 0; } else { sawContent = true; + contentReads += 1; const signature = treeFingerprint(tree); const now = Date.now(); @@ -1493,16 +1499,24 @@ async function waitForIdle( // way — and both verdicts below would be claims about an app that was never // observed for long enough to make one. The parser rejects a `timeout:` too // short to fit a settle, so what reaches here is a source slow enough to eat - // the wait, which is worth saying rather than dressing up as motion. - if (readsSucceeded <= MIN_STILL_INTERVALS) { + // the wait, or a window blank for most of it, either of which is worth + // saying rather than dressing up as motion. + // + // Counted in reads that CARRIED CONTENT, not in reads that answered: a blank + // one resets both holds and measures no interval. Counting it let a window + // that was blank for all but its last two reads sail past this guard and + // assert instead that "the screen never held still ... something on it never + // stops" — a claim about motion drawn from a single measured interval. + if (contentReads <= MIN_STILL_INTERVALS) { return { ok: true, warning: - `the screen was read ${readsSucceeded} time${readsSucceeded === 1 ? "" : "s"} in ` + - `${timeoutMs}ms, and a settle takes ${MIN_STILL_INTERVALS + 1} reads spanning ` + - `${MIN_STILL_INTERVALS} ${IDLE_POLL_MS}ms polls — so this step ended without ever being ` + - `able to tell whether the screen was moving. Raise its \`timeout:\`, and gate the next ` + - `action on a stable element rather than on stillness.`, + `the screen came back with content on ${contentReads} read` + + `${contentReads === 1 ? "" : "s"} in ${timeoutMs}ms, and a settle takes ` + + `${MIN_STILL_INTERVALS + 1} of them spanning ${MIN_STILL_INTERVALS} ${IDLE_POLL_MS}ms ` + + `polls — so this step ended without ever being able to tell whether the screen was ` + + `moving. Raise its \`timeout:\`, and gate the next action on a stable element rather ` + + `than on stillness.`, }; } // The tree was settled as of the last read and no pair of captures ever diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index c12a88706..834da89aa 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -472,13 +472,37 @@ steps: ); const step = (await run("ready")).steps.at(-1)!; expect(step.status).toBe("pass"); - expect(step.warning).toContain("read 1 time in 1200ms"); + expect(step.warning).toContain("content on 1 read in 1200ms"); expect(step.warning).toContain("`timeout:`"); // The screen was static the whole time; it must not be described as moving. expect(step.warning).not.toContain("never held still"); expect(step.warning).not.toContain("no pair of screenshots"); }); + // A blank read is an observation — it resets both holds — but it measures no + // interval, so it is not one of the three a settle takes. Counting it let a + // window blank for all but its last two reads slip past the guard above and + // reach for the motion verdict instead, telling the author that "something on + // it never stops" on the strength of one measured interval. + it("does not count a blank read as a look at the screen", async () => { + let reads = 0; + // A 250ms read plus a 200ms poll fits three rounds in 1200ms; the first + // comes back blank, so only one interval could ever be measured. + treeDelayMs = 250; + currentTree = () => (reads++ < 1 ? n({ role: "AXWindow", frame: FULL }) : screenWith("Home")); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1200, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("content on 2 reads in 1200ms"); + expect(step.warning).not.toContain("never held still"); + }); + it("settles on the tree alone when no screenshot can be captured, and says so", async () => { currentFrame = () => undefined; await writeFlow( From b4a0068549f1bc3647eb60ea03d2ec7e2c94a355 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:10:34 +0200 Subject: [PATCH 81/98] fix(flow): end an assert+idle mix in one error instead of two MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assert: { idle: true, visible: X }` reported the mixing error first, and doing what it said — splitting the two conditions — produced `assert: { idle: true }`, which is not valid either: idle has no assert form. The assert arm of the mixing check could never yield a body that subsequently parsed. Let the assert body go straight to the error that ends the matter, and have that error name where the other condition belongs, so one message describes the whole repair. The `await` mixing error is unchanged, and the fix is pinned by parsing the exact spelling the new message tells the author to write. --- .../tool-server/src/tools/flows/flow-utils.ts | 16 +++++++++++++++- .../test/flows/flow-idle-condition.test.ts | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 58709d70e..0a90b1188 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -1771,9 +1771,19 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") const entry = { [kind]: raw }; if (kind !== "await") { + // Name the other condition's home too when the body carries one. Reporting + // the mixing error first sent the author to a second round trip: splitting + // `assert: { idle: true, visible: X }` as instructed yields + // `assert: { idle: true }`, which has no assert form either. + const mixed = WAIT_CONDITIONS.filter((c) => c in raw); badEntry( entry, - "idle has no assert form — it waits for the screen to stop changing, which is an `await`" + "idle has no assert form — it waits for the screen to stop changing, which is an `await`" + + (mixed.length > 0 + ? `. Give it its own step as \`await: { idle: true }\` and leave \`${mixed.join( + "`, `" + )}\` in the assert — a step checks exactly one condition` + : "") ); } rejectUnknownKeys(entry, raw, ["idle", "minStableMs", "timeout"], kind); @@ -1832,6 +1842,10 @@ function isIdleCondition(raw: unknown, kind: "await" | "assert"): boolean { if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return false; const body = raw as Record; if (!(IDLE_CONDITION in body)) return false; + // An `assert` body naming idle is wrong however it is spelled, so let + // parseIdleFields raise the one error that ends the matter — it folds the + // mixing advice in rather than making the author earn it on a second run. + if (kind === "assert") return true; const selectorConditions = WAIT_CONDITIONS.filter((c) => c in body); if (selectorConditions.length > 0) { badEntry( diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index dbccfa8a0..bf85b8953 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -123,6 +123,25 @@ describe("condition families are mutually exclusive", () => { ); }); + // The same mix under `assert` used to cost two round trips: it reported the + // mixing first, and the split the author was told to write — + // `assert: { idle: true }` — is not valid either. One error has to end it. + it("sends an assert body that mixes the two straight to the form it needs", () => { + let message = ""; + try { + parseSteps(` - assert: { idle: true, visible: { id: x } }\n`); + } catch (err) { + message = err instanceof Error ? err.message : String(err); + } + expect(message).toContain("idle has no assert form"); + expect(message).toContain("await: { idle: true }"); + expect(message).toContain("`visible`"); + // And what it tells the author to write must itself parse. + expect(() => + parseSteps(` - await: { idle: true }\n - assert: { visible: { id: x } }\n`) + ).not.toThrow(); + }); + it("rejects a stray key rather than ignoring it", () => { expect(() => parseSteps(` - await: { idle: true, settleMs: 500 }\n`)).toThrow(/settleMs/); }); From 741f125ac44a7e11a7227c133b14b892700817e5 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:12:37 +0200 Subject: [PATCH 82/98] fix(flow): size the localized-motion floor in pixels, not in frame area MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spinner/caret warning's floor was a fraction of frame area, so it grew with the window: 0.005% is ~10 pixels on a phone frame but ~46 on a desktop-sized Chromium one — above every indicator ever measured for it, which turned the warning silently off on exactly the largest windows. Both numbers the comment cited to justify the fraction were phone-sized. What a spinner and a caret have in common is a size in captured pixels, not a share of the frame: 50-66 pixels for a spinner on an iPhone 16 Pro, 57 on a Pixel 5, 50 for a caret on a Pixel 7 — within a factor of two across three frame sizes. Make the floor that count. A static capture pair changed 0 pixels of 237k, so ten still clears the noise it exists for. MOTION_FRACTION stays a fraction: a transition really does move a share of the screen. --- .../src/tools/flows/flow-pixels.ts | 39 +++++++++++-------- .../test/flows/flow-pixels.test.ts | 13 +++++++ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/packages/tool-server/src/tools/flows/flow-pixels.ts b/packages/tool-server/src/tools/flows/flow-pixels.ts index ee9e213c4..73e03b5bb 100644 --- a/packages/tool-server/src/tools/flows/flow-pixels.ts +++ b/packages/tool-server/src/tools/flows/flow-pixels.ts @@ -67,25 +67,30 @@ const PIXEL_THRESHOLD_SQUARED = PIXEL_THRESHOLD * PIXEL_THRESHOLD * MAX_RGB_DIST // the gate alone; loosening this fraction does not widen that blind spot. const MOTION_FRACTION = 0.002; -// Below MOTION_FRACTION but at or above this one, the change is LOCALIZED: too -// small to be the screen moving, too large to be capture noise. A stock 40pt -// spinner measures 0.03-0.15% of a phone screen and a text caret about 0.01%, -// both under MOTION_FRACTION — which is how a still-loading screen used to -// report as settled with nothing said about it. +// Below MOTION_FRACTION but at or above this many CHANGED PIXELS, the change +// is LOCALIZED: too small to be the screen moving, too large to be capture +// noise. A spinner and a caret both sit in that gap — which is how a +// still-loading screen used to report as settled with nothing said about it. // -// Both fractions are of frame AREA, which makes them resolution-independent: -// an object of a fixed on-screen size covers the same fraction of the frame -// whatever the capture scale, so the same numbers hold on a 158k-pixel Pixel -// capture and a full-resolution desktop one. +// A COUNT, not a fraction, because what these indicators have in common is a +// size in captured pixels rather than a share of the frame. Measured at +// CAPTURE_SCALE: a live spinner changed 50-66 pixels on an iPhone 16 Pro +// (302x656) and 57 on a Pixel 5 (270x585), and a blinking caret 50 on a +// Pixel 7 (162k pixels) — all within a factor of two of each other across +// three frame sizes. Expressed as a fraction those numbers only held at +// phone size: 0.005% of a desktop-sized Chromium window is ~46 pixels, above +// every indicator measured here, so the warning was silently off on exactly +// the windows that are largest. // // The floor is not zero because a capture pair is not guaranteed byte-identical -// on every backend. Measured at CAPTURE_SCALE: two captures of a static iOS -// screen changed 0 pixels of 237k, while a live spinner on an iPhone 16 Pro -// changed 50 of 198k and a blinking caret on a Pixel 7 changed 50 of 162k. A -// floor of ~10 pixels on a phone-sized frame therefore sits well clear of both -// ends. An indicator smaller than that stays invisible, which is the residual -// limit of comparing whole frames. -const LOCALIZED_MOTION_FRACTION = 0.00005; +// on every backend; two captures of a static iOS screen changed 0 pixels of +// 237k. Ten sits an order of magnitude under the smallest indicator measured +// and clear of that noise. One smaller than ten pixels stays invisible, which +// is the residual limit of comparing whole frames. +// +// MOTION_FRACTION above stays a fraction: a transition, a scroll or a +// carousel moves a share of the screen, which is what scales with it. +const LOCALIZED_MOTION_MIN_PIXELS = 10; // Top band excluded from the comparison on a device with a system status bar, // as a fraction of frame height. The same 6% screenshot-diff ignores @@ -367,5 +372,5 @@ export function comparePixels(a: PixelFrame, b: PixelFrame, maskTopFraction = 0) } const fraction = changed / total; if (fraction > MOTION_FRACTION) return "moving"; - return fraction >= LOCALIZED_MOTION_FRACTION ? "localized" : "still"; + return changed >= LOCALIZED_MOTION_MIN_PIXELS ? "localized" : "still"; } diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index b5bfa6c9b..a2f148391 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -204,6 +204,19 @@ describe("comparePixels", () => { expect(comparePixels(...changed(IPHONE, 3))).toBe("still"); }); + // The floor is a pixel COUNT, not a share of the frame: a spinner and a + // caret are the same handful of captured pixels whatever window they sit + // in. As a fraction it only held at phone size — on a desktop-sized + // Chromium window the same 0.005% is ~46 pixels, above every indicator + // ever measured, so the warning was silently off on the largest windows. + it("still sees a caret on a desktop-sized window", () => { + const DESKTOP = [1200, 767] as const; // 920k px, where the old floor was ~46 + expect(comparePixels(...changed(DESKTOP, 10))).toBe("localized"); + expect(comparePixels(...changed(DESKTOP, 45))).toBe("localized"); + // And the floor still holds at that size: noise stays noise. + expect(comparePixels(...changed(DESKTOP, 9))).toBe("still"); + }); + // The two frames below are the ones the runner was actually caught // comparing on a static iPhone 16 Pro screen: the run-level status-bar pin // lands a few hundred milliseconds AFTER the run starts, so frame A holds From c93d5158dc0b4c523f7a0ea714c9d6fe46387766 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:23:50 +0200 Subject: [PATCH 83/98] docs(flow): make every account of the idle step add up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six descriptions of the same check disagreed with it or with each other. - The skill listed four warnings; the implementation raises five. The too-few-reads one is now listed, so the skill and the code comment that enumerates them describe the same set. - The skill said only a source that fails outright or wedges stops the run. There is a third path — one that never answers within the step — whose advice is the opposite: it may merely be slow, so raise the `timeout`. Named, along with what an `error` costs (the run is not ok, later steps skipped). - "the 600ms a settle costs — three reads spanning two 200ms polls" adds to 400. The missing term is the budget the closing round needs to be allowed to start; the skill and the parser error both say so now, and a test pins the boundary the arithmetic claims against the one the parser enforces. - The `flow-execute` description — the surface an authoring agent reads — promised idle "never fails a run" without mentioning that an unreadable tree scores `error` and skips every remaining step. The skill carried the caveat; now both do. - The localized-motion warning said something "kept changing the whole time" while the flag is set by any one interval of the winning hold. It now claims what the flag means: something small was moving while the screen settled. - The spinner's size was stated four different ways. It is the measured one, in the units the floor is now expressed in. The docs test that only phrase-matched two documents now binds the numbers to the parser's own constants and checks the smallest timeout the documented arithmetic allows against the smallest the parser accepts. --- .../skills/skills/argent-create-flow/SKILL.md | 7 ++-- .../src/tools/flows/flow-actions.ts | 15 +++++-- .../tool-server/src/tools/flows/flow-run.ts | 20 +++++---- .../tool-server/src/tools/flows/flow-utils.ts | 4 +- .../test/flows/flow-idle-run.test.ts | 2 +- .../test/flows/flow-skill-docs.test.ts | 42 ++++++++++++++++++- 6 files changed, 71 insertions(+), 19 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 92a342cfd..0749b843d 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -85,16 +85,17 @@ This condition-as-key form is the only spelling. `await` also accepts an optiona For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. -**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. +**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls, plus the 200ms of budget the closing round has to have left to be allowed to start — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. It **never fails a run.** Readiness is not an acceptance criterion, so every outcome short of a clean settle passes carrying a `warning` on the step — read it rather than stepping over it: - **the screen never held still** — it spent the timeout and went ahead. Plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text); a screen that never finished loading looks the same from here. -- **a small part of it kept changing** — a spinner, a caret, a progress dot. Too small to be the screen moving, so the settle completed anyway; if it is a loading spinner, the screen was still loading when this step returned. +- **a small part of it was still changing** — a spinner, a caret, a progress dot, moving during the stretch of stillness the step settled on. Too small to be the screen moving, so the settle completed anyway; if it is a loading spinner, the screen was still loading when this step returned. - **the tree stayed empty** — the screen rendered no accessible content. Sometimes the app (a canvas, a video surface), sometimes a screen that never arrived. - **settled on the UI tree alone** — the screen could not be screenshotted often enough to compare a pair, so presentation-layer motion (a push, a fade, a dismissing modal) was not waited out. +- **the screen came back with content on too few reads** — a settle takes three of them spanning two polls, and this step got fewer, so it ended without ever being able to tell whether the screen was moving. A slow tree source, or a window that was blank for most of the wait. -Only a tree source that cannot be read stops the run, as an `error` — one that fails outright, or one that answers and then wedges. That is a broken window, not a verdict about the app. +Only a tree source that cannot be read stops the run, as an `error` — one that fails outright, one that answers and then wedges, or one that never answers at all within the step (that last one may simply be slow: raise the step's `timeout` before suspecting the app). That is a broken window, not a verdict about the app: the run is not ok and every later step is skipped. It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. There is no `assert` form (waiting is the whole point), no `when:` form, and the recorder cannot emit one — every `idle` step is hand-written. Do not sprinkle it after every step: each one costs a settle, and it cannot fail, so a flow full of them is slower without being stricter. diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 82f17aa7a..ae04e8a29 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1192,15 +1192,22 @@ const MIN_STILL_INTERVALS = IDLE_MIN_STILL_INTERVALS; const MIN_ROUND_BUDGET_MS = IDLE_POLL_MS; /** - * The screen settled, but something small on it never stopped. A spinner is the - * case that matters: it is far too small to move the screen (a stock one covers - * ~0.1% of a phone display) and it does not move the tree either, since it + * The screen settled, but something small on it moved while it did. A spinner + * is the case that matters: it is far too small to move the screen (a stock one + * measured 50-66 changed pixels of a phone capture — see + * LOCALIZED_MOTION_MIN_PIXELS) and it does not move the tree either, since it * spins in a layer without its box ever changing — so both halves of the check * agree the screen is at rest while it is still loading. This is the only place * that difference is visible, so it is said outright. + * + * The claim is deliberately about the settle being reported and not about the + * whole step: the flag is set by ANY interval of the winning hold and cleared + * with the hold, so what it promises is that something small moved inside the + * stretch of stillness this step is passing on — not that it moved from the + * first read to the last. */ const LOCALIZED_MOTION_WARNING = - `the screen settled, but a small part of it kept changing the whole time — a spinner, a ` + + `the screen settled, but a small part of it was still changing while it did — a spinner, a ` + `caret, a progress dot. If it is a loading spinner then the screen had not finished loading, ` + `and stillness cannot tell those apart: look at what is moving, and gate the next action on ` + `the element the loading produces rather than on this settle.`; diff --git a/packages/tool-server/src/tools/flows/flow-run.ts b/packages/tool-server/src/tools/flows/flow-run.ts index 793ba4761..e7cfb6413 100644 --- a/packages/tool-server/src/tools/flows/flow-run.ts +++ b/packages/tool-server/src/tools/flows/flow-run.ts @@ -190,13 +190,13 @@ export interface StepReport { reason?: string; /** * The step passed, but the WAY it passed weakens it as proof. Rendered as a - * "⚠" suffix by the MCP client. Raised by `await: { idle: true }`: the screen - * never settled at all (it waits, then goes ahead); something small on it - * never stopped, which is what a spinner looks like; it rendered no content - * to settle; the step ran out of reads before it could judge anything; or its - * captures never produced a comparable pair, leaving stillness proved on the - * UI tree alone without the presentation-layer motion the pixel half exists - * to catch. + * "⚠" suffix by the MCP client, and under the step line by the CLI. Raised by + * `await: { idle: true }`: the screen never settled at all (it waits, then + * goes ahead); something small on it never stopped, which is what a spinner + * looks like; it rendered no content to settle; too few reads came back with + * content for it to judge anything; or its captures never produced a + * comparable pair, leaving stillness proved on the UI tree alone without the + * presentation-layer motion the pixel half exists to catch. */ warning?: string; /** Underlying tool id for `tool` steps. */ @@ -929,8 +929,10 @@ when \`on\` is omitted; distinct from the \`rotate\` tool, which changes device for a UI condition, and additionally takes the one condition that has no selector: \`idle: true\` waits until the screen has content and stops moving in BOTH the UI tree and the rendered pixels (it never fails a run — a screen that never settles passes carrying a \`warning\`, which is what makes it safe to -persist; it says nothing about WHICH screen settled — a dropped tap leaves the source screen perfectly -idle — so pair it with the element check that names the destination); \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` +persist; the one outcome that does stop the run is an \`error\` for a tree source that could not be read +at all — a broken window rather than a verdict about the app, which leaves the run not-ok and skips +every later step; it says nothing about WHICH screen settled — a dropped tap leaves the source screen +perfectly idle — so pair it with the element check that names the destination); \`wait\` pauses for a fixed number of milliseconds; \`assert\` checks one now; \`snapshot\` diffs a screenshot — or, with \`cropOn: \`, one element's cropped region — against a stored baseline (a missing baseline fails the step — set updateBaselines to adopt the current screen; a cropped element whose size drifted fails on dimensions); \`echo\` annotates; \`run\` executes another flow diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index 0a90b1188..a2474e460 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -1826,7 +1826,9 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") `idle needs a timeout of at least ${needed}ms to hold still for ` + `${step.minStableMs === undefined ? `the default ` : ``}${minStableMs}ms: a settle is ` + `${IDLE_MIN_STILL_INTERVALS + 1} reads spanning ${IDLE_MIN_STILL_INTERVALS} ` + - `${IDLE_POLL_MS}ms polls, and the wait has to contain them as well as the hold. Raise ` + + `${IDLE_POLL_MS}ms polls, plus the ${IDLE_POLL_MS}ms of budget the closing round has to ` + + `have left to be allowed to start, and the wait has to contain all of that as well as ` + + `the hold. Raise ` + `\`timeout\`${step.minStableMs === undefined ? "" : " or lower `minStableMs`"}` ); } diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index 834da89aa..e6a185fb1 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -309,7 +309,7 @@ steps: expect(r.ok).toBe(true); const step = r.steps.at(-2)!; expect(step).toMatchObject({ kind: "idle", status: "pass" }); - expect(step.warning).toContain("small part of it kept changing"); + expect(step.warning).toContain("small part of it was still changing"); expect(step.warning).toContain("spinner"); // The warning is about how the settle was reached, not a refusal to settle. expect(step.warning).not.toContain("never held still"); diff --git a/packages/tool-server/test/flows/flow-skill-docs.test.ts b/packages/tool-server/test/flows/flow-skill-docs.test.ts index 5d8247337..085310c27 100644 --- a/packages/tool-server/test/flows/flow-skill-docs.test.ts +++ b/packages/tool-server/test/flows/flow-skill-docs.test.ts @@ -2,7 +2,14 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; import * as path from "node:path"; import type { Registry } from "@argent/registry"; -import { parseFlow } from "../../src/tools/flows/flow-utils"; +import { + IDLE_DEFAULT_MIN_STABLE_MS, + IDLE_DEFAULT_TIMEOUT_MS, + IDLE_MIN_STILL_INTERVALS, + IDLE_POLL_MS, + IDLE_SETTLE_OVERHEAD_MS, + parseFlow, +} from "../../src/tools/flows/flow-utils"; import { createRunFlowTool } from "../../src/tools/flows/flow-run"; /** @@ -60,6 +67,39 @@ describe("create-flow SKILL.md scope snippets", () => { expect(skill).toContain("It **never fails a run.**"); // The one outcome that does stop a run is the window, never the app. expect(skill).toMatch(/Only a tree source that cannot be read stops the run/); + // Both surfaces have to carry that caveat: the description is what an + // authoring agent reads, and "never fails a run" on its own is not true + // of a tree nobody could read. + expect(description).toMatch(/unreadable|cannot be read|could not be read/); + }); + + // The claims above are prose until something ties them to the runner. These + // pin the numbers the skill quotes to the constants the parser enforces, so + // a default that moves takes the sentence describing it with it. + it("the skill's idle defaults and settle cost are the ones the parser enforces", () => { + const skill = readFileSync(SKILL, "utf8"); + expect(skill).toContain(`default ${IDLE_DEFAULT_MIN_STABLE_MS}`); + expect(skill).toContain(`default ${IDLE_DEFAULT_TIMEOUT_MS}`); + expect(skill).toContain(`${IDLE_SETTLE_OVERHEAD_MS}ms a settle costs`); + expect(skill).toContain(`${IDLE_POLL_MS}ms polls`); + // The gloss has to add up to the cost it explains: the polls the intervals + // span, plus the round-start floor. Without the second term it described + // 400ms while demanding 600. + expect(IDLE_SETTLE_OVERHEAD_MS).toBe((IDLE_MIN_STILL_INTERVALS + 1) * IDLE_POLL_MS); + expect(skill).toContain(`plus the ${IDLE_POLL_MS}ms of budget the closing round`); + }); + + it("the smallest timeout the skill's arithmetic allows is the one the parser accepts", () => { + // The skill tells an author the wait has to contain the hold plus the + // settle. Take it at its word and check the boundary both ways — a parser + // that demanded a millisecond more would make the documented sum a lie. + const smallest = IDLE_DEFAULT_MIN_STABLE_MS + IDLE_SETTLE_OVERHEAD_MS; + expect(() => + parseFlow(`steps:\n - await: { idle: true, timeout: ${smallest} }\n`) + ).not.toThrow(); + expect(() => + parseFlow(`steps:\n - await: { idle: true, timeout: ${smallest - 1} }\n`) + ).toThrow(new RegExp(`at least ${smallest}ms`)); }); it("the paragraph's rejected `any` spelling really is rejected", () => { From 22f38d51fcb67ce5012ce353c7b18b70a0d443db Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Thu, 6 Aug 2026 18:25:30 +0200 Subject: [PATCH 84/98] test(flow): hold the idle branches nothing was watching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The capture-noise case asserted only `r.ok`, which is true for every idle outcome but an unreadable tree. What it is about is that a +1 drift settles CLEANLY, so it now asserts no warning at all — neither motion nor spinner. - The localized-motion reset when a hold breaks: a spinner that STOPS, then a screen that goes still, must settle without being told something was moving while it did. Deleting the reset now fails. - The `minStableMs` term of the tree-only hold: every case that reached it ran with `minStableMs: 0`, where the term is vacuous. A tree that had only just stopped moving must not be reported as having settled on the hierarchy. - The reads a blank window does not buy, the `idle` step's absent report target, the ten-minute ceiling on `minStableMs`, Vega and tvOS in the warm-capture decision, a Chromium capture that answers with no data, a frame with no pixels to compare, a truncated capture buffer, and the recorder summary for a hand-written idle step — each was deletable with the suite still green. Every case here was checked by making the change it guards against and watching it fail. --- .../test/flows/flow-idle-condition.test.ts | 10 +++ .../test/flows/flow-idle-run.test.ts | 68 ++++++++++++++++++- .../test/flows/flow-pixels.test.ts | 54 +++++++++++++++ .../tool-server/test/flows/flow-tools.test.ts | 22 ++++++ 4 files changed, 153 insertions(+), 1 deletion(-) diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index bf85b8953..6db72b04e 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -51,6 +51,16 @@ describe("await { idle }", () => { expect(() => parseSteps(` - await: { idle: true, minStableMs: 1.5 }\n`)).toThrow( /idle.minStableMs/ ); + // And from above, so a hold written in the wrong unit (seconds, a pasted + // timestamp) is rejected as a number rather than becoming a gate no run + // can pass. The ceiling is ten minutes; a `timeout` wide enough to contain + // it is what the case below checks separately. + expect(() => parseSteps(` - await: { idle: true, minStableMs: 600001 }\n`)).toThrow( + /between 0 and 600000/ + ); + expect(parseSteps(` - await: { idle: true, minStableMs: 600000, timeout: 600600 }\n`)).toEqual( + [{ kind: "idle", minStableMs: 600000, timeout: 600600 }] + ); }); // A wait that cannot contain the settle it asks for is a gate that fails on diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index e6a185fb1..e87241b16 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -372,7 +372,15 @@ steps: - await: { idle: true, timeout: 2000, minStableMs: 0 } ` ); - expect((await run("ready")).ok).toBe(true); + const r = await run("ready"); + expect(r.ok).toBe(true); + // `ok` alone would hold for every idle outcome but an unreadable tree, so + // it says nothing about the threshold. What this case is about is that + // +1 per channel settles CLEANLY: no motion warning, and none about a + // spinner either — noise must not be reported as something small moving. + const step = r.steps.at(-1)!; + expect(step).toMatchObject({ kind: "idle", status: "pass" }); + expect(step.warning).toBeUndefined(); }); // A reversing animation — a cross-fade, a pulse, a bounce — has a turning @@ -516,6 +524,9 @@ steps: expect(r.ok).toBe(true); const step = r.steps.at(-1)!; expect(step.status).toBe("pass"); + // The step has no target beyond the screen itself, and the renderer + // already prints the kind — a target here would read "idle screen idle". + expect(step.target).toBeUndefined(); expect(step.warning).toContain("UI tree alone"); // Attributed to the capture, not to the platform: on a device where // screenshots normally work this is a per-capture failure, not a property @@ -523,6 +534,61 @@ steps: expect(step.warning).not.toContain("could not be captured on"); }); + // The tree-only report is a claim that the HIERARCHY settled, so it owes the + // same hold every other settle does. Every case that reaches it elsewhere + // runs with `minStableMs: 0`, where the hold term is vacuous — drop it and + // the suite stays green while a tree that had only just stopped moving is + // reported as having settled. + it("does not report a tree-only settle when the hold was never served", async () => { + currentFrame = () => undefined; + // The tree keeps changing for the first 700ms, then holds. The last read + // lands ~1200ms in, so the hierarchy has been still for well under the + // 800ms hold, however many agreeing intervals it managed. + const startedAt = Date.now(); + let churn = 0; + currentTree = () => screenWith(Date.now() - startedAt < 700 ? `Loading ${churn++}` : "Settled"); + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 1400, minStableMs: 800 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).not.toContain("UI tree alone"); + expect(step.warning).toContain("never held still for 800ms"); + }); + + // The localized flag describes the hold being REPORTED, so a hold that broke + // has to clear it. Without the reset, a spinner that stopped — then a screen + // that went still — still told the author something "kept changing the whole + // time". + it("forgets small motion that stopped before the settle that gets reported", async () => { + // Round 1 has no predecessor; 2 moves a spinner's worth; 3 moves the whole + // screen, breaking the hold; 4 and 5 are identical, which is the settle. + const WHOLE_FRAME = 300 * 600; + const frames = [ + frameWithMovingPixels(0, 0), + frameWithMovingPixels(40, 0), + frameWithMovingPixels(WHOLE_FRAME, 90), + frameWithMovingPixels(WHOLE_FRAME, 90), + frameWithMovingPixels(WHOLE_FRAME, 90), + ]; + let i = 0; + currentFrame = () => frames[Math.min(i++, frames.length - 1)]; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step).toMatchObject({ kind: "idle", status: "pass" }); + expect(step.warning).toBeUndefined(); + }); + // A capture that goes missing used to cost the settle TWO intervals, not // one: the missing frame was also stored as the previous frame, so the next // round had nothing to compare against either. Holding the last good frame diff --git a/packages/tool-server/test/flows/flow-pixels.test.ts b/packages/tool-server/test/flows/flow-pixels.test.ts index a2f148391..233fb6850 100644 --- a/packages/tool-server/test/flows/flow-pixels.test.ts +++ b/packages/tool-server/test/flows/flow-pixels.test.ts @@ -108,6 +108,29 @@ describe("comparePixels", () => { expect(comparePixels(solid(30, 30, [0, 0, 0]), solid(30, 31, [0, 0, 0]))).toBe("moving"); }); + it("reads a frame with no pixels to compare as still, never as motion", () => { + // Same dimensions, so the branch above does not catch it, and there is + // nothing to count — a decoder that handed back an empty frame must not + // manufacture a verdict either way. Also the shape a full-height mask + // would take. + expect(comparePixels(solid(0, 0, [0, 0, 0]), solid(0, 0, [0, 0, 0]))).toBe("still"); + expect(comparePixels(solid(30, 30, [0, 0, 0]), solid(30, 30, [255, 255, 255]), 1)).toBe( + "still" + ); + }); + + it("compares only the bytes both frames actually carry", () => { + // Same declared dimensions but a truncated buffer — a partially decoded + // capture. Reading past the shorter one would compare against undefined + // and count NaN distances, so the loop stops at the shared length. + const short = solid(200, 200, [0, 0, 0]); + short.data = short.data.subarray(0, 40); // ten pixels' worth + expect(comparePixels(solid(200, 200, [0, 0, 0]), short)).toBe("still"); + // Only those ten differ, and ten of 40k is localized — not the whole + // frame a run past the buffer's end would report. + expect(comparePixels(solid(200, 200, [255, 255, 255]), short)).toBe("localized"); + }); + it("ignores a sub-threshold per-pixel color drift (encoder / resample noise)", () => { // +5 on every channel is well under the per-pixel tolerance, so no pixel // counts as changed — two captures of a static screen must read as still. @@ -576,6 +599,26 @@ describe("capturePixels routing", () => { expect(pixelAt(pixels, 0)).toEqual(VISIBLE_BAND_RGB); }); + it("reads a Chromium capture that came back with no data as no evidence", async () => { + // The compositor answering without `data` is a capture failure like any + // other: soft, so the settle records "no visual evidence this round" + // rather than failing the step on it. + const device: DeviceInfo = { platform: "chromium", kind: "app", id: "chromium-cdp-9222" }; + const send = vi.fn(async (method: string) => + method === "Page.getLayoutMetrics" ? { cssVisualViewport: { pageX: 0, pageY: 0 } } : {} + ); + const api = { cdp: { send }, getViewport: () => ({ width: 900, height: 700 }) }; + + expect( + await capture( + envFor( + device, + vi.fn(async () => api) + ) + ) + ).toBeUndefined(); + }); + it("falls back to the document origin when the layout metrics cannot be read", async () => { // A renderer that will not answer the metrics read leaves the capture no // worse off than never asking — an unscrolled clip, not a failed settle. @@ -688,5 +731,16 @@ describe("capturePixelsWithin", () => { expect( pixelCaptureTimeoutMs({ platform: "chromium", kind: "app", id: "chromium-cdp-9222" }, true) ).toBe(PIXEL_CAPTURE_TIMEOUT_MS); + // Nor does Vega, which shells out to the emulator console — no stream + // either. Untested, this arm could be deleted and only Chromium would tell. + expect(pixelCaptureTimeoutMs({ platform: "vega", kind: "vvd", id: "vega-serial" }, true)).toBe( + PIXEL_CAPTURE_TIMEOUT_MS + ); + // A tvOS simulator shells out too, but nothing here can tell it from an + // iOS one without an async probe, so it keeps the wider bound it will not + // spend. + expect(pixelCaptureTimeoutMs({ platform: "ios", kind: "simulator", id: "tv-udid" }, true)).toBe( + FIRST_PIXEL_CAPTURE_TIMEOUT_MS + ); }); }); diff --git a/packages/tool-server/test/flows/flow-tools.test.ts b/packages/tool-server/test/flows/flow-tools.test.ts index afdc03d46..0d7b4fc33 100644 --- a/packages/tool-server/test/flows/flow-tools.test.ts +++ b/packages/tool-server/test/flows/flow-tools.test.ts @@ -1549,6 +1549,28 @@ describe("flow-finish-recording", () => { expect(result.summary).toEqual(["1. echo: Before tap", '2. tool: tap {"x":0.5}']); }); + // `idle` has no recorder command — it is written by hand into the YAML, + // which the finish re-reads. Without a case here it fell through to the + // `tool:` default and the summary described the step as a tool call. + it("summarizes a hand-written idle step as the wait it is", async () => { + await flowStartRecordingTool.execute( + {}, + { name: "idle-summary", project_root: tmpDir, executionPrerequisite: PREREQ } + ); + await fs.writeFile( + path.join(tmpDir, ".argent", "flows", "idle-summary.yaml"), + `executionPrerequisite: ${JSON.stringify(PREREQ)}\nsteps:\n - await: { idle: true }\n`, + "utf8" + ); + + const result = await flowFinishRecordingTool.execute( + {}, + { name: "idle-summary", project_root: tmpDir } + ); + + expect(result.summary).toEqual(["1. await: screen idle"]); + }); + it("distinguishes contains, equals, and regex text comparisons in the summary", async () => { const name = "text-comparison-summary"; await flowStartRecordingTool.execute( From 1c843d80c68ac3b0c4a275e54e7b9da123b1c5c9 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Fri, 7 Aug 2026 11:54:37 +0200 Subject: [PATCH 85/98] fix(flow): stop a blip on the closing poll from reddening an idle wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `waitForIdle` treated `lastRead === "error"` as an unreadable window on its own, with no regard for how long the source had actually been dark. Since a screen that never settles always exits through the bottom of the loop, its last read is the one that decides — so a single failed `describe` on the closing poll turned a run red, while the identical transient one poll earlier restarted the hold and passed with a warning. Whether a flow survived a screen this step is explicit about wanting to pass — a video, a shimmer, a carousel, live-updating text — came down to where the blip landed. Measure the tail instead, the way `waitForCondition` already does with `CONDITION_DARK_TAIL_TOLERANCE_MS`: keep the timestamp of the last read that answered, and hard-stop only once the failing stretch outlasts two polls. The verdict a tolerated blip falls through to carries the failed read with it rather than dropping it. Verified end to end against a headless Chromium over a CDP proxy that drops exactly one read on the closing poll: before, three of three runs errored and skipped the rest of the flow; after, three of three pass with the motion warning and the appended read failure. A window that stays dark for two or more polls still errors and still stops the run. --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-actions.ts | 59 ++++++++++++++++-- .../test/flows/flow-idle-run.test.ts | 60 +++++++++++++++++++ 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 0749b843d..a0cf5a1eb 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -95,7 +95,7 @@ It **never fails a run.** Readiness is not an acceptance criterion, so every out - **settled on the UI tree alone** — the screen could not be screenshotted often enough to compare a pair, so presentation-layer motion (a push, a fade, a dismissing modal) was not waited out. - **the screen came back with content on too few reads** — a settle takes three of them spanning two polls, and this step got fewer, so it ended without ever being able to tell whether the screen was moving. A slow tree source, or a window that was blank for most of the wait. -Only a tree source that cannot be read stops the run, as an `error` — one that fails outright, one that answers and then wedges, or one that never answers at all within the step (that last one may simply be slow: raise the step's `timeout` before suspecting the app). That is a broken window, not a verdict about the app: the run is not ok and every later step is skipped. +Only a tree source that cannot be read stops the run, as an `error` — one that is still failing when the wait ends, one that answers and then wedges, or one that never answers at all within the step (that last one may simply be slow: raise the step's `timeout` before suspecting the app). That is a broken window, not a verdict about the app: the run is not ok and every later step is skipped. A single failed read is not that window: the hold restarts from the next good one, and a read that fails at the very end of the wait is named in the warning rather than stopping the run. It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. There is no `assert` form (waiting is the whole point), no `when:` form, and the recorder cannot emit one — every `idle` step is hand-written. Do not sprinkle it after every step: each one costs a settle, and it cannot fail, so a flow full of them is slower without being stricter. diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index ae04e8a29..59ea82e3b 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -1227,6 +1227,27 @@ const LOCALIZED_MOTION_WARNING = */ const HUNG_TREE_READ_MS = 2_000; +/** + * Evidence-gap bound for the post-loop verdict, and the idle twin of + * {@link CONDITION_DARK_TAIL_TOLERANCE_MS}: how long the tree source may have + * been failing at the end of the wait before "the source stopped answering" is + * the better account of the window than whatever the screen was doing. + * + * A tree-source blip is expected mid-settle — the loop restarts the hold and + * carries on — and the read that happens to END the step is no more meaningful + * than any other. Without this bound, one failed read on the last poll turned + * every benign outcome into a run-stopping error, while the identical + * transient one poll earlier passed with a warning: whether a flow survived a + * screen this step is explicit about wanting to pass came down to where the + * blip landed. + * + * Two polls is what one blip costs: up to a poll of sleep since the last read + * that answered, plus a poll's worth of latency for the failing one. A longer + * tail means consecutive reads went dark, which is the window this step cannot + * describe. + */ +const IDLE_DARK_TAIL_TOLERANCE_MS = IDLE_POLL_MS * 2; + /** How the last tree read ended. Only `value` licenses a verdict about the app. */ type TreeReadOutcome = "value" | "error" | "timeout"; @@ -1293,6 +1314,12 @@ async function waitForIdle( // and every arm of that round sets it. let lastRead!: TreeReadOutcome; let treeErrorMessage: string | undefined; + // Date.now() of the most recent read that ANSWERED — 0 until one does, which + // the `readsSucceeded === 0` guard below returns on before anything measures + // from it. Post-loop it anchors the dark tail: how long the window's final + // stretch went without a look at the screen (blank counts — it is an + // observation; see the blank branch). + let lastAnsweredReadAt = 0; let treeReadHung = false; let sawContent = false; let pixelsEverMoved = false; @@ -1348,6 +1375,7 @@ async function waitForIdle( } else { lastRead = "value"; readsSucceeded += 1; + lastAnsweredReadAt = Date.now(); treeErrorMessage = undefined; // It answered, so whatever wedged it has cleared. treeReadHung = false; @@ -1464,7 +1492,16 @@ async function waitForIdle( // session. One early success does not license a verdict drawn from a window // that went dark afterwards. (A read that merely ran out of budget is NOT // this case — it is the step ending, and the evidence below still stands.) - if (lastRead === "error" && treeErrorMessage !== undefined) { + // + // Measured as a tail, not as a single read: the source failing on the last + // poll and the source having stopped answering are different windows, and + // only the second is unreadable. See IDLE_DARK_TAIL_TOLERANCE_MS. + const darkTailMs = Date.now() - lastAnsweredReadAt; + if ( + lastRead === "error" && + treeErrorMessage !== undefined && + darkTailMs > IDLE_DARK_TAIL_TOLERANCE_MS + ) { return unreadable(treeErrorMessage); } // The same window going dark the other way: the source answered, then stopped @@ -1482,6 +1519,17 @@ async function waitForIdle( `app reads the same as a backgrounded one)`, }; } + // A tolerated blip is not a silently dropped error: whichever warning below + // describes the window carries the failed read with it, the way + // waitForCondition appends its own. (The tree-only settle is the one verdict + // below that cannot be reached with a failed final read — that read cleared + // `treeSettledAtLastRead` — so it is left alone rather than given a note it + // could never print.) + const blipNote = + lastRead === "error" && treeErrorMessage !== undefined + ? ` (the read that ended the wait failed: ${treeErrorMessage})` + : ""; + // Readable throughout and never once carrying content: the screen rendered // nothing, which is not the same claim as "it never stopped moving". // @@ -1498,7 +1546,8 @@ async function waitForIdle( `the UI tree stayed empty for ${timeoutMs}ms — the screen never rendered content, so ` + `there was nothing to settle. If the screen is meant to render accessible content, this ` + `is where it did not; if it is a canvas or a video surface, it has none to read. Gate ` + - `the next action on an element check either way.`, + `the next action on an element check either way.` + + blipNote, }; } // Too few reads to have judged anything. A settle needs three of them @@ -1523,7 +1572,8 @@ async function waitForIdle( `${MIN_STILL_INTERVALS + 1} of them spanning ${MIN_STILL_INTERVALS} ${IDLE_POLL_MS}ms ` + `polls — so this step ended without ever being able to tell whether the screen was ` + `moving. Raise its \`timeout:\`, and gate the next action on a stable element rather ` + - `than on stillness.`, + `than on stillness.` + + blipNote, }; } // The tree was settled as of the last read and no pair of captures ever @@ -1550,7 +1600,8 @@ async function waitForIdle( `ahead without waiting it out. Either something on it never stops (a video, a looping ` + `animation, a carousel, live-updating text) or the screen never finished loading. Look at ` + `what is moving, and make sure the next action is gated on a stable element rather than on ` + - `stillness.`, + `stillness.` + + blipNote, }; } diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index e87241b16..7c24b54db 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -710,6 +710,66 @@ steps: expect(reads).toBe(5); }); + // A blip on the read that ENDS the step is still a blip. Which poll it lands + // on used to decide the whole verdict: one poll earlier it restarted the hold + // and the step passed with a warning, on the last poll it stopped the run and + // skipped every later step. So a screen this check is explicit about wanting + // to pass — a video, a shimmer, a carousel, live-updating text — turned a run + // red on timing luck alone. + it("does not stop the run when only the read that ended the wait failed", async () => { + let firstReadAt: number | undefined; + let tick = 0; + // Never settles, so the step always exits through the bottom of the loop + // and its last read is the one that decides. Measured from the first read + // rather than from the run, the threshold sits midway between the closing + // two rounds of a 900ms step (600ms and 800ms in), so only the last throws. + currentTree = () => { + firstReadAt ??= Date.now(); + if (Date.now() - firstReadAt >= 700) throw new Error("transient describe failure"); + return screenWith(`frame ${tick++}`); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 900, minStableMs: 0 } + - echo: reached +` + ); + const r = await run("ready"); + expect(r.ok).toBe(true); + const step = r.steps.find((s) => s.kind === "idle")!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("never held still"); + // Tolerated, not swallowed: the read that failed is still named. + expect(step.warning).toContain("transient describe failure"); + // And the checks that actually carry the flow's verdict still run. + expect(r.steps.at(-1)).toMatchObject({ kind: "echo", status: "pass" }); + }); + + // The same blip against the other window it used to redden: a screen that + // read back empty throughout is an observation about the app, and a transient + // on the closing read does not turn it into a window nobody could see. + it("still reports an empty screen as empty when the closing read failed", async () => { + let firstReadAt: number | undefined; + currentTree = () => { + firstReadAt ??= Date.now(); + if (Date.now() - firstReadAt >= 700) throw new Error("transient describe failure"); + return n({ role: "AXWindow", frame: FULL, children: [] }); + }; + await writeFlow( + "ready", + `executionPrerequisite: "" +steps: + - await: { idle: true, timeout: 900, minStableMs: 0 } +` + ); + const step = (await run("ready")).steps.at(-1)!; + expect(step.status).toBe("pass"); + expect(step.warning).toContain("never rendered content"); + expect(step.warning).toContain("transient describe failure"); + }); + // The same for a screen that goes blank in the middle: an observation that // resets both holds, not a gap and not a reason to give up. it("restarts the hold after the screen goes blank, and still settles", async () => { From ac8abe143fd4404473c8bc7fb618a10953b6fd9d Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Fri, 7 Aug 2026 12:32:40 +0200 Subject: [PATCH 86/98] refactor(flow): spell the idle hold `stableFor` Every time-valued key the flow language defines for itself is a bare number of milliseconds - `wait`, `timeout`, `duration` - so `minStableMs` was the only one carrying the unit in its name. The one `Ms`-suffixed key in a flow file, `delayMs`, sits on a `tool:` step, which mirrors an MCP tool call verbatim and inherits that layer's naming rather than the DSL's. `await-screen-idle` keeps its own `minStableMs`: the split already exists one key over, where the tool's `timeoutMs` is spelled `timeout` in a flow. --- .../skills/skills/argent-create-flow/SKILL.md | 2 +- .../src/tools/flows/flow-actions.ts | 12 ++-- .../tool-server/src/tools/flows/flow-utils.ts | 31 ++++------ .../test/flows/flow-idle-condition.test.ts | 44 ++++++------- .../test/flows/flow-idle-run.test.ts | 62 +++++++++---------- .../test/flows/flow-skill-docs.test.ts | 6 +- 6 files changed, 76 insertions(+), 81 deletions(-) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index a0cf5a1eb..d6f65fddc 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -85,7 +85,7 @@ This condition-as-key form is the only spelling. `await` also accepts an optiona For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. -**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `minStableMs` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls, plus the 200ms of budget the closing round has to have left to be allowed to start — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. +**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `stableFor` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls, plus the 200ms of budget the closing round has to have left to be allowed to start — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. It **never fails a run.** Readiness is not an acceptance criterion, so every outcome short of a clean settle passes carrying a `warning` on the step — read it rather than stepping over it: diff --git a/packages/tool-server/src/tools/flows/flow-actions.ts b/packages/tool-server/src/tools/flows/flow-actions.ts index 59ea82e3b..b0bbca2ba 100644 --- a/packages/tool-server/src/tools/flows/flow-actions.ts +++ b/packages/tool-server/src/tools/flows/flow-actions.ts @@ -46,7 +46,7 @@ import { import { describeSelector, describeTextExpectation, - IDLE_DEFAULT_MIN_STABLE_MS, + IDLE_DEFAULT_STABLE_FOR_MS, IDLE_DEFAULT_TIMEOUT_MS, IDLE_MIN_STILL_INTERVALS, IDLE_POLL_MS, @@ -1283,7 +1283,7 @@ async function waitForIdle( step: Extract ): Promise { const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; - const minStableMs = step.minStableMs ?? IDLE_DEFAULT_MIN_STABLE_MS; + const stableFor = step.stableFor ?? IDLE_DEFAULT_STABLE_FOR_MS; // Resolved once: it depends only on the device, and on iOS it costs a // runtime probe the capture path memoizes anyway. const maskTopFraction = await statusBarMaskFraction(env.device); @@ -1400,7 +1400,7 @@ async function waitForIdle( // Stillness is a property of an INTERVAL, so no verdict comes from one // observation — and, per MIN_STILL_INTERVALS, none comes from one - // interval either. `minStableMs: 0` therefore still means three reads: + // interval either. `stableFor: 0` therefore still means three reads: // a single sample proves nothing about motion, and a single agreeing // pair can be two points of an animation that reversed between them. const treeHeld = signature === treeSignature; @@ -1412,7 +1412,7 @@ async function waitForIdle( treeStillIntervals += 1; } treeSettledAtLastRead = - treeStillIntervals >= MIN_STILL_INTERVALS && now - treeSince >= minStableMs; + treeStillIntervals >= MIN_STILL_INTERVALS && now - treeSince >= stableFor; // A missing frame is the ABSENCE of visual evidence, never evidence of // stillness. Letting it stand in for "the pixels held" is what turned a @@ -1442,7 +1442,7 @@ async function waitForIdle( if (treeHeld && pixelsHeld) { stillIntervals += 1; if (localizedThisInterval) localizedMotionDuringHold = true; - if (stillIntervals >= MIN_STILL_INTERVALS && now - bothSince >= minStableMs) { + if (stillIntervals >= MIN_STILL_INTERVALS && now - bothSince >= stableFor) { return localizedMotionDuringHold ? { ok: true, warning: LOCALIZED_MOTION_WARNING } : { ok: true }; @@ -1596,7 +1596,7 @@ async function waitForIdle( return { ok: true, warning: - `the screen never held still for ${minStableMs}ms within ${timeoutMs}ms, so this step went ` + + `the screen never held still for ${stableFor}ms within ${timeoutMs}ms, so this step went ` + `ahead without waiting it out. Either something on it never stops (a video, a looping ` + `animation, a carousel, live-updating text) or the screen never finished loading. Look at ` + `what is moving, and make sure the next action is gated on a stable element rather than on ` + diff --git a/packages/tool-server/src/tools/flows/flow-utils.ts b/packages/tool-server/src/tools/flows/flow-utils.ts index a2474e460..1824df085 100644 --- a/packages/tool-server/src/tools/flows/flow-utils.ts +++ b/packages/tool-server/src/tools/flows/flow-utils.ts @@ -713,7 +713,7 @@ export type FlowStep = * is a property of the whole screen. There is no `assert` form: "has it * stopped moving yet" is inherently a wait. */ - | { kind: "idle"; timeout?: number; minStableMs?: number } + | { kind: "idle"; timeout?: number; stableFor?: number } | { kind: "wait"; ms: number } | { kind: "scroll-to"; target: FlowSelector; direction: ScrollDirection; within?: FlowSelector } | { kind: "pinch"; selector?: FlowSelector; scale: number } @@ -857,7 +857,7 @@ type YamlTextWaitCondition = Extract; * body carries either a selector condition or this one, never a mix — so it is * parsed by {@link parseIdleFields} rather than by parseWaitFields. */ -type YamlIdleCondition = { idle: true; minStableMs?: number; timeout?: number }; +type YamlIdleCondition = { idle: true; stableFor?: number; timeout?: number }; /** `scroll-to` body: a bare target (scrolls down), or a map with options. */ type YamlScrollBody = @@ -1173,7 +1173,7 @@ function waitToYaml( */ function idleToYaml(step: Extract): YamlStep { const body: YamlIdleCondition = { idle: true }; - if (step.minStableMs !== undefined) body.minStableMs = step.minStableMs; + if (step.stableFor !== undefined) body.stableFor = step.stableFor; if (step.timeout !== undefined) body.timeout = step.timeout; return { await: body }; } @@ -1697,7 +1697,7 @@ const IDLE_CONDITION = "idle"; * gates. The runner imports them back. */ export const IDLE_DEFAULT_TIMEOUT_MS = 7500; -export const IDLE_DEFAULT_MIN_STABLE_MS = 250; +export const IDLE_DEFAULT_STABLE_FOR_MS = 250; /** `idle` poll cadence, matching `await-screen-idle`'s own. */ export const IDLE_POLL_MS = 200; @@ -1731,7 +1731,7 @@ export const IDLE_SETTLE_OVERHEAD_MS = (IDLE_MIN_STILL_INTERVALS + 1) * IDLE_POL * gate no run can pass. The relationship that actually matters is with * `timeout`, checked separately. */ -const IDLE_MAX_MIN_STABLE_MS = 600_000; +const IDLE_MAX_STABLE_FOR_MS = 600_000; /** * The `timeout` sibling key an `await` may carry, spelled once for both the @@ -1786,7 +1786,7 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") : "") ); } - rejectUnknownKeys(entry, raw, ["idle", "minStableMs", "timeout"], kind); + rejectUnknownKeys(entry, raw, ["idle", "stableFor", "timeout"], kind); // `idle: true` only. A falsey value would spell "assert the screen is NOT // settled", which no flow wants and the runner cannot answer. @@ -1796,13 +1796,8 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") const step: Extract = { kind: "idle" }; if ("timeout" in raw) step.timeout = parseAwaitTimeout(entry, raw.timeout); - if (raw.minStableMs !== undefined) { - step.minStableMs = parseBoundedMs( - entry, - raw.minStableMs, - "idle.minStableMs", - IDLE_MAX_MIN_STABLE_MS - ); + if (raw.stableFor !== undefined) { + step.stableFor = parseBoundedMs(entry, raw.stableFor, "idle.stableFor", IDLE_MAX_STABLE_FOR_MS); } // A wait that cannot contain the settle it asks for is a gate that never @@ -1816,20 +1811,20 @@ function parseIdleFields(raw: Record, kind: "await" | "assert") // Checked against the EFFECTIVE hold, not just a written-out one: the // default is what most steps run with, so leaving it out was the way to get // an unsatisfiable step past the parser (`timeout: 100` was accepted while - // the identical `timeout: 100, minStableMs: 250` was rejected). + // the identical `timeout: 100, stableFor: 250` was rejected). const timeoutMs = step.timeout ?? IDLE_DEFAULT_TIMEOUT_MS; - const minStableMs = step.minStableMs ?? IDLE_DEFAULT_MIN_STABLE_MS; - const needed = minStableMs + IDLE_SETTLE_OVERHEAD_MS; + const stableFor = step.stableFor ?? IDLE_DEFAULT_STABLE_FOR_MS; + const needed = stableFor + IDLE_SETTLE_OVERHEAD_MS; if (timeoutMs < needed) { badEntry( entry, `idle needs a timeout of at least ${needed}ms to hold still for ` + - `${step.minStableMs === undefined ? `the default ` : ``}${minStableMs}ms: a settle is ` + + `${step.stableFor === undefined ? `the default ` : ``}${stableFor}ms: a settle is ` + `${IDLE_MIN_STILL_INTERVALS + 1} reads spanning ${IDLE_MIN_STILL_INTERVALS} ` + `${IDLE_POLL_MS}ms polls, plus the ${IDLE_POLL_MS}ms of budget the closing round has to ` + `have left to be allowed to start, and the wait has to contain all of that as well as ` + `the hold. Raise ` + - `\`timeout\`${step.minStableMs === undefined ? "" : " or lower `minStableMs`"}` + `\`timeout\`${step.stableFor === undefined ? "" : " or lower `stableFor`"}` ); } return step; diff --git a/packages/tool-server/test/flows/flow-idle-condition.test.ts b/packages/tool-server/test/flows/flow-idle-condition.test.ts index 6db72b04e..78ef427bf 100644 --- a/packages/tool-server/test/flows/flow-idle-condition.test.ts +++ b/packages/tool-server/test/flows/flow-idle-condition.test.ts @@ -31,9 +31,9 @@ describe("await { idle }", () => { }); it("carries the optional hold and timeout", () => { - expect(expectRoundTrip(` - await: { idle: true, minStableMs: 400, timeout: 9000 }\n`)).toEqual( - [{ kind: "idle", minStableMs: 400, timeout: 9000 }] - ); + expect(expectRoundTrip(` - await: { idle: true, stableFor: 400, timeout: 9000 }\n`)).toEqual([ + { kind: "idle", stableFor: 400, timeout: 9000 }, + ]); }); it("has no assert form — waiting is the whole point of the check", () => { @@ -44,23 +44,23 @@ describe("await { idle }", () => { expect(() => parseSteps(` - await: { idle: false }\n`)).toThrow(/idle takes only/); }); - it("bounds minStableMs", () => { - expect(() => parseSteps(` - await: { idle: true, minStableMs: -1 }\n`)).toThrow( - /idle.minStableMs/ + it("bounds stableFor", () => { + expect(() => parseSteps(` - await: { idle: true, stableFor: -1 }\n`)).toThrow( + /idle.stableFor/ ); - expect(() => parseSteps(` - await: { idle: true, minStableMs: 1.5 }\n`)).toThrow( - /idle.minStableMs/ + expect(() => parseSteps(` - await: { idle: true, stableFor: 1.5 }\n`)).toThrow( + /idle.stableFor/ ); // And from above, so a hold written in the wrong unit (seconds, a pasted // timestamp) is rejected as a number rather than becoming a gate no run // can pass. The ceiling is ten minutes; a `timeout` wide enough to contain // it is what the case below checks separately. - expect(() => parseSteps(` - await: { idle: true, minStableMs: 600001 }\n`)).toThrow( + expect(() => parseSteps(` - await: { idle: true, stableFor: 600001 }\n`)).toThrow( /between 0 and 600000/ ); - expect(parseSteps(` - await: { idle: true, minStableMs: 600000, timeout: 600600 }\n`)).toEqual( - [{ kind: "idle", minStableMs: 600000, timeout: 600600 }] - ); + expect(parseSteps(` - await: { idle: true, stableFor: 600000, timeout: 600600 }\n`)).toEqual([ + { kind: "idle", stableFor: 600000, timeout: 600600 }, + ]); }); // A wait that cannot contain the settle it asks for is a gate that fails on @@ -70,29 +70,29 @@ describe("await { idle }", () => { // spanning two 200ms polls, plus the budget the closing round has to start // with. it("rejects a wait that could never contain the settle it asks for", () => { - expect(() => - parseSteps(` - await: { idle: true, timeout: 500, minStableMs: 1000 }\n`) - ).toThrow(/idle needs a timeout of at least 1600ms to hold still for 1000ms/); + expect(() => parseSteps(` - await: { idle: true, timeout: 500, stableFor: 1000 }\n`)).toThrow( + /idle needs a timeout of at least 1600ms to hold still for 1000ms/ + ); // With no explicit hold the DEFAULT is what has to fit — the spelling that - // slipped through, since leaving `minStableMs` out was the way past the + // slipped through, since leaving `stableFor` out was the way past the // check that only looked at a written-out one. expect(() => parseSteps(` - await: { idle: true, timeout: 100 }\n`)).toThrow( /idle needs a timeout of at least 850ms to hold still for the default 250ms/ ); // Which is the same step as writing the default out, so it is rejected the // same way. - expect(() => parseSteps(` - await: { idle: true, timeout: 100, minStableMs: 250 }\n`)).toThrow( + expect(() => parseSteps(` - await: { idle: true, timeout: 100, stableFor: 250 }\n`)).toThrow( /idle needs a timeout of at least 850ms/ ); // With no explicit timeout the default is what the hold has to fit inside. - expect(() => parseSteps(` - await: { idle: true, minStableMs: 9000 }\n`)).toThrow( + expect(() => parseSteps(` - await: { idle: true, stableFor: 9000 }\n`)).toThrow( /idle needs a timeout of at least 9600ms/ ); // The boundary itself is legal on both sides. - expect(parseSteps(` - await: { idle: true, timeout: 900, minStableMs: 300 }\n`)).toEqual([ - { kind: "idle", timeout: 900, minStableMs: 300 }, + expect(parseSteps(` - await: { idle: true, timeout: 900, stableFor: 300 }\n`)).toEqual([ + { kind: "idle", timeout: 900, stableFor: 300 }, ]); - expect(() => parseSteps(` - await: { idle: true, timeout: 899, minStableMs: 300 }\n`)).toThrow( + expect(() => parseSteps(` - await: { idle: true, timeout: 899, stableFor: 300 }\n`)).toThrow( /idle needs a timeout of at least 900ms/ ); }); @@ -113,7 +113,7 @@ describe("await { idle }", () => { // the timeout is. it("rejects a timeout too small to settle in without inventing a range", () => { const parse = (): FlowStep[] => - parseSteps(` - await: { idle: true, timeout: 0.5, minStableMs: 0 }\n`); + parseSteps(` - await: { idle: true, timeout: 0.5, stableFor: 0 }\n`); expect(parse).toThrow(/idle needs a timeout of at least 600ms/); expect(parse).not.toThrow(/between 0 and -/); }); diff --git a/packages/tool-server/test/flows/flow-idle-run.test.ts b/packages/tool-server/test/flows/flow-idle-run.test.ts index 7c24b54db..2b7a1f51d 100644 --- a/packages/tool-server/test/flows/flow-idle-run.test.ts +++ b/packages/tool-server/test/flows/flow-idle-run.test.ts @@ -169,7 +169,7 @@ describe("await: { idle }", () => { "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const r = await run("ready"); @@ -180,7 +180,7 @@ steps: }); // Stillness is a property of an interval, and one interval can alias (see - // the reversing-animation case below), so `minStableMs: 0` still means "the + // the reversing-animation case below), so `stableFor: 0` still means "the // first two agreeing intervals" — three reads — not "the first read". it("never settles on one read or one interval, even with no hold requested", async () => { let reads = 0; @@ -192,7 +192,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); expect((await run("ready")).ok).toBe(true); @@ -204,7 +204,7 @@ steps: expect(captureFirstFlags).toEqual([true, false, false]); }); - // `minStableMs` is a clock, not a label: a screen that is still from the + // `stableFor` is a clock, not a label: a screen that is still from the // first read must still be held for it before the step returns. Without that // the option means nothing, since three reads take about 400ms whatever it // is set to. @@ -213,7 +213,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2500, minStableMs: 800 } + - await: { idle: true, timeout: 2500, stableFor: 800 } ` ); const started = Date.now(); @@ -232,7 +232,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 300 } + - await: { idle: true, timeout: 900, stableFor: 300 } ` ); const r = await run("ready"); @@ -255,7 +255,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 300 } + - await: { idle: true, timeout: 900, stableFor: 300 } - echo: reached ` ); @@ -276,7 +276,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 600, minStableMs: 0 } + - await: { idle: true, timeout: 600, stableFor: 0 } ` ); const r = await run("ready"); @@ -301,7 +301,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } - echo: reached ` ); @@ -340,7 +340,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -354,7 +354,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); expect((await run("ready")).steps.at(-1)!.warning).toBeUndefined(); @@ -369,7 +369,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2000, minStableMs: 0 } + - await: { idle: true, timeout: 2000, stableFor: 0 } ` ); const r = await run("ready"); @@ -401,7 +401,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1500, minStableMs: 0 } + - await: { idle: true, timeout: 1500, stableFor: 0 } ` ); const r = await run("ready"); @@ -420,7 +420,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 800, minStableMs: 0 } + - await: { idle: true, timeout: 800, stableFor: 0 } ` ); const started = Date.now(); @@ -450,7 +450,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2500, minStableMs: 0 } + - await: { idle: true, timeout: 2500, stableFor: 0 } - echo: reached ` ); @@ -502,7 +502,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1200, minStableMs: 0 } + - await: { idle: true, timeout: 1200, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -517,7 +517,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 0 } + - await: { idle: true, timeout: 900, stableFor: 0 } ` ); const r = await run("ready"); @@ -536,7 +536,7 @@ steps: // The tree-only report is a claim that the HIERARCHY settled, so it owes the // same hold every other settle does. Every case that reaches it elsewhere - // runs with `minStableMs: 0`, where the hold term is vacuous — drop it and + // runs with `stableFor: 0`, where the hold term is vacuous — drop it and // the suite stays green while a tree that had only just stopped moving is // reported as having settled. it("does not report a tree-only settle when the hold was never served", async () => { @@ -551,7 +551,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1400, minStableMs: 800 } + - await: { idle: true, timeout: 1400, stableFor: 800 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -581,7 +581,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -604,7 +604,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -670,7 +670,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 0 } + - await: { idle: true, timeout: 900, stableFor: 0 } - echo: unreachable ` ); @@ -699,7 +699,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -732,7 +732,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 0 } + - await: { idle: true, timeout: 900, stableFor: 0 } - echo: reached ` ); @@ -761,7 +761,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 900, minStableMs: 0 } + - await: { idle: true, timeout: 900, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -782,7 +782,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, minStableMs: 0 } + - await: { idle: true, stableFor: 0 } ` ); const step = (await run("ready")).steps.at(-1)!; @@ -829,7 +829,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 700, minStableMs: 0 } + - await: { idle: true, timeout: 700, stableFor: 0 } ` ); const r = await run("ready"); @@ -857,7 +857,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2500, minStableMs: 0 } + - await: { idle: true, timeout: 2500, stableFor: 0 } ` ); const r = await run("ready"); @@ -877,7 +877,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2500, minStableMs: 0 } + - await: { idle: true, timeout: 2500, stableFor: 0 } ` ); expect((await run("ready")).steps.at(-1)!.warning).toContain("never held still"); @@ -902,7 +902,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 2000, minStableMs: 0 } + - await: { idle: true, timeout: 2000, stableFor: 0 } ` ); const r = await run("ready"); @@ -919,7 +919,7 @@ steps: "ready", `executionPrerequisite: "" steps: - - await: { idle: true, timeout: 1600, minStableMs: 900 } + - await: { idle: true, timeout: 1600, stableFor: 900 } ` ); const r = await run("ready"); diff --git a/packages/tool-server/test/flows/flow-skill-docs.test.ts b/packages/tool-server/test/flows/flow-skill-docs.test.ts index 085310c27..41eaeaaf0 100644 --- a/packages/tool-server/test/flows/flow-skill-docs.test.ts +++ b/packages/tool-server/test/flows/flow-skill-docs.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from "node:fs"; import * as path from "node:path"; import type { Registry } from "@argent/registry"; import { - IDLE_DEFAULT_MIN_STABLE_MS, + IDLE_DEFAULT_STABLE_FOR_MS, IDLE_DEFAULT_TIMEOUT_MS, IDLE_MIN_STILL_INTERVALS, IDLE_POLL_MS, @@ -78,7 +78,7 @@ describe("create-flow SKILL.md scope snippets", () => { // a default that moves takes the sentence describing it with it. it("the skill's idle defaults and settle cost are the ones the parser enforces", () => { const skill = readFileSync(SKILL, "utf8"); - expect(skill).toContain(`default ${IDLE_DEFAULT_MIN_STABLE_MS}`); + expect(skill).toContain(`default ${IDLE_DEFAULT_STABLE_FOR_MS}`); expect(skill).toContain(`default ${IDLE_DEFAULT_TIMEOUT_MS}`); expect(skill).toContain(`${IDLE_SETTLE_OVERHEAD_MS}ms a settle costs`); expect(skill).toContain(`${IDLE_POLL_MS}ms polls`); @@ -93,7 +93,7 @@ describe("create-flow SKILL.md scope snippets", () => { // The skill tells an author the wait has to contain the hold plus the // settle. Take it at its word and check the boundary both ways — a parser // that demanded a millisecond more would make the documented sum a lie. - const smallest = IDLE_DEFAULT_MIN_STABLE_MS + IDLE_SETTLE_OVERHEAD_MS; + const smallest = IDLE_DEFAULT_STABLE_FOR_MS + IDLE_SETTLE_OVERHEAD_MS; expect(() => parseFlow(`steps:\n - await: { idle: true, timeout: ${smallest} }\n`) ).not.toThrow(); From 91eb07b3aeaf5bdd2a9f308ff764ae2fada52296 Mon Sep 17 00:00:00 2001 From: Hubert Gancarczyk Date: Tue, 4 Aug 2026 09:23:24 +0200 Subject: [PATCH 87/98] docs(skills): split argent-create-flow into references and add argent-qa-flows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `argent-create-flow`'s SKILL.md had grown to roughly 5,000 words — every word of it loaded on every invocation, whether the task was replaying a two-step fragment or authoring a regression test. It is now ~900 words that route to three references read on demand: live-authoring (recording a walkthrough), flow-yaml (the file format and selector vocabulary), and reliability-and-recovery (what to do when a step or a replay goes wrong). `argent-qa-flows` is new: turning a test case, ticket, or acceptance criteria into a repeatable regression test is a different job from recording a path worth replaying. It orchestrates create-flow as its engine, records the first walkthrough live, requires every requested screen and state to be proved with stable evidence, and completes only after the unchanged flow passes twice consecutively. The routing rules gain the distinction the three skills now need, because they were being confused with each other by name alone: a one-off interactive check is argent-test-ui-flow, a saved replayable path is argent-create-flow, and a saved test with acceptance criteria and two-pass proof is argent-qa-flows. "Record a flow" also stops being ambiguous with screen recording, which is video. One new rule is worth calling out: if a flow or QA test may be recorded, do not interact with the app first — start the recorder before the first launch. A path already walked cannot be recorded retroactively, and re-walking it was one of the more expensive mistakes in practice. Proving a navigation is spelled out here for the first time, and it is now two element-level checks rather than a route read: an `await:` on something that exists ONLY on the destination, then `await: { idle: true }`. Neither implies the other — a dropped tap leaves the source screen perfectly idle, and the destination's elements enter the tree while the transition is still animating over them — so both are required after every screen change. The readiness half is a directive with no live tool behind it, so it joins `snapshot:` on the short list of insertions allowed during polish. A test parses the YAML frontmatter of every bundled skill, so a malformed new one fails here rather than at install time. --- .../test/skills-frontmatter.test.ts | 29 ++ packages/skills/rules/argent.md | 15 +- .../skills/skills/argent-create-flow/SKILL.md | 349 ++---------------- .../references/flow-yaml.md | 183 +++++++++ .../references/live-authoring.md | 239 ++++++++++++ .../references/reliability-and-recovery.md | 146 ++++++++ .../skills/argent-device-interact/SKILL.md | 53 ++- .../skills/skills/argent-qa-flows/SKILL.md | 119 ++++++ .../test/flows/flow-skill-docs.test.ts | 117 ++---- 9 files changed, 814 insertions(+), 436 deletions(-) create mode 100644 packages/argent-installer/test/skills-frontmatter.test.ts create mode 100644 packages/skills/skills/argent-create-flow/references/flow-yaml.md create mode 100644 packages/skills/skills/argent-create-flow/references/live-authoring.md create mode 100644 packages/skills/skills/argent-create-flow/references/reliability-and-recovery.md create mode 100644 packages/skills/skills/argent-qa-flows/SKILL.md diff --git a/packages/argent-installer/test/skills-frontmatter.test.ts b/packages/argent-installer/test/skills-frontmatter.test.ts new file mode 100644 index 000000000..67cb33440 --- /dev/null +++ b/packages/argent-installer/test/skills-frontmatter.test.ts @@ -0,0 +1,29 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parse as parseYaml } from "yaml"; +import { describe, expect, it } from "vitest"; + +const skillsDir = fileURLToPath(new URL("../../skills/skills/", import.meta.url)); + +describe("bundled skill frontmatter", () => { + it("parses every SKILL.md YAML block", () => { + const skillFiles = fs + .readdirSync(skillsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(skillsDir, entry.name, "SKILL.md")) + .filter((filePath) => fs.existsSync(filePath)); + + expect(skillFiles.length).toBeGreaterThan(0); + for (const filePath of skillFiles) { + const content = fs.readFileSync(filePath, "utf8"); + const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/)?.[1]; + expect(frontmatter, `${filePath} is missing YAML frontmatter`).toBeDefined(); + expect( + () => parseYaml(frontmatter!), + `${filePath} has invalid YAML frontmatter` + ).not.toThrow(); + } + }); +}); diff --git a/packages/skills/rules/argent.md b/packages/skills/rules/argent.md index 588e2f917..04cdc1c9f 100644 --- a/packages/skills/rules/argent.md +++ b/packages/skills/rules/argent.md @@ -76,6 +76,7 @@ Decision order: - Interaction tools (`gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically. Call `screenshot` separately only for a baseline before any action or after a delay. - Always open apps with `launch-app` or `open-url` — never tap home screen icons. +- If you may record a flow or QA test, do not interact with the app first: load `argent-create-flow` and start the recorder before the first launch or in-app action. A path already walked cannot be recorded retroactively. - Always use `run-sequence` when performing multiple sequential device actions where you don't need to observe the screen between steps. More in `argent-device-interact` skill. - When the session ends or the user says they are done: call `stop-all-simulator-servers` with `devices: [...]` naming the devices this session actually used. One tool-server is shared by every other agent using this @@ -131,7 +132,7 @@ When: Explicit visual regression, screenshot diff, compare screenshots, before/a SCREEN RECORDING (VIDEO CAPTURE) Skill: `argent-screen-recording` -When: The user wants a video of the device screen — recording a flow, interaction, animation, or bug reproduction as a clip, or documenting app behavior beyond what a still screenshot shows. Covers the start → interact → stop lifecycle, the reminder discipline that keeps a recording from being left running, and retrieving the mp4 artifact. +When: The user wants a video of the device screen — recording a flow, interaction, animation, or bug reproduction as a clip, or documenting app behavior beyond what a still screenshot shows. Covers the start → interact → stop lifecycle, the reminder discipline that keeps a recording from being left running, and retrieving the mp4 artifact. Not for a replayable saved sequence: route that meaning of "record a flow" to `argent-create-flow`. Prompt keywords: record, recording, screen recording, video, capture video, clip, mp4 RUNNING / BUILDING / DEBUGGING REACT NATIVE APP @@ -154,15 +155,21 @@ PERFORMANCE OPTIMIZATION Use skill: `argent-react-native-optimization` When: App feels slow, user asks to optimize, reducing bundle size, improving startup time, fixing re-renders, optimizing lists/images/navigation, or any performance-related task. This is the entry-point skill for all performance work — it delegates to `argent-react-native-profiler` for measurement. -END-TO-END UI TESTING +INTERACTIVE UI TESTING (ONE-OFF, NOT SAVED) Skill: `argent-test-ui-flow` -When: Verifying complete user flows, running interact → screenshot → verify loops, testing features by using the app, executing manual QA steps, or validating visible UI changes or visual behavior after implementation. +When: Verifying complete user flows, running interact → screenshot → verify loops, testing features by using the app, executing manual QA steps, or validating visible UI changes or visual behavior after implementation. Not for a test that must be saved and re-run — see GENERATED QA REGRESSION TESTS below. RECORDING & REPLAYING FLOWS Use skill: `argent-create-flow` -When: A multi-step interaction sequence needs to be repeated — re-profiling after a fix, A/B comparisons, regression checks, user says "again" / "run that flow", or you worked through a complex path worth saving. Also use proactively: if you are about to repeat steps you already performed, record first, then replay. +When: A multi-step interaction sequence needs to be repeated — re-profiling after a fix, A/B comparisons, user says "again" / "run that flow", or you worked through a complex path worth saving. Also use proactively: if you are about to repeat steps you already performed, record first, then replay. For a QA test case, ticket, or acceptance criteria to keep as a regression test, use `argent-qa-flows` (it loads this skill as its engine). Prompt keywords: flow, repeat, test X times +GENERATED QA REGRESSION TESTS +Use skill: `argent-qa-flows` +When: The user gives a test case, ticket, or acceptance criteria to keep as a repeatable test — "generate a QA test", "turn this test case into a flow", "automate this regression check", "make a test that does X and checks Y". Orchestrates `argent-create-flow`, records the first walkthrough live, verifies each requested screen/state with stable evidence, and completes only after the unchanged full flow passes twice consecutively. iOS, Android, and Chromium. +Prompt keywords: QA test, regression test, test case, automate this test, automate an e2e test, keep this e2e test, generate a test +Saved-artifact rule: one-off interactive check → `argent-test-ui-flow`; saved replayable path → `argent-create-flow`; saved test with acceptance criteria and two-pass proof → `argent-qa-flows`. + PROPOSING DESIGN VARIANTS FOR HUMAN SELECTION Use skill: `argent-lens` When: The user asks for design alternatives / options / A-B choices for a screen or component, or you have produced more than one candidate look for an element and want a human to pick before committing. Covers the build → navigate → screenshot → propose_variant loop and the single blocking await_user_selection call. (Gated behind the `argent-lens` flag, off by default — run `argent enable argent-lens` first.) diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index d6f65fddc..d633fcd0d 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -1,343 +1,52 @@ --- name: argent-create-flow -description: Record a reusable flow (scripted sequence of MCP tool calls) that can be replayed later with a single command. Use when the user asks to create, record, or build a flow, or to script a sequence of device actions. Also used proactively, without an explicit request, when a multi-step interaction sequence is about to be repeated (re-profiling, re-testing, or a complex path worth saving). +description: Create, record, edit, replay, or repair reusable Argent flow YAML files for device interaction. Use when the user asks to create or record a flow, script a repeatable device path, replay the same interaction, preserve a non-trivial path for profiling or A/B comparison, or invoke the authoring engine for argent-qa-flows. Also use proactively before repeating a path of three or more interactions. Do NOT use for a one-off interactive check (use argent-test-ui-flow), a saved QA/regression test driven by acceptance criteria (use argent-qa-flows), or a screen video or bug-reproduction clip (use argent-screen-recording). --- -## Overview +# Create an Argent flow -A flow is a sequence of steps saved to a `.yaml` file in the `.argent/flows/` directory. Each recorded step is **executed live** as you add it, so you verify it works before it becomes part of the flow. Replay a finished flow with `flow-execute`, or — for an e2e flow — headlessly with `argent flow run checkout` (a saved flow's name) or `argent flow run path/to/checkout.yaml`. +An Argent flow is a replayable sequence in `.argent/flows/.yaml`. -Flows store **no device id**: the runner binds a device (the single booted one, or pass `device`/`platform`). A recorded coordinate `gesture-tap` is captured as a portable `tap: { selector }` step whenever the tapped element has stable text/identifier. +**If the request is a QA test case, ticket, or acceptance criteria to keep as a regression test, load `argent-qa-flows` first** — it owns the QA contract (self-contained state, discriminating checks, and two consecutive passes) and uses this skill as its engine. -The one exception is a device _scope_ rather than a target: `stop-all-simulator-servers`' `devices` list is kept in the YAML, because without it the step means the machine-wide sweep and would tear down devices other agents are mid-session on. Replay rebinds a recorded scope only when you pass `device` explicitly — an auto-detected device would retarget the teardown at a device the flow never named, which is the cross-agent teardown the scope exists to prevent. So the recorded ids are what run when you replay without `device`, and when you hand-run the step (see _Strategy 2 — Manual recovery + continue_); on another host they reap nothing and come back in `unmatched`. Re-record the cleanup flow there, or pass `device`. A step that recorded NO scope is still narrowed onto whatever device the run resolved, since binding can only make the machine-wide sweep smaller. A cleanup flow whose only step is that teardown needs no device and runs whether none or several are booted. +## Read the relevant reference -**Two flow types** +- **Required before creating or changing a flow:** read [Live authoring](references/live-authoring.md) completely for the recorder workflow, polish pass, and final audit. +- **When writing or reviewing YAML by hand:** [Flow YAML](references/flow-yaml.md) — flow types, directives, conditions, composition, and runner syntax. For Vega, read its [Composition and platform limits](references/flow-yaml.md#composition-and-platform-limits) before recording remote/keyboard tools. +- **On a capture warning or a replay failure:** [Reliability and recovery](references/reliability-and-recovery.md) — coordinates or raw gestures appear, a transition is mistimed, an overlay may obstruct a target, the target is `com.apple.*`, an iOS app was not Argent-launched, or a platform's tree source is unavailable. -- **e2e** — begins with a `launch:` step, which starts that app from scratch (terminate + relaunch), so the flow controls its own start state. No `executionPrerequisite`. May `run:` other flows, and may itself be a `run:` target — when nested, its `launch` runs inline, restarting the app for that sub-scenario. **On Chromium a launch is a process, not a relaunch:** the "device" is the booted app (its id is the CDP port). The runner needs a device before step 1, so it boots for the launch the run _begins_ with, following a leading `run:` — a fragment whose first step composes a chromium e2e flow boots that flow's app (pass `--platform chromium` when the launch names several platforms, or the target is ambiguous and auto-detection is used instead). That first launch then just settles the instance it was booted for; every _later_ launch — a nested e2e flow's own, or a mid-flow `launch:` of the same app — boots its own instance and the run moves onto it for the remaining steps, replacing the one the runner already owns for that app. Every instance the runner boots is torn down at run end; one you pinned with `--device` is attached to, never killed — so relaunching _that_ app mid-flow fails if it holds a single-instance lock. A launch that names no id for the run's platform is an error — a `chromium:` entry does not make a flow runnable on iOS, and the run never switches platforms mid-flight. Record one by adding a `restart-app` of the app under test as the **first** step — it is captured as the `launch` step. Not on Chromium, though: `restart-app` has no chromium support and only successful calls are recorded, so a recorded chromium flow is always a fragment — write the `launch: { chromium: }` line into the YAML yourself afterward, and delete any `executionPrerequisite` the recording declared: with its own launch the flow controls its start state, and a launch-first flow must not carry one. -- **fragment** — doesn't begin with a launch; runs against the device's current state. May declare an `executionPrerequisite` (a documented entry-state contract). Invoked from other flows via a `run:` step, or directly by you at any time. One exception to "current state": a fragment whose **first** step `run:`s a chromium e2e flow takes on that flow's launch — the runner boots that app before step 1 (see the e2e bullet; pass `--device` to attach to a running instance instead). +## Non-negotiable rules -Both run via `argent flow run ` — a fragment simply runs against whatever is on screen (its prerequisite is printed as a reminder), except the chromium leading-`run:` case above, which boots. A bare name is read from `.argent/flows/.yaml`; anything ending in `.yaml` is a path. Only e2e flows are meaningful CI/suite entries, since only they give a deterministic verdict from a clean start. +1. **Record the path live.** The first walkthrough _is_ the recording; never rehearse a path and reconstruct it afterward. [Live authoring](references/live-authoring.md) has the per-platform start order and the discover → echo → `flow-add-step` → inspect cycle. +2. **Record each check when its state appears** — immediately after the transition or outcome it proves, before the next action. An echo or raw `screenshot` is diagnostic context, not an executable verdict; a reviewed `snapshot:` baseline is for inherently pixel-level requirements. Record absence as a trio in order — `visible` on the selector, the action that removes it, then `hidden` on the same selector; a `hidden` whose selector was never established cannot fail, so it proves nothing. +3. **Target semantics, not screen positions.** Prefer strict `{ id: ... }`, then a stable `{ text: ... }` or accessibility label; `scroll-to` for an off-screen target. The recorder converts a `gesture-tap` to a selector whenever it can and **warns when it had to keep the raw point** — stop on that warning and fix the target while the screen is still in front of you, rather than at audit time. Keep a coordinate only after the **coordinate fallback gate** in [Reliability and recovery](references/reliability-and-recovery.md#coordinate-fallback-gate), and report each one. +4. **After every screen change, prove identity, then readiness** — `await:` on an element that exists _only_ on the destination (never a shared tab bar, a source-screen element, or a positional id), then `await: { idle: true }`. Neither implies the other and a successful tap proves neither. [Live authoring](references/live-authoring.md#record-identity-then-readiness-after-every-navigation) has the procedure; [Flow YAML](references/flow-yaml.md#prove-a-navigation-identity-then-readiness) has why both are needed. +5. **Polish only what ran.** Rewriting a recorded raw action into an equivalent directive is allowed. Everything else must come from a recorded step; return to the live workflow to execute any missing **action** through the recorder. -### Step directives + Exactly three unrecorded insertions are allowed, each only where you saw the condition live — see [Live authoring](references/live-authoring.md#finish-and-polish). -Beyond raw `tool:` steps and `echo:`, flows support declarative directives interpreted by the runner (they are **not** agent-callable tools). **Every directive hard-stops the flow on failure**; later steps are reported `skip`. +6. **Replay the finished file end to end.** A flow is not done because its steps worked separately. `argent-qa-flows` adds the stronger requirement of two consecutive full passes. -| Directive | YAML | Meaning | -| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `launch` | `- launch: com.acme.app` or `- launch: { ios: …, android: … }` | start the app from scratch (terminate + relaunch) and wait until ready | -| `tap` | `- tap: Login`, `- tap: { x: 0.5, y: 0.57 }`, `- tap: { on: Login, times: 2 }`, `- tap: { on: { x: 0.5, y: 0.57 }, times: 2 }` | tap by selector (auto-waits) or raw point; `times` (2 = double-tap) needs the target nested under `on:` — a selector or a point (`{ x, y, times }` is rejected) | -| `long-press` | `- long-press: Row 3`, `- long-press: { x: 0.5, y: 0.6 }`, `- long-press: { on: , duration: 1200 }`, `- long-press: { on: { x: 0.5, y: 0.6 }, duration: 1200 }` | press and hold an element or raw point (default 800ms; Chromium: mouse press-hold); `duration` needs the target nested under `on:` — a selector or a point | -| `type` | `- type: { into: email, text: "a@b.com" }` | focus a field, type, then press Enter to submit + dismiss the keyboard | -| `scroll-to` | `- scroll-to: "Order #1234"` (scrolls down) or `- scroll-to: { target: …, direction: right, within: … }` | momentum-free scroll until the target is visible | -| `pinch` | `- pinch: { on: "Map", scale: 3 }` or `- pinch: { scale: 0.5 }` | two-finger zoom in (`scale` > 1) or out (`< 1`); big scales chain gestures; `on` optional — defaults to screen center; open-loop — assert the visible result | -| `rotate` | `- rotate: { on: "Map", by: 90 }` or `- rotate: { by: -45 }` | two-finger rotation by degrees (+ CW, − CCW, within ±3000°; options map only); `on` optional — screen center default; not `tool: rotate` (orientation) | -| `await` | `- await: { visible: Home }` or `- await: { idle: true }` | wait for a UI condition, or for the screen to stop moving | -| `wait` | `- wait: 500` | pause for a fixed number of milliseconds (last resort — prefer `await`) | -| `assert` | `- assert: { visible: Welcome }` | check a condition, hard-fail if it never holds | -| `snapshot` | `- snapshot: home` or `- snapshot: { name: home, maxMismatch: 0.5, cropOn: { id: order-summary } }` | diff a screenshot — or one element's region — against a stored baseline | -| `run` | `- run: login.yaml` | execute another flow's steps inline (fragment or e2e); a YAML path resolved relative to the flow file that contains the step (e.g. `../shared/login.yaml`); `.yaml` is optional (`run: login` = `login.yaml` beside the flow) | -| `when` | `- when: { visible: "What's new" }` + `steps: [...]` | run a guarded step block only when the condition holds (no else) | +### Stable selectors -### Selectors +A selector is **stable** when its value is fixed by the app's code, not by data, locale, time, count, or position — it would survive a content refresh, a different account, and a re-order. IDs such as `save-button`, `settings-screen`, and `logout-action` are stable; `3 unread`, `Today`, `Item 4`, a username, or any data-derived number are not. Treat visible text as stable only when the app's code keeps it identical across every locale and environment the flow supports. -A **selector** is `{ text?, id?, role? }` plus the optional scopes below (all-must-match; `text`/`role` are case-insensitive substrings, `id` matches the element's testID / accessibilityIdentifier / resource-id exactly, case-insensitive, also accepting the unqualified Android resource-id name — `submit` matches `com.example.app:id/submit`) — the same semantics `await-ui-element` uses, though that tool spells the `id` field `identifier` (flow YAML also accepts `identifier` as an alias for `id`, but `id` is the canonical spelling and what the recorder writes). A bare string is a _loose_ selector: it resolves **identifier-first, then falls back to text** (label/value), so `tap: Login` matches a `testID="Login"` or, failing that, visible text "Login" — no need to know which. Loose fallback applies uniformly to every selector slot (`tap`, `type.into`, `await`, `assert`, `scroll-to`). Use the map form to be strict: `{ id: submit-btn }` (identifier only) or `{ text: Login }` (text only, no fallback). +### Flow-only selector scopes -`text` also takes a **regex matcher map** — `{ text: { matches: '^Order #\d+$' } }`, in any selector slot — for dynamic text no literal can pin. It tests each node's native **own** label/value (not the adapter-hoisted `subtreeText`), though on iOS a container's own label may itself aggregate descendant text, so a wrapper and its leaf can both match. Same regex rules as `text.in`'s `matches` (see _`await` and `assert`_): unanchored, **case-sensitive**, single-quoted, invalid pattern fails at parse. So `assert: { visible: { text: { matches: '^Taps: \d+$' } } }` asserts a counter is on screen with no locator at all, and `tap: { text: { matches: '^Order #\d+$' } }` taps a dynamic row — though a stable `id` stays the more robust action target. +During YAML polish, use the frame-based `within`, `after`, and `next` scopes to disambiguate repeated elements. Read [Flow YAML: Relational scopes](references/flow-yaml.md#relational-scopes) for syntax, exact semantics, and the traps. -#### Scopes — CSS combinators, read off frames +## Workflow -A map selector may also carry a **scope**: a nested selector naming another element the match must sit in a given spatial relation to. These are the geometric readings of the CSS combinators that survive a flattened tree — the child combinator `>` has no analog, since parent/child structure does not reach replay. They combine, and nest up to six scopes per selector: +1. Choose the flow type: + - **e2e:** first non-echo step is `launch:`; it controls process start and is suitable as a standalone entry point. + - **fragment:** no leading launch; declare a precise `executionPrerequisite` and run against that state. +2. Work through [Live authoring](references/live-authoring.md) end to end: start the recorder, build and verify one step at a time, finish, polish, run the blocking audit, replay. +3. Report the flow path, replay command, verification result, prerequisites or side effects, and every accepted coordinate/raw-gesture exception. -| Scope | CSS | Reads as | Example | -| --------------- | ------- | --------------------------------------------------------------- | --------------------------------------------------------------------- | -| `within: ` | `A B` | the match's frame sits **inside** the scope element's frame | `tap: { text: Delete, within: { id: profile-card } }` | -| `after: ` | `A ~ B` | the match **follows** the scope element in reading order | `assert: { visible: { role: Button, after: { text: Danger zone } } }` | -| `next: ` | `A + B` | as `after`, narrowed to the **nearest** follower of each anchor | `tap: { role: Switch, next: { text: Wi-Fi } }` | +## Proactive recording -And `any: true` is the CSS `*` universal selector. It carries no locator of its own, so the parser enforces two rules: it needs **at least one scope** (a bare `any: true` would match the whole screen), and it may **not** sit beside `text`/`id`/`role` (which it would only make redundant — write `{ role: Switch, next: … }`, not `{ any: true, role: Switch, next: … }`). Only the literal `true` is accepted. `assert: { hidden: { any: true, within: { id: empty-state } } }` asserts an empty container. It works for actions too — `tap: { any: true, next: { text: Airplane Mode } }` taps whatever sits right after that label — but on a real tree "whatever" includes spacers and wrappers, so **name the target** (`role`, `id`) whenever you can and keep `any` for conditions and for `next`, which reduces to a single element per anchor. - -All of it is **visual (frame-based), not tree ancestry** — flow trees are flattened, and "inside the card" / "the switch after this label" mean what the screen shows — the same frame-based reading of "within" that `scroll-to`'s container anchor uses. Every scope needs a **distinct element** (nothing scopes itself, so `{ id: card, within: { id: card } }` needs two nested elements), the synthetic screen root never counts, and a scope only narrows _where_ to look — the selector still needs its own `text`/`id`/`role` naming _what_ to find there, or `any: true`. The nested slot takes every selector form: a bare string keeps the loose identifier-first fallback (`within: profile-card`), the map form stays strict, the regex matcher works (`within: { text: { matches: '^Card \d+$' } }`). Scopes are flow-YAML only; the raw `await-ui-element` tool's selector accepts none of them. Prefer a unique `id` on the target itself when one exists — reach for a scope when the target has no unique locator of its own (repeated row actions, per-card buttons, list cells). - -**`within`** — `tap: { text: Delete, within: { id: profile-card } }` taps the Delete button in the profile card even when other cards show identical ones; `tap: { text: "Pin feed", within: { text: "For You" } }` picks one card's button out of a whole list. Scope to a container with a **tight frame** (a row, card, dialog, toast): a full-screen wrapper contains everything and scopes nothing. It chains outward: `{ text: Save, within: { id: cards, within: Settings } }` reads "Save inside cards inside Settings", each container's frame inside the next. - -**`after` / `next`** — reading order is row-band aware: an element **follows** the anchor when it starts below the anchor's bottom edge, _or_ shares its row band and sits entirely to its right. That is what makes `{ role: Switch, next: { text: Wi-Fi } }` resolve the Wi-Fi row's own switch even though the taller switch's frame starts a couple of pixels _higher_ than the label's. `next` keeps only the nearest follower — a match in the anchor's own row beats anything on the rows below, leftmost first — while `after` keeps them all, so `assert: { hidden: { role: Button, after: { text: Danger zone } } }` holds when nothing button-like appears past that heading. Both union over anchors exactly as CSS does: with three rows on screen, `{ role: Switch, next: { role: AXStaticText } }` yields all three switches, one per label. Note that "follows" is **not transitive** — against a tall anchor, an element can follow something that itself follows the anchor without following the anchor directly — so nesting `after` scopes is not the same as chaining CSS `~`: `{ after: { …, after: … } }` can match elements a single `after` excludes. Nest them only when each link is a container-sized step. An element sitting _inside_ the anchor does not follow **that** anchor — containment is not reading order — so scope by `within` for that. - -**Prefer the map form for a scope's anchor.** A bare-string scope (`next: wifi-row`) keeps the loose identifier-first fallback, and the runner takes the first pass that finds a _visible_ match — so if some unrelated element carries `testID="wifi-row"`, the identifier pass wins and the text pass never runs. Reproduced: with a decoy `testID="Wi-Fi"` elsewhere on screen, `tap: { role: Switch, next: Wi-Fi }` taps the decoy's neighbour and reports a pass. This is the ordinary identifier-first doctrine, but a scope makes a decoy likelier to "succeed": a decoy _container_ only wins if it actually holds a match, while a decoy _anchor_ wins if anything at all sits after it — which on a real screen it usually does. Spell an anchor you care about as a map: `next: { text: Wi-Fi }` or `next: { id: wifi-row }`. - -One way `next` is deliberately **looser than CSS `+`**: where `A + B` matches nothing unless the very next sibling is a `B`, `next` keeps looking and returns the nearest match further on. That is what makes it survive the wrapper and spacer nodes a flattened tree is full of, but it also means a row that is _missing_ the control you asked for silently resolves to the next row's — `{ role: Switch, next: { text: Wi-Fi } }` on a Wi-Fi row rendered without a switch returns the _Bluetooth_ row's switch rather than failing. When a row may legitimately lack the control, assert it first (`assert: { visible: { role: Switch, within: { id: wifi-row } } }`) or scope by `within` instead. - -Scopes compose, and it matters which one carries them. `{ role: Button, next: { text: Name, within: { id: card-b } } }` scopes the **anchor** — one label, so one pick, but that pick may land outside card-b. `{ role: Button, next: { text: Name }, within: { id: card-b } }` scopes the **target** — every label is still an anchor, but only card-b's buttons can be picked. They agree on a well-formed screen and diverge when card-b has no button: the first reaches on to the next card's, the second returns nothing. Scope the target when the container is the thing you trust. Conditions honor scopes like any other selector: `assert: { hidden: { text: Saved, within: { id: toast-area } } }` holds when nothing matching "Saved" is inside the toast area — matches elsewhere on screen don't count, and a missing scope element satisfies `hidden` (and fails `visible`/`exists`). - -Selectors resolve against the **full native hierarchy** (iOS: the UIView tree; Android: the complete accessibility hierarchy including not-important views) — strictly more than `describe` or the raw `await-ui-element` tool see (both use the trimmed tree), with complete `testID`/`resource-id` coverage. So an `id` selector works even when `describe` collapses or omits the element — don't fall back to coordinate taps just because a testID isn't visible in `describe` output. And when several elements match — including wrappers whose native text aggregates descendant content — the action directives (`tap`, `type`, `scroll-to`) pick the **most specific** match: an exact text/identifier match beats a substring hit (for a regex matcher, a pattern consuming the element's whole text counts as exact), then the smallest frame wins. (A universal `any: true` selector has no field to be exact about, so its matches rank by reading order instead — the first element in the scope, which is the element a condition reads too. Where two matches share a top-left corner, an action breaks the tie toward the smaller, more specific one and a condition does not, so those two can name different elements.) - -**Quote strings YAML would mangle.** An unquoted `#` starts a YAML comment — `tap: Order #1234` silently parses as `tap: Order` — and bare `yes`/`no`/`on`/`off`/numbers coerce to non-strings. When a selector or typed text contains `#`, `:`, quotes, or could read as a boolean/number, wrap it: `tap: "Order #1234"`. - -### `await` and `assert` - -The **condition is the key**, and its value is the selector: - -- `{ visible: Home }`, `{ exists: { id: row } }`, `{ hidden: spinner }` -- `{ text: { in: , contains: "Taps:" } }` or `{ text: { in: , equals: "Taps: 0" } }` — `text` locates an element (`in`) and checks its rendered content against exactly one of `contains` (case-insensitive substring) or `equals` (case-insensitive exact match — use it when boundaries matter: `contains: "Taps: 3"` is also satisfied by "Taps: 30"). Reach for `text` only when the locator is an identifier/role; to assert a string is simply on screen, prefer `{ visible: "Taps: 0" }`. -- `{ text: { in: total, matches: 'Total: \$\d+\.\d{2}' } }` — the third comparator: a JS regex for dynamic content (counters, prices, dates) that neither literal mode can pin. Unanchored like `contains` (anchor with `^…$` for the `equals` analog) and — unlike the literal modes — **case-sensitive**: the pattern carries its own semantics. An invalid pattern fails at parse time. **Quote the pattern in single quotes**: single-quoted and plain YAML scalars keep backslashes; double quotes would need `\\d`. To assert a dynamic string is simply on screen with no locator, prefer a regex **selector** — `{ visible: { text: { matches: '^Taps: \d+$' } } }` (see Selectors); `text.in` + `matches` is for checking a specific element's aggregated text. -- A container's text aggregates its descendants' text (space-joined), so `text` can assert what a testID wrapper visibly shows even when the string lives in a child node. That also means `equals` against a wrapper must match _everything_ it shows or exactly the wrapper's own label/value — targeting the leaf holding exactly the value (or using `contains`) stays the clearer spelling. - -This condition-as-key form is the only spelling. `await` also accepts an optional `timeout` sibling key in milliseconds — `- await: { visible: Home, timeout: 15000 }` — for a transition that legitimately needs longer than the default budget. **Omit `timeout` by default**: the default budget covers normal transitions, and a habitual generous override just delays failure reporting on every broken step. Add one only after a step demonstrably needs it — it timed out at the default and the wait is legitimately slow (a cold start, a network round-trip, a long animation). `assert` has no timeout override: a check that needs seconds to become true is a wait — spell it `await`. - -For a custom poll interval or bundleId, drop to an explicit `- tool: await-ui-element` step — but the raw tool polls the trimmed `describe` tree, so a testID it reports as not found can still resolve fine as an `await:` directive (see Selectors). Prefer the directive. - -**`await: { idle: true }` — the one condition with no selector.** It waits until the screen has content and stops moving in **both** the UI tree and the rendered pixels. Options: `stableFor` (how long stillness must hold, default 250) and `timeout` (default 7500, and it has to leave room for the hold plus the 600ms a settle costs — three reads spanning two 200ms polls, plus the 200ms of budget the closing round has to have left to be allowed to start — or the parser rejects the step). Reach for it when a transition's motion is invisible to the tree — an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all — which is exactly when an element `await:` returns while the screen is still sliding, and the next tap lands on a moving target. - -It **never fails a run.** Readiness is not an acceptance criterion, so every outcome short of a clean settle passes carrying a `warning` on the step — read it rather than stepping over it: - -- **the screen never held still** — it spent the timeout and went ahead. Plenty of healthy screens never stop (a video, a shimmer, a carousel, live-updating text); a screen that never finished loading looks the same from here. -- **a small part of it was still changing** — a spinner, a caret, a progress dot, moving during the stretch of stillness the step settled on. Too small to be the screen moving, so the settle completed anyway; if it is a loading spinner, the screen was still loading when this step returned. -- **the tree stayed empty** — the screen rendered no accessible content. Sometimes the app (a canvas, a video surface), sometimes a screen that never arrived. -- **settled on the UI tree alone** — the screen could not be screenshotted often enough to compare a pair, so presentation-layer motion (a push, a fade, a dismissing modal) was not waited out. -- **the screen came back with content on too few reads** — a settle takes three of them spanning two polls, and this step got fewer, so it ended without ever being able to tell whether the screen was moving. A slow tree source, or a window that was blank for most of the wait. - -Only a tree source that cannot be read stops the run, as an `error` — one that is still failing when the wait ends, one that answers and then wedges, or one that never answers at all within the step (that last one may simply be slow: raise the step's `timeout` before suspecting the app). That is a broken window, not a verdict about the app: the run is not ok and every later step is skipped. A single failed read is not that window: the hold restarts from the next good one, and a read that fails at the very end of the wait is named in the warning rather than stopping the run. - -It is **not** a screen check either: a dropped tap leaves the source screen perfectly idle. Put it **after** the element `await:` that names the destination, never instead of one. There is no `assert` form (waiting is the whole point), no `when:` form, and the recorder cannot emit one — every `idle` step is hand-written. Do not sprinkle it after every step: each one costs a settle, and it cannot fail, so a flow full of them is slower without being stricter. - -### `type` and `scroll-to` - -`type` presses Enter after typing to commit the value and dismiss the keyboard, so it can't cover later targets. For a chained form whose fields feed one explicit submit — e.g. email then password then a `tap: "Log in"` — set `submit: false` on the intermediate fields so a premature Enter doesn't fire the form early: `type: { into: password, text: "hunter2", submit: false }`. - -Never record a real credential into a flow — the YAML is committed to the repo. Use a secret placeholder instead: `type: { into: password, text: "{{secret:APP_PASSWORD}}" }`. The placeholder is stored verbatim (the YAML stays secret-free) and is resolved at run time by the tool-server, from the `ARGENT_SECRET_APP_PASSWORD` environment variable or an argent secrets file — so one flow runs unchanged everywhere: - -- **CI** — the job exports `ARGENT_SECRET_APP_PASSWORD` from its secret store; the environment wins over every file. -- **A developer's machine** — the value lives in a dotenv file instead, so nothing has to be exported and no session has to be restarted: `APP_PASSWORD=…` in `~/.argent/secrets.env` (per user, any project) or in the project's `.argent/secrets.env` (**gitignore it** — the rest of `.argent/` is committed), or `ARGENT_SECRET_APP_PASSWORD=…` in the project's `.env` / `.env.local`, where only prefixed keys are exposed. - -Same placeholder for a value that is merely _external_ rather than sensitive (a test account's email, a staging tenant id) — it keeps the flow environment-independent. Note that argent treats every such value as a secret: it is redacted from errors and never echoed back, so don't use it for something a report should show. - -`scroll-to` takes an optional `direction` (`up` | `down` | `left` | `right`, default `down` — so the common case is just `- scroll-to: `) and optionally a `within: ` that anchors the scroll inside a specific container — required to drive a **nested** scroller (e.g. a horizontal carousel inside a vertical list), since the device can't be asked which container to scroll. This step-level `within` (a sibling of `target`) anchors the _gesture_, and it is the **only** scope key the step body takes — `after:`/`next:`/`any:` beside `target` are rejected. It is distinct from a selector's scopes (see Selectors), which `target` may itself carry — `scroll-to: { target: { text: Delete, within: { id: cards } }, within: { id: settings-list } }` scrolls the settings list until the Delete button _inside the cards container_ is visible. It scrolls in bounded momentum-free increments, re-checks after each, and stops if a scroll reveals nothing new (end of the container). `tap`/`type` do **not** scroll — add a `scroll-to` before any target that may be off-screen. It's a no-op when the target is already visible, so a defensive `scroll-to` costs nothing on replay and keeps the flow working on smaller screens. - -### `snapshot` cropping - -`cropOn: ` narrows a snapshot's comparison to one element's region — `- snapshot: { name: cart-total, cropOn: { id: order-summary } }`. The selector resolves like a directive target (settled tree, auto-wait, the standard not-found failure; selector-only, no point form), and the **cropped** image is what gets compared, stored as the baseline, and reported as the `current` artifact; the baseline filename still keys on the full capture's resolution — plus a `-crop-` selector suffix — so device-class drift is still caught. A cropped comparison never masks any region — every pixel of the crop is compared — so prefer elements clear of the top status-bar band (a crop overlapping it leans on best-effort status-bar pinning, and the clock/battery may diff). Crop **fixed-size containers**, addressed by `id`: a text selector resolves to the smallest matching node — typically the label itself, whose frame tracks text metrics — and the crop tracks the element's frame, so an element that grew or shrank by a pixel fails on dimensions ("nothing was compared") rather than on content. Baseline storage and seeding are covered under _Standalone runner_. - -### TV targets (Vega) - -A Vega (Fire TV) device is remote-driven — there is no touch input, so the touch directives (`tap`, `long-press`, `type`, `scroll-to`, `pinch`, `rotate`) fail on it with guidance. Drive focus with `tool: tv-remote` steps and type with `tool: keyboard` instead; everything else (`launch`, `await`, `assert`, `wait`, `snapshot`, `echo`, `run`, selectors) works unchanged — the tree comes from the on-device automation toolkit, which attaches at app launch (the `launch` step waits for it, so a leading `launch` also guarantees selectors resolve). - -```yaml -steps: - - launch: com.example.app.main # the interactive component id from manifest.toml - - await: { visible: Home } - - tool: tv-remote - args: { button: [down, select] } # move focus, then confirm — one step per navigation - - await: { visible: Explore Screen } - - snapshot: explore -``` - -Since a `tv-remote` path is positional (like a coordinate tap), gate each navigation with an `await` on the destination screen and echo where focus should be — that is what makes the flow diagnosable when the focus order changes. - -### Standalone runner - -`argent flow run [--device ] [--platform ios|android|chromium|vega] [--update-baselines] [--output ] [-r|--recursive] [--json]` runs a flow with no LLM in the loop and exits non-zero on any failure — suitable for CI (e2e flows; a fragment runs against the current device state, useful while authoring — unless its first step `run:`s a chromium e2e flow, which boots that app). The argument is either a saved flow's name, read from `.argent/flows/.yaml` under the current directory, or a `.yaml` file path (relative to the current directory, or absolute) for a flow kept anywhere else. The two never collide: a name carries no separator and no extension, so an argument ending in `.yaml` is always a path and never falls back to the flows directory. Either way the filename (minus `.yaml`) names the run's report and artifacts, so it must contain only letters, numbers, `_`, or `-`. A directory path runs every flow in it sequentially, printing only failing steps plus a final `passed/failed/skipped` flow summary (`--json` prints one aggregate object); `-r`/`--recursive` walks subdirectories too, skipping dot-directories and `node_modules`. An invalid flow file fails alone and the batch continues; an infra error stops the batch, counting the remaining flows skipped. Only a path reaches a directory — a name always resolves to one `.yaml` file. `argent flow list` prints runnable paths for flows saved under `.argent/flows/` — a nested one is addressable by its path only. - -The standalone command uses only the auto-started local tool server. It is unavailable while `ARGENT_TOOLS_URL` or `argent link` routing is active; unset `ARGENT_TOOLS_URL` or run `argent unlink` first. This restriction applies only to the CLI: an agent may continue calling `flow-execute` with `name` and `project_root`, including through a remote tool server — but a remote call uploads the one YAML into a temp directory on the server, and both `run:` targets (whose referenced files stay behind on the client) and `__baselines__/` resolve beside that copy, so only a self-contained flow replays remotely (see _Replaying_). - -`snapshot` baselines live beside the **real** (symlink-resolved) **top-level** flow file in `/__baselines__//` — directory _and_ key both come from the resolved file (for a saved flow that is a regular file, simply `.argent/flows/__baselines__//`; for one that is a symlink, `.argent/flows/smoke.yaml` → `../vault/a-smoke.yaml` keys `../vault/__baselines__/a-smoke/`, so two projects symlinking their own `smoke.yaml` into one shared vault keep separate baselines — commit baselines beside the real file under the real file's name, and move any set already committed under `.argent/flows/__baselines__//` there once) — snapshots inside composed fragments are keyed by and stored with the root flow, wherever the fragment file lives — keyed by platform + resolution; a `snapshot` step **fails** when no baseline exists for the run's device class, so seed baselines with `--update-baselines` and have the user review and commit `__baselines__/` — and pin the device class in CI (`--device`/`--platform`, same simulator model) so runs compare against the committed key. The status bar is pinned (iOS `simctl status_bar`, Android demo mode) for the run so it doesn't drive visual diffs. `--output ` writes each failed snapshot's baseline/current/diff images to `//` — a stable path for CI artifact upload. When two different flow files share a filename and one `--output` dir, the later export lands in `/-/` instead (deterministic per flow path, with a warning) so neither flow's evidence overwrites the other's. - -## Tools - -| Tool | Purpose | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `flow-start-recording` | Start recording — takes `name` + `project_root` and (fragments only) an optional `executionPrerequisite`; creates the file, truncating any existing one | -| `flow-add-step` | Execute a tool call live and record it if it succeeds | -| `flow-add-echo` | Add a label/comment that prints during replay | -| `flow-finish-recording` | Stop recording and get a summary | -| `flow-read-prerequisite` | Read a flow's execution prerequisite without running it (same `name`/`flow_path` sources) | -| `flow-execute` | Replay a flow — a saved one by `name`, or any flow YAML by absolute `flow_path` | - -Every tool during recording returns the current flow file contents, so you can track what has been recorded. Rules: - -- **Every step runs live.** You see the real tool result (including screenshots) — verify the step worked before continuing. **Only successful steps are recorded**: a failed call writes nothing to the flow file; fix the issue and try again. -- **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. -- **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. -- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` (in the return value) counts the discarded take, but can be absent even on a restart. Starting a _different_ flow abandons nothing. -- **Pick a name unique to your task.** The key is `(project_root, name)` with no ownership check: if another agent starts the same name + project while you are recording, your file is truncated and it takes the key. **Usually nothing tells you.** The liveness check runs only while an append is executing, so if you were between calls (the common case) your next `flow-add-step`/`flow-add-echo` simply re-resolves the key and appends into the other agent's recording, reporting success. You are told only in the lucky case where a step happened to be in flight at the moment of the takeover: `Recording of "" in is no longer active — it was restarted while this step was running…`. `flow-finish-recording` re-resolves the key the same way, so it finishes and clears whichever recording currently holds it — possibly the other agent's, which leaves _them_ with `Active recordings: none in this project` and no step in flight to have warned them. Either way, restart under a fresh name instead of re-adding the step. A name that only _resolves_ to the same file — a differently-cased one on macOS/Windows, or a flow (or `.argent/flows`) symlinked into a shared vault from two projects — is the same key, because the key is the file the filesystem resolves to, not the spelling you passed. That collision is reported rather than silent: the second start says `restarted` with a `discardedSteps` count, and the first recording's next call fails with `… are the same file on this filesystem …` naming both spellings. -- **Start before adding.** Calling those tools for a flow with no recording in progress returns `No active recording for flow "" in . If you have not started it yet, call flow-start-recording — but note it truncates, so if already holds a take you want (finished, or interrupted by a restart), copy it aside or record under a fresh name instead. Active recordings: ...`. The truncation caveat is there because this same error is what you get when your take was finished or dropped by the concurrent-recording cap — and on those branches the `.yaml` on disk is fully populated, so starting again destroys it. (A takeover by another agent is different: it resolves to _their_ recording and succeeds — see the previous bullet — rather than reaching this error.) The tail names only the flows live under **the `project_root` you passed** — `"checkout"`, or `none in this project` — and merely counts any others as `(plus N in other projects)`, since a shared tool-server serves callers whose project paths are not yours to see. So a mistyped `name` is spelled out for you; a wrong `project_root` shows up as your flow missing from a project you expected it in. -- **Mistakes can be edited out.** Edit the `.yaml` file directly to remove or reorder steps. - -### flow-add-step arguments - -The `command` parameter is the MCP tool name; `args` is a **JSON string** (not an object), omitted entirely for tools with no arguments: - -``` -name: "checkout-e2e" project_root: "/Users/dev/MyApp" -command: "gesture-tap" -args: "{\"udid\": \"\", \"x\": 0.5, \"y\": 0.35}" - -name: "checkout-e2e" project_root: "/Users/dev/MyApp" -command: "await-ui-element" -args: "{\"udid\": \"\", \"condition\": \"visible\", \"selector\": {\"text\": \"Continue\"}}" -``` - -Recording a `flow-execute` step carries **two** `name`s: the top-level `name` is the recording being appended to, `args.name` is the flow being run (captured as a `run:` step). - -``` -name: "checkout-e2e" project_root: "/Users/dev/MyApp" -command: "flow-execute" -args: "{\"name\": \"login\", \"project_root\": \"/Users/dev/MyApp\"}" -``` - -Caveat to the "only successful steps are recorded" rule: if that sibling is a fragment with an `executionPrerequisite`, `flow-execute` returns its prerequisite **notice** instead of running - still a successful return, so `run: login` is recorded even though nothing executed. Add `"prerequisiteAcknowledged": true` to `args` to actually run it. - -Record an `await-ui-element` step to **gate** the next step on a screen transition — it blocks until the element is `visible`/`hidden` (or contains `text`), so the following step runs only once the screen has actually settled; prefer this over a fixed `delayMs`. If its condition is not met before the timeout, replay **stops at that step** (the steps after it assume the transition happened). See the `await-ui-element` section of `argent-device-interact` for the full condition/selector reference. The live call sees only the trimmed `describe` tree — if it can't find an identifier you know exists, gate on visible text to get the step recorded, then retarget the identifier in the `await:` form during polish (the directive resolves the full hierarchy — see Selectors); don't conclude the testID is unusable in the flow. - -## Recording - -1. **Start, then launch as the first step (e2e) or set the stage yourself (fragment).** Call `flow-start-recording` with a descriptive name and the absolute `project_root`. For an **e2e** flow, record a `restart-app` of the app under test as the **first** step — it runs live (resetting the device for the rest of the recording) and is captured as the flow's `launch` step (`restart-app` has no chromium support, so on Chromium record the flow as a fragment against the running app and add the `launch:` line to the YAML afterward, deleting the `executionPrerequisite` line if you passed one — a launch-first flow must not declare it). For a **fragment**, bring the device to the entry state _before_ recording and pass an `executionPrerequisite` describing it (e.g. "App on the login screen") to `flow-start-recording` instead. -2. **Build step-by-step**: for each action, call `flow-add-step` with the same `name` + `project_root`, plus the tool name and args. The tool runs immediately — check the result before moving on, and gate each navigation with an `await-ui-element` step. -3. **Add labels**: use `flow-add-echo` (same `name` + `project_root`) between steps — echo the expected state, not just the action (see _Making flows resilient_). -4. **Finish**: call `flow-finish-recording` with the same `name` + `project_root`. It returns the file path where the flow was saved and a summary of all steps. -5. **Polish**: **read the saved `.yaml` file** and convert the raw `tool:` steps that have a cleaner directive form (the recorder leaves these as tools): - - `tool: keyboard` typing into a field → `type: { into: "", text: "…" }`, folding in the `tap` that focused the field. - - `tool: await-ui-element` gating a transition → `await: { visible: "…" }` / `{ hidden: … }` / `{ text: { in: …, equals: … } }`, carrying a custom `timeoutMs` over as a `timeout` sibling key. Converting also upgrades the wait from the trimmed `describe` tree to the flow's full-hierarchy tree (see Selectors). Keep the raw `tool: await-ui-element` step only when it sets a custom `pollIntervalMs`/`bundleId` the directive can't express. - - A scroll-to-reach-an-element — a `tool: gesture-swipe` (or its chromium analog, `gesture-scroll`) used to bring a specific element on screen before interacting with it (a `tap`, `type`, `assert`, …) → `scroll-to: { target: "", direction: … }`, dropping the swipe. This is far more robust than a fixed-distance swipe: it scrolls momentum-free and stops exactly when the target appears, so it survives layout and content changes. (`tap`/`type` do not scroll, so a raw swipe whose fling lands differently on another device leaves the following tap unresolved — always prefer the `scroll-to` rewrite.) Keep a `gesture-swipe` as a raw `tool:` step when it isn't scrolling toward a specific element — especially a velocity-dependent gesture like swipe-to-dismiss, edge-swipe-back, or swipe-to-reveal a row action, which a momentum-free `scroll-to` would not reproduce. - - `tool: gesture-pinch` → `pinch: { on: "", scale: … }`, deriving `scale` as `endDistance / startDistance`. Set `on:` to the element under the pinch center when the pinch was aimed at one (the map or image being zoomed); omit it for a screen-center pinch. Don't carry the recorded distances/angle over — the directive re-derives the geometry (finger placement, system-edge avoidance, chaining of large scales) at run time, so the conversion swaps device-specific coordinates for a portable selector with auto-wait. Keep the raw `tool: gesture-pinch` step when the pinch is anchored at a specific point _inside_ a large element (zooming toward a particular map location, not the map's center) or deliberately pans via `endCenterX`/`endCenterY` — `on:` takes only a selector and re-centers the pinch on the element's frame center, so converting would silently move the zoom anchor. - - `tool: gesture-rotate` → `rotate: { on: "", by: … }`, deriving `by` as `endAngle − startAngle` (the tool's `endAngle` > `startAngle` turns clockwise, matching the directive's positive `by`). Set `on:` to the element under the rotation center when the rotation was aimed at one (the map or image being rotated); omit it for a screen-center rotation. Don't carry the recorded `centerX`/`centerY`, radii (`radius` or `radiusX`/`radiusY`), `startAngle`, or `durationMs` over — the directive re-derives the geometry (finger placement, physical-circle radius, system-edge avoidance) and runs at a fixed pace (~90° per 300 ms), so the conversion swaps device-specific coordinates for a portable selector with auto-wait. Keep the raw `tool: gesture-rotate` step when the rotation is anchored at a specific point _inside_ a large element rather than its center (the directive re-centers on the element's frame center, so converting would silently move the pivot), when the gesture's speed itself matters (the directive's pace is fixed), or when the sweep exceeds the directive's ±3000° bound. - -Every other recorded tool (a velocity-dependent `gesture-swipe`, a fixed-distance `gesture-scroll` not aimed at an element, `button`, `screenshot`, …) has no directive form — leave it as a `tool:` step. The recorder already handles the rest: coordinate `gesture-tap`s are captured as portable `tap:` selector steps, a `restart-app` is captured as a `launch:` step, a `flow-execute` of a sibling fragment is captured as a `run: .yaml` composition directive, and device ids are stripped. Captured selectors are emitted in the strict map form (`tap: { text: General }`), never as a loose bare string — the recorder verified the exact element the tap hit, and a bare string would re-parse as loose and route through the identifier-first fallback it was never checked against. After editing, re-run with `flow-execute` to confirm the cleaned flow still passes. - -### Example session - -``` -flow-start-recording { name: "open-about", project_root: "/Users/dev/MyApp" } -flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "Start Settings from scratch" } -flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "restart-app", args: "{\"udid\": \"ABC\", \"bundleId\": \"com.apple.Preferences\"}" } # ⇒ captured as `- launch: com.apple.Preferences` — this is now an e2e flow -flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "On the Settings root list, tapping the 'General' row" } -flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.35}" } # ⇒ captured as `- tap: { text: General }` (portable selector, no udid) -flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"About\"}}" } # gate the transition -flow-add-echo { name: "open-about", project_root: "/Users/dev/MyApp", message: "On Settings > General, tapping 'About'" } -flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "gesture-tap", args: "{\"udid\": \"ABC\", \"x\": 0.5, \"y\": 0.17}" } -flow-add-step { name: "open-about", project_root: "/Users/dev/MyApp", command: "await-ui-element", args: "{\"udid\": \"ABC\", \"condition\": \"visible\", \"selector\": {\"text\": \"Model Name\"}}" } -flow-finish-recording { name: "open-about", project_root: "/Users/dev/MyApp" } -``` - -Then polish the saved file: the two `await-ui-element` steps become `await:` directives (see the file below). - -## Replaying - -Call `flow-execute` with exactly one flow source: `name` for a flow saved under `.argent/flows/` (this form also works through a remote tool server), or `flow_path` — an absolute path to any flow `.yaml`. A flow's `run:` targets and `__baselines__/` resolve on the **tool server's** filesystem, beside the YAML it actually reads. `flow_path` requires the agent and the tool server to share a filesystem and is refused when they don't; `name` is what still runs then, but it is not a way to keep siblings and baselines — a remote call reaches the server as an upload of that one YAML into a fresh temp directory, so a `run:` target errors as a missing fragment and a `snapshot` step fails for a missing baseline under a temp path (and `updateBaselines` writes the baseline there, to be deleted with the directory). Remotely, replay self-contained flows; a flow that composes or snapshots needs the agent and the tool server on one filesystem. Pass `project_root` too — it is always required here, and replaying reads no recording state, so an in-progress recording never stands in for it. **Pass `device` explicitly whenever more than one device is booted:** auto-detection resolves only when exactly one booted device matches — optionally narrowed by `platform` — and otherwise throws, listing what is available. (A Chromium e2e flow boots and tears down its own instance, but only when the launch resolves to a real Electron app path — a `launch: { chromium: }` map, or `platform: "chromium"` with `device` unset. **Don't force it with `platform: "chromium"` on a recorded flow:** the recorder writes a bare-string `launch:` holding a bundle _id_, which the boot branch reads as an app path and fails with `Electron boot: path does not exist: …`. Hand-edit the launch to `{ chromium: }` first.) If the flow has an execution prerequisite, the tool returns a **notice** with the prerequisite text instead of running — verify the prerequisite is met (you can also inspect it beforehand with `flow-read-prerequisite`, which takes the same `name`/`flow_path` pair) and call `flow-execute` again with `prerequisiteAcknowledged: true`. A flow without a prerequisite runs immediately. The run executes all steps in order and returns a structured report: `{ ok, passed, failed, skipped, errored, steps }`. - -**What each step reports.** Raw `tool:` steps include the underlying tool's full `result` (screenshots and other outputs render as usual). The directive steps are summarized: `tap`/`type`/`await`/`assert` report only `status` + `reason`, and `snapshot` adds `artifacts` only when there is something to look at — a failed comparison (baseline/current/diff paths), a missing-baseline failure (`current` only), or a baseline write; a clean pass reports just `status` + `reason`. So converting a `tool: gesture-tap` into a `tap:` directive during cleanup drops only that tap's (uninteresting) raw result — output-bearing tools like `screenshot` have no directive form and stay `tool:` steps, so their results keep flowing through. - -## Flow file format - -The top-level is an object with `steps` (array) and — fragments only — `executionPrerequisite` (an e2e flow, one beginning with `launch:`, has none). Besides the directives above: - -- `- echo: ` — a label printed during replay -- `- tool: ` with optional `args:` — a raw tool call. A tool step may also carry `delayMs: ` to sleep that long before it runs. (`await-ui-element` is an ordinary tool step; see _flow-add-step arguments_ and _Making flows resilient_ for when to gate a transition with one.) -- **`when:` blocks** handle one-sided divergences (interstitials, coach marks): `- when: { visible: "What's new" }` with a sibling `steps: [...]` list runs the block only if the condition holds — checked once with the short assert grace (~1s), so a skipped block barely costs a clean run. Guards are one condition key (`exists`/`visible`/`hidden`/`text`, the await/assert shapes) or `platform: ios|android|chromium|vega`. **No else** (parse-rejected): a block exists to dismiss the divergence and reconverge, never to test two paths — two paths are two flows. Failures inside an entered block are real failures; a skipped block reports `skip` lines. Tap-if-present is a one-step block (`when: { visible: "Got it" }` + `steps: [tap: "Got it"]`); there is NO per-step `optional:` key — it is rejected at parse with a pointer to `when:`. - -The polished result of the example session above: - -```yaml -steps: - - echo: Start Settings from scratch - - launch: com.apple.Preferences - - echo: On the Settings root list, tapping the 'General' row - - tap: { text: General } - - await: { visible: About } - - echo: On Settings > General, tapping 'About' - - tap: { text: About } - - await: { visible: Model Name } -``` - -Note there is **no device id** anywhere in the file — the recorder strips them and the runner injects the bound device. - -## When to proactively record a flow - -Proactive recording is part of this skill's scope (see the description). Record a flow without waiting to be asked — telling the user you are doing so — when you recognize any of these patterns: - -- **About to re-profile**: You completed a profiling session and are about to apply a fix and re-profile. Record the interaction steps now so the re-profile replays them identically (see `argent-react-native-profiler` and `argent-native-profiler` skills). -- **Repeating steps**: You have already performed a multi-step interaction sequence once and the task requires doing it again (comparison, retry, re-test). -- **Complex path discovered**: You worked through a non-trivial sequence of taps/swipes/navigation to reach a desired app state. Capture it before it is lost. -- **User says "again" / "one more time"**: Any request to redo what you just did is a signal to record first, then replay. +Tell the user and start a flow before re-running any path of three or more interactions for re-testing, profiling comparison, or another attempt. If the path already ran once, it cannot be recorded retroactively; start the recorder before the next execution. ## Flow self-improvement -Flows break. UI layouts change, coordinates drift, screens get added or removed. When `flow-execute` returns a failure, follow this procedure to diagnose and fix the flow instead of silently re-recording or giving up. - -### Classify the result - -After every `flow-execute`, classify the outcome before proceeding: - -| Outcome | Signal | Action | -| ---------------------- | --------------------------------------------------------------------- | ------------------ | -| **Success** | All steps completed, final screenshot shows expected state | Continue with task | -| **Hard error** | A step has `ERROR` in the result — engine stopped there | Enter **Diagnose** | -| **Silent misfire** | All steps completed but final screenshot shows wrong screen | Enter **Diagnose** | -| **Partial divergence** | Intermediate screenshot shows wrong state even though later steps ran | Enter **Diagnose** | - -For silent misfires and partial divergence, echo annotations (see _Making flows resilient_) are your reference for what each screen _should_ look like. - -### Diagnose - -1. Note the failure step index and error message (if hard error). -2. Call `screenshot` to see where the app actually is now. -3. Call `describe` or `debugger-component-tree` to get the current element tree. Remember `describe` shows less than the flow tree — a testID missing from its output can still resolve as a selector (see Selectors). - - `debugger-component-tree` is an **authoring aid only — never record a `debugger-*` step into a flow.** `device_id` is stripped at record time and re-injected at replay, but `port` is not a device-bind key, so a recorded debugger step carries whatever `port` it was given (or falls through to the 8081 default at replay) and runs against whatever Metro happens to be on that port. - - When calling any `debugger-*` tool directly, mind the shared-Metro rule: `port` is the **only** project discriminator (default `8081`), so with two RN projects running, pass the `port` of the one under test — otherwise the call lands on whichever Metro owns 8081. - -4. Compare current state to what the failed step expected. Classify the root cause: - -| Root cause | Symptoms | -| ---------------- | --------------------------------------------------------------- | -| Coordinate drift | Tap succeeded but hit wrong element; elements shifted positions | -| Missing element | Target element not present in element tree | -| Wrong screen | Screenshot shows entirely different page than expected | -| Timing | Element exists in tree but tap missed; loading spinner visible | -| State mismatch | First step fails — executionPrerequisite was not actually met | - -5. State the diagnosis in one sentence before attempting any correction. - -### Correct - -Choose the lightest strategy that fits: - -**Strategy 1 — Edit the YAML** (coordinate drift, parameter changes). -Read `.argent/flows/.yaml`, update the broken step's `x`/`y`, `bundleId`, `text`, or other args. Re-run `flow-execute` to verify. - -**Strategy 2 — Manual recovery + continue** (timing/transient issues, one-off replay). -Manually execute the failed step with corrected coordinates from the Diagnose step, then manually execute remaining steps. Does not fix the YAML — use only when re-recording is not worth it. - -**Strategy 3 — Re-record from failure point** (structural changes, new intermediate screens). -Navigate the app to the state just before the failure point. Call `flow-start-recording` with the same `name` + `project_root` — the start truncates the saved `.yaml` immediately, so copy the working prefix out of the file first. Re-add that prefix via `flow-add-step` (same `name` + `project_root`), then continue recording new steps from the divergence point. Call `flow-finish-recording` with the same `name` + `project_root`. - -**Strategy 4 — Full re-record** (major changes, unclear diagnosis, or 3+ broken steps). -Reset the app to prerequisite state (`restart-app` + `launch-app`). Record from scratch with the same `name` + `project_root` — the start truncates the old `.yaml`, so keep a copy if you may want to diff against it. - -**Decision heuristic:** - -- 1 step broken, parameter-only change → Strategy 1 -- 1 step broken, transient issue, not worth persisting → Strategy 2 -- 2–3 steps broken or flow structure partially changed → Strategy 3 -- 3+ steps broken, or unclear root cause → Strategy 4 -- Flow used for profiling comparison (must be identical) → Strategy 4 - -### Verify and bound retries - -After applying a correction, re-run `flow-execute` to verify. - -- If it succeeds → done. Report what changed (e.g. "Fixed step 4: updated tap coordinates from 0.5,0.35 to 0.5,0.42"). -- If it fails at a **different** step → return to Diagnose for a second attempt. -- If this is already the second correction attempt → **stop**. Report the diagnosis to the user and recommend a full re-record or manual investigation. - -**Hard cap: 2 correction cycles.** Do not enter an unbounded fix loop. - -### Making flows resilient - -Apply these when recording new flows to reduce future breakage: - -- **Echo expected state, not just actions.** Write `"On Settings > General screen, about to tap About"` not `"Tap About"`. During diagnosis these tell you what the screen _should_ look like. -- **Gate transitions with `await-ui-element`, not fixed delays.** After a tap that triggers a navigation, record an `await-ui-element` step that waits for the next screen's element to be `visible` (or a spinner to be `hidden`) before the following step — converted to an `await:` directive during polish. This removes the **Timing** failure mode in Diagnose (the element is in the tree but the tap fired before the screen settled) and is more reliable than `delayMs` or an extra `screenshot`. An unmet wait stops replay at that step, so a mistimed step can never run blind. -- **Add screenshot steps after critical navigation.** Insert `screenshot` steps after screen transitions. These produce images in the flow result you can inspect during diagnosis. -- **Write specific executionPrerequisites.** `"App on home tab, user logged in, simulator UDID is "` — not `"App running"`. Verify with `screenshot` + `describe` before acknowledging. -- **Prefer launch-app / open-url over navigation chains.** Deep links are more resilient to layout changes than tap sequences. -- **Echo accessibility labels for coordinate taps.** When recording a tap, add an echo with the target's label or testID: `"Tapping 'Submit' button (testID: submit-btn) at 0.5, 0.82"`. During repair, use `describe` to find the element by label and update coordinates. Only use `screenshot` for permission or system overlays when `describe` cannot expose the target reliably. +When a saved flow fails, do not silently discard it or patch around the failed check. Follow [Reliability and recovery](references/reliability-and-recovery.md): classify the failure, inspect the actual screen/tree, repair the smallest justified unit, audit again, and replay the full flow. Stop after two unsuccessful correction cycles and report the remaining blocker. diff --git a/packages/skills/skills/argent-create-flow/references/flow-yaml.md b/packages/skills/skills/argent-create-flow/references/flow-yaml.md new file mode 100644 index 000000000..79a674f5c --- /dev/null +++ b/packages/skills/skills/argent-create-flow/references/flow-yaml.md @@ -0,0 +1,183 @@ +# Flow YAML + +Read this reference when polishing, manually reviewing, or composing a flow. + +[File shape and flow type](#file-shape-and-flow-type) · [Selectors](#selectors) · [Directives](#directives) · [Verification conditions](#verification-conditions) · [Prove a navigation](#prove-a-navigation-identity-then-readiness) · [Optional divergences](#optional-divergences) · [Composition and platform limits](#composition-and-platform-limits) · [Snapshots and standalone runs](#snapshots-and-standalone-runs) · [YAML safety](#yaml-safety) + +## File shape and flow type + +```yaml +steps: + - launch: com.example.app + - await: { visible: { id: home-screen } } + - await: { idle: true } +``` + +An e2e flow's first non-echo step is `launch:`, and it must not declare `executionPrerequisite` — the combination is a parse error. Put the named start state in a leading `echo:` instead. A fragment has no leading launch and may declare: + +```yaml +executionPrerequisite: User is signed in and viewing Settings +steps: [] +``` + +Flows never store a device id. The runner binds the selected/booted device. `launch:` restarts the app process; it does **not** clear persisted app, account, or backend data. + +## Selectors + +Use selector values that meet the [stable-selector definition](../SKILL.md#stable-selectors). Write explicit selector maps: + +```yaml +{ id: save-button } # exact testID/accessibilityIdentifier/resource-id +{ text: Save } # case-insensitive text/label substring +{ role: button } # case-insensitive role substring +{ id: settings-row, text: Notifications } # fields all must match +``` + +`id` is exact and case-insensitive; an unqualified Android id such as `save-button` also matches `com.example:id/save-button`. `identifier` parses as an alias for `id`, but `id` is canonical and is what the recorder writes. A bare string is loose shorthand that tries id first and then text. Never write a bare string in a flow you author; always write the explicit map. + +For dynamic native text, use an anchored, case-sensitive regex and single quotes: + +```yaml +{ text: { matches: '^Order #\d+$' } } +``` + +### The runner tree is not the discovery tree + +Flow selectors resolve against the runner's tree. The agent-facing `describe` tool and the live `await-ui-element` tool read a **different** projection of the same screen, and how the two differ is platform-specific — it decides what a missing element means: + +| Platform | Runner tree | `describe` / `await-ui-element` | How they differ | +| -------- | ------------------------------------------------------------------------------------------ | --------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| iOS | native UIView hierarchy | accessibility tree | Each holds elements the other lacks; the role vocabularies are disjoint (`AXButton` exists only in `describe`) | +| Android | full accessibility hierarchy, including not-important views | the same dump, trimmed to interactables | Each holds elements the other lacks; the trim drops testID-only containers and merges nodes | +| Chromium | the same DOM walk, keeping only nodes with an id, label, value, clickable or focused state | the whole DOM walk | The runner tree is a strict **subset** | +| Vega | toolkit page source | the same source | Same elements, re-shaped | + +Two consequences, both load-bearing: + +- **On iOS and Android**, a missing id in `describe` is not proof that the flow selector cannot resolve: prefer the id and verify it in a scratch fragment. **On Chromium the reverse holds** — what `describe` does not show, no selector can reach, and an element it does show carrying none of those five attributes is invisible to the runner. Give that element a testid instead of hunting for another selector. +- A live `await-ui-element` check can pass against the tool's tree and mean nothing to the runner. The recorded `tool:` step still replays (that tool reads the tree it passed against); it is the `await:`/`assert:` directive polish converts it into that may not resolve. Replay the flow after polish and treat an unresolved converted wait as a polish-time blocker, not a recording failure. + +When several visible nodes match, an exact text/identifier match beats a substring hit, then the smallest frame wins. Use a stricter map when that ranking could still select the wrong repeated element. + +### Relational scopes + +Flow YAML selectors also accept geometric, CSS-like relations. They work in every flow selector slot (`tap`, `type.into`, `await`, `assert`, `scroll-to`, `pinch.on`, `rotate.on`, `snapshot.cropOn`, and nested scopes), but not in the live `await-ui-element` tool. + +```yaml +- tap: { text: Delete, within: { id: profile-card } } # inside a container +- assert: { visible: { role: Button, after: { text: Danger zone } } } # any follower +- tap: { role: Switch, next: { text: Wi-Fi } } # nearest matching follower +``` + +The relations are frame-based because platform flow trees are flattened. `within` means visual containment. `after` and `next` use top-to-bottom/left-to-right reading order. Every anchor must be distinct; the synthetic screen root never counts. + +`next` means the nearest **matching** follower, so it deliberately skips wrappers, spacers, and other non-matches. This differs from literal CSS `+`: if a Wi-Fi row has no switch, `{ role: Switch, next: { text: Wi-Fi } }` may find the next row's switch. Prefer `{ role: Switch, within: { id: wifi-row } }`, or assert that row-local control before acting, whenever the control may be absent. + +Scopes may combine and nest, with at most six scope keys per selector. Scope the target by a trusted container when a missing row control must fail instead of reaching another row. Use a strict map for an anchor you care about; a bare string keeps the loose identifier-first fallback and can bind to an unrelated id. + +## Directives + +Every directive hard-stops the flow on failure; later steps are skipped. The set is `launch`, `tap`, `long-press`, `type`, `scroll-to`, `pinch`, `rotate`, `await`, `assert`, `wait`, `snapshot`, `run`, `when`, `echo`, and `tool`; `flow-execute`'s own tool description spells out each one's shape and options. What follows is only what that description does not say. + +A bare `launch: com.acme.app` applies to every platform — and on Chromium it is read as the app **path**, so any flow that must run cross-platform needs the map form. `native:` covers iOS, Android, and Vega with one id: `- launch: { native: com.acme.app, chromium: ../../app }`. An Android app that must start on a non-launcher activity has no `launch:` form at all; record `restart-app` with its `activity` and accept that the flow is a fragment. + +A `scroll-to` map is always the options form — the target goes under `target:`. Only the bare-string form (`scroll-to: Logout`) omits it, and that spelling is a loose selector. + +`tap`, `type`, and `long-press` do not auto-scroll. Add `scroll-to` first whenever the target may be off-screen. `scroll-to` defaults to `down`, is a no-op if already visible, and needs `within` for a nested scroller. + +`type` presses Enter unless `submit: false`. A polished focus-tap + raw keyboard pair normally needs `submit: false`, because the recording did not submit. Store secrets as `{{secret:APP_PASSWORD}}`; the runner resolves them from `ARGENT_SECRET_APP_PASSWORD` or a secrets file (`.argent/secrets.env`, `~/.argent/secrets.env`, `.env`), so one flow runs unchanged in CI and locally. Use it for any external value, not only sensitive ones — but every resolved value is redacted from output, so never use it for something a report must show. + +## Verification conditions + +The condition name is the key and its value is the selector: + +```yaml +- await: { visible: { id: settings-screen } } +- await: { hidden: { id: loading-spinner }, timeout: 15000 } +- assert: { exists: { id: notifications-toggle } } +- assert: { text: { in: { id: preference-status }, equals: Enabled } } +- assert: { text: { in: { id: result-count }, matches: '^\d+ results$' } } +``` + +`text.in` locates one selector and compares its rendered/descendant text with exactly one of `contains` (case-insensitive substring), `equals` (case-insensitive exact match), or `matches` (case-sensitive JS regex). Substring boundaries are not implied: `contains: "Taps: 3"` is also satisfied by `Taps: 30`, so use `equals` or an anchored `matches` pattern when the complete value matters. Use a regex selector under `visible` when only the shape of free-standing text matters. + +Use `await` for an outcome that may take time and `assert` for settled state. The default `await` wait is 7500 ms; `assert`'s fixed grace is 1000 ms. Add an `await.timeout` only after the 7500 ms default demonstrably expires, and only above it — a value below 7500 shortens the wait. `assert` rejects `timeout`; a timed check is an `await`. + +A negative condition such as `hidden` only says that no visible match exists in the current tree. It is true before the element ever appears, true if the selector is misspelled, and true on the wrong screen. Establish it positively first: prove the containing screen, and prove the same selector `visible` at some earlier point in the flow. A `hidden` whose selector never matched anywhere in the flow passes on every replay no matter what the app does, so it is not a check at all. When possible also verify a positive replacement or empty state. + +## Prove a navigation: identity, then readiness + +A navigation needs two checks, and no single one covers both: + +```yaml +- await: { visible: { id: profile-screen } } # identity: WHICH screen +- await: { idle: true } # readiness: it stopped moving +``` + +**They are independent.** A dropped tap leaves the source screen perfectly idle, so readiness never proves identity. A destination element enters the tree while the transition is still animating over it, so identity never proves readiness. + +Identity is an element that exists **only** on the destination ([which ones qualify](reliability-and-recovery.md#strong-transition-gates)). + +### `idle` — readiness + +The one condition that carries no selector, because stillness is a property of the whole screen. It holds until the screen has content and stops moving in **both** the UI tree and the rendered pixels, and fails if it never does. A tree that stays empty or unreadable is `errored` instead — the check could not run, which is not a verdict about the app. + +```yaml +- await: { idle: true, minStableMs: 400, timeout: 9000 } +``` + +The pixel half is why it exists: an iOS push or modal dismissal commits its hierarchy up front and then animates a layer for a few hundred milliseconds, and a cross-fade or scrim moves no node at all. A tree-only wait returns while the screen is still sliding, and the next tap lands on a moving target. + +`minStableMs` (default 250) is how long stillness must hold; it must be shorter than `timeout` (default 7500) or the gate could never pass, and parse rejects it. Stillness is measured across intervals, so a settle takes at least three reads: `minStableMs: 0` means "the first two agreeing intervals", not "the first read". + +This is the persistable counterpart of the `await-screen-idle` tool, whose soft `settled: false` cannot carry a verdict on an unattended replay. Prefer it over `wait:` for any transition with no element to gate on. It has no `assert` form — waiting is the whole point. + +Three limits. It says nothing about **which** screen settled, so it never replaces the identity gate. Where no screenshot could be read it still passes on the tree alone, reporting a ⚠ warning that says so — treat that as the half-proof it names. And it cannot settle on a screen that never stops moving (a looping animation, a video, an advancing carousel) — gate on the element you actually need there. + +Add one during polish after each screen change, not after every step: it is a directive with no live tool behind it, and each costs roughly 0.5-1.5 s warm — more on the first capture of a run. + +## Optional divergences + +Use `when:` only for a one-sided optional path such as a coach mark: + +```yaml +- when: { visible: { text: Got it } } + steps: + - tap: { text: Got it } +``` + +The guard may be one `exists`/`visible`/`hidden`/`text` condition or `{ platform: ios|android|chromium|vega }`. It uses the short assert grace. There is no `else` and no per-step `optional`; separate behavioral paths belong in separate flows. A skipped optional block is not a failure, but a required acceptance check must never live behind a condition that may skip it. + +## Composition and platform limits + +- iOS/Android e2e flows may run fragments or other e2e flows inline; a nested e2e launch restarts its app. +- Chromium boots one Electron app for the top-level run. Do not nest a Chromium e2e flow with its own launch; make it top-level or turn the nested flow into a fragment. `pinch` is rejected there — drive the app's own zoom controls instead. +- Vega is remote-driven. Touch directives (`tap`, `long-press`, `type`, `scroll-to`, `pinch`) are unsupported; record `tool: tv-remote` and raw `tool: keyboard`, then gate every focus/navigation result with `await`. + +## Snapshots and standalone runs + +`argent flow run [--device ] [--platform ios|android|chromium|vega] [--update-baselines] [--output ] [--json]` runs without an LLM and exits non-zero on failure. + +A raw `screenshot` captures evidence but never compares or fails on visual drift. Keep it for live inspection, diagnosis, or a human-reviewed before/after result. A `snapshot:` directive is executable verification: it compares the current pixels with a stored baseline and hard-fails on a missing baseline, excessive mismatch, or (for `cropOn`) region-size drift. + +Use a snapshot when pixels are part of the requirement and structural selectors cannot prove the rendering: + +- light/dark theme or other global color-mode changes; +- layout, position, size, spacing, typography, clipping, overflow, image, or icon rendering; +- a stable component whose appearance matters beyond its accessibility state. + +Pair visual and structural evidence when the requirement is mixed. Use a full-screen snapshot for a global theme/layout and `cropOn` for one stable component to reduce unrelated noise. + +Do not use a snapshot as the only proof of navigation, persistence, data correctness, accessibility state, logs, or network behavior. Avoid it when timestamps, random/live data, ads, uncontrolled animation, or device drift make the pixels unstable. First make the screen deterministic and gate its identity/readiness with `await:` or `assert:`. + +Snapshot baselines live under `.argent/flows/__baselines__//`, keyed by platform and full-capture resolution; `cropOn` also keys by selector. A `snapshot:` step fails when no baseline exists for the run's device class. Seed from a known-good state with `--update-baselines`, inspect every generated baseline against the requirement, tell the user it requires review, and do not commit it yourself. Baseline creation or update is not a test pass. Never update a baseline merely to make a failing diff green. The default `maxMismatch` is `0.5` percent. + +For iOS, Android, or Vega, record the seeding run's `--platform` and `--device` values and pass the same flags in CI so the device class stays pinned. For Chromium, persist the window-size argument in `launch.chromium.args`, run with `--platform chromium`, and omit `--device` so the runner boots the declared target with those arguments; pinning an already-running Chromium device bypasses the launch. The runner pins the iOS/Android status bar during a run to keep clock, battery, and signal changes out of full-screen visual diffs. + +`--output ` writes failed snapshot baseline/current/diff images under `//`, a stable directory for CI artifact upload. + +## YAML safety + +Quote strings containing `#`, `:`, or quotes. A bare `true`/`false` or a number in a text slot is rejected at parse, so quote those too; `yes`/`no`/`on`/`off` parse as plain strings. Use single quotes for regex containing backslashes. + +Parse failures include invalid directives, selector shapes, regexes, `else`, unsupported options, and an e2e flow that combines a leading `launch:` with `executionPrerequisite`. diff --git a/packages/skills/skills/argent-create-flow/references/live-authoring.md b/packages/skills/skills/argent-create-flow/references/live-authoring.md new file mode 100644 index 000000000..c1f2bda54 --- /dev/null +++ b/packages/skills/skills/argent-create-flow/references/live-authoring.md @@ -0,0 +1,239 @@ +# Live authoring + +Read this file before creating or changing a flow. The saved path must be exercised through the recorder as it is discovered; only the final syntax cleanup happens afterward. + +[Recorder tools](#recorder-tools) · [Start in the correct order](#start-in-the-correct-order) · [Record the first walkthrough](#record-the-first-walkthrough) · [Finish and polish](#finish-and-polish) · [Worked example](#worked-example) · [Blocking audit](#blocking-audit) · [Replay](#replay) + +## Recorder tools + +`flow-add-step`'s `command` parameter takes an MCP tool name. Its `args` value is a JSON **string**, not an object, and is omitted for a no-argument tool: + +```text +command: "gesture-tap" +args: "{\"udid\":\"DEVICE\",\"x\":0.5,\"y\":0.35}" +``` + +Recording a `flow-execute` carries **two** flow names: the top-level `name` is the recording being appended to, `args.name` is the sibling being run (captured as a `run:` step). + +### Recording contract + +- **Every recording tool takes `name` + `project_root`.** `flow-add-step`, `flow-add-echo`, and `flow-finish-recording` each name the recording they address, repeating the `name` and the absolute `project_root` (an error is returned if the path is not absolute) given to `flow-start-recording`. Nothing is carried over between calls. +- **Recording _state_ is isolated; the device is not.** A recording is keyed by its output file, `/.argent/flows/.yaml`, so several can be open at once — different names, different projects — and one recording's steps never land in another's file. Nothing is isolated on the device: every step runs live, so two recordings driving one device interleave real UI actions, and one flow's recorded `restart-app` resets the app under the other. Give each concurrent recording its own device. +- **Starting always truncates the `.yaml`.** `flow-start-recording` resets `/.argent/flows/.yaml` to an empty flow on every call — including a name that is only a saved file with no recording in progress, so starting under the name of a committed flow wipes it. `restarted: true` is reported only when a LIVE recording of that flow was discarded, so its **absence does not mean nothing was overwritten**. `discardedSteps` (in the return value) counts the discarded take, but can be absent even on a restart. Starting a _different_ flow abandons nothing. +- **Pick a name unique to your task.** `(project_root, name)` has no ownership check, so another agent starting the same pair silently takes it over and your later appends land in _their_ recording, reporting success. If a call ever reports the recording is no longer active, restart under a fresh name rather than re-adding the step. +- **Start before adding.** Adding to or finishing a flow with no recording in progress returns `No active recording for flow ...`, listing the flows live under the `project_root` you passed. Do not answer it with `flow-start-recording`: you get the same error when your take was finished or dropped by the concurrent-recording cap, and on those branches the `.yaml` is fully populated, so starting truncates a take you wanted. Copy the file aside or record under a fresh name. (A takeover by another agent does not reach this error at all — it resolves to _their_ recording and succeeds, per the previous bullet.) +- **Only successful steps are recorded.** A failed call writes nothing to the flow file; fix the issue and try again. Every recording tool returns the current flow file contents, so you can track what has been recorded as you go. +- **Edit mistakes out after finishing.** Remove or reorder steps in the `.yaml` once `flow-finish-recording` has run; editing it while the recording is still active can be overwritten by the in-memory copy. + +## Start in the correct order + +### iOS, Android, and Vega e2e flows + +1. Call `flow-start-recording` before launching or touching the app. +2. Make the first non-echo recorded action a plain `restart-app` with the device id and app id only. The recorder stores it as `launch:`. Extra restart arguments or a `delayMs` prevent that conversion — an Android `activity` is the case that turns up in practice, and it leaves a raw `tool:` step, so the flow is a fragment rather than e2e. +3. Immediately record `await-ui-element` for the real first screen. Launch waits for platform automation readiness, not for app-specific loading or splash completion. + +Never build a selector, landmark, or echo reference from splash-screen content; wait for the real first screen and base the recorded path on that state. + +On iOS, Argent must launch the app for the full selector tree to exist, and only `restart-app` guarantees that — `launch-app` foregrounds an already-running, uninstrumented process instead. See [Reliability and recovery: iOS selector recovery](reliability-and-recovery.md#ios-selector-recovery). + +### Chromium e2e flows + +**Default Chromium window size: `1366 × 768`.** Unless the user or test contract explicitly requires another window size, boot the target with `boot-device`, `electronAppPath`, and `electronArgs: ["--window-size=1366,768"]`. This is the native browser-window size, not page-viewport emulation. Do not record against an already-running target whose window size came from host or session state; launch a fresh target with the explicit size first. If the target cannot honor the requested size, stop and report the blocker instead of recording at a different size. + +Call `flow-start-recording` after that boot and before the first in-app action, then record the first-screen wait live. During polish, insert a leading Chromium launch that preserves the same app path and arguments, for example: + +```yaml +steps: + - launch: + chromium: + path: ../../app + args: ["--window-size=1366,768"] +``` + +The path is relative to the flow file's own directory, `.argent/flows/`, so `../../app` means `/app`; an absolute path is also accepted. Preserve any other live boot arguments, and keep exactly one `--window-size` argument. An explicit user or test-contract size replaces the default in both the live boot and saved launch. This packaging exception represents the same app boot used for the live walkthrough; it is not permission to rehearse the UI path. + +### Fragments + +Stage the documented entry state before recording, then call `flow-start-recording` with a precise `executionPrerequisite`. Describe UI/account/platform state, never a concrete device id. Start recording before the first interaction that belongs to the fragment. + +## Record the first walkthrough + +For Vega, read [Flow YAML: Composition and platform limits](flow-yaml.md#composition-and-platform-limits) before applying this cycle — it is remote-driven and takes no touch gestures. + +**Reach every screen by tapping through the app's own UI.** A recorded `open-url` skips the navigation the flow exists to exercise, so a broken entry point still passes. Starting the app is not navigation. + +Repeat this cycle for every action: + +1. **Discover without mutating.** Call `describe`, `native-find-views` / `native-describe-screen` (iOS only), `debugger-component-tree` (React Native), or `screenshot` directly. These calls are intentionally not recorded. Never record a `debugger-*` step: `port` is not a device-bind key, so a recorded one replays against whatever Metro owns that port. +2. **Choose a durable target.** Prefer an id, then a text/accessibility label meeting the [stable-selector definition](../SKILL.md#stable-selectors). For iOS ids, use `native-find-views` / `native-describe-screen`; the trimmed accessibility description may omit them. +3. **Narrate before failure can occur.** Add an echo naming the current state, intended action, and expected destination/outcome. +4. **Execute and record immediately.** Call `flow-add-step`; inspect `toolResult`, the `message`, and the returned flow file before moving on. +5. **Verify immediately.** After navigation, record identity then readiness (below). Record requested outcome checks when the state first appears. + +### Record identity, then readiness, after every navigation + +On the destination screen, in this order: + +1. **Identity.** Record `await-ui-element` `visible` on an element that exists **only** on the destination — its root id, or a control no other screen in the flow shows. Anything the source screen also has passes without the navigation happening, so it proves nothing. +2. **Readiness.** Add `- await: { idle: true }` during polish, and name in the preceding echo what it is waiting out. It is a directive, not a tool, so it cannot be recorded live — do not reach for the `await-screen-idle` tool instead, whose soft `settled: false` carries no verdict on an unattended replay. + + Where a specific control marks the screen usable, record an `await-ui-element` on that control as well — but it does not replace the `idle` gate, which is what waits out motion the tree cannot see. + +[What qualifies as destination-only](reliability-and-recovery.md#strong-transition-gates). + +### Record absence in three steps, in this order + +A `hidden` wait whose selector never matched anywhere in the flow can never fail — it passes just as happily on a typo'd selector or the wrong screen. Record the trio: + +1. `await-ui-element` `visible` on the selector, while the element is on screen; +2. the action that removes it; +3. `await-ui-element` `hidden` on the same selector. + +Step 1 is what makes step 3 falsifiable, so it must carry the **same** selector — an id or spelling that drifts between the two leaves the absence check proving nothing. A step-1 locator with no identity term of its own (a regex `text` match, a role-only selector) is no evidence either. + +### Taps + +`flow-add-step` cannot receive a flow selector directly. Locate the element by id/text first, then record `gesture-tap` at the center of its discovered frame. The recorder reads the **pre-tap** tree and stores a strict `tap: { id: ... }` or `tap: { text: ... }` selector. The live coordinates are transport for the gesture, not an acceptable final locator. + +When the element cannot be addressed the recorder keeps the raw point, appends the step anyway, and **warns with the reason and the retarget**. Act on that warning before the next action: to replace a kept coordinate, return to the screen the tap started from — with direct MCP calls, never through `flow-add-step` — record the corrected tap, and delete the coordinate step after `flow-finish-recording`. Do not leave both. Keep a point only after the **coordinate fallback gate** in [Reliability and recovery](reliability-and-recovery.md#coordinate-fallback-gate). + +**Never record a tap on the on-screen keyboard.** Some platforms expose the whole keyboard as ONE addressable node, so a tap on a key records a selector for the keyboard and replays at its centre — a different key, reported as a pass. The recorder cannot tell that node from a legitimate large control. Type with `keyboard`, which polish folds into `type:`. + +### Typing + +Record the focus tap, then use `describe` to confirm the field is focused before recording `keyboard`. Use `describe` or an app validation marker to confirm the complete value appeared; for a secure field, do not expose the value in a screenshot or echo. If characters were lost, restore the field with direct MCP tool calls (never through `flow-add-step`). Do not record a duplicate typing step. + +Polish folds the focus tap and keyboard step into `type:`, which at replay re-taps the field and waits for it to take focus before injecting keys. That wait is a best effort, not a guarantee — keys go wherever focus actually is, at replay as much as during the walkthrough — which is why the live `describe` check stays and why the value is verified after typing. + +Never record a credential literal. Use `{{secret:NAME}}`, resolved at run time from `ARGENT_SECRET_NAME` or a secrets file — see the `keyboard` section of `argent-device-interact` for the full source order. + +### Scrolling and swiping + +During the live walkthrough, record `gesture-swipe` or Chromium `gesture-scroll` when movement is required. During polish: + +- movement whose purpose is to reveal an element becomes selector-targeted `scroll-to`; +- a raw swipe survives only when the gesture itself is under test, such as swipe-to-dismiss, edge-back, or reveal-row-actions. + +For every retained raw gesture, add an echo naming the gesture target and record an `await-ui-element` condition that proves its result; polish that condition to `await:` or `assert:`. + +### Live waits and checks + +Record `await-ui-element` through `flow-add-step`. If its condition is unmet, the response is not an error: `message` contains `step NOT recorded` and `toolResult.success` is `false`. Nothing was appended. Fix the selector or justified timeout and call it again. Never proceed as though the gate passed. Read the `await-ui-element` section of `argent-device-interact` for the complete live condition and selector reference. + +**A wait that passes live can still be unconvertible**, because the tool and the runner read different projections of the screen ([per-platform table](flow-yaml.md#the-runner-tree-is-not-the-discovery-tree)). The raw step replays fine either way; it is the `await:`/`assert:` conversion that can fail to resolve. Check each converted wait at polish by replaying the flow, and either re-record it with a selector present in both trees or keep the step raw on purpose. + +The live wait tool spells an identifier selector as `identifier`; polished flow YAML uses canonical `id`. + +### Wrong turns + +Stop immediately. Restore the last valid screen with direct MCP tool calls — invoked normally, never through `flow-add-step`, so nothing is appended to the flow — note the bad recorded step, and continue only from verified state. Do not edit a flow file while its recording is in progress: remote/client recording may keep an authoritative in-memory copy. Remove the bad step after `flow-finish-recording`; if recovery changed or skipped meaningful behavior, re-record that portion live instead of fabricating it. + +## Finish and polish + +Call `flow-finish-recording`, then read the saved YAML. For recorded steps, apply only semantics-preserving conversions: + +| Recorded form | Finished form | +| ----------------------------------------- | ------------------------------------------------------------------- | +| focus `tap` + `tool: keyboard` | `type: { into: , text: ..., submit: false }` | +| keyboard text ending in Enter | `type:` without `submit: false`; remove Enter from `text` | +| `tool: await-ui-element` transition/check | `await:` or `assert:` | +| element-seeking swipe/scroll | `scroll-to:` with target selector, direction, and optional `within` | +| coordinate `tap`/`long-press` | strict id/text selector after the coordinate fallback gate | +| `tool: gesture-pinch` | `pinch: { on: , scale: endDistance / startDistance }` | +| generic `tool: flow-execute` of a sibling | recorder-captured `run:` directive | + +Only three unrecorded insertions are allowed during polish, each where you saw the condition live: + +- `snapshot:`, which captures state without performing a new app action. Follow [Flow YAML: Snapshots and standalone runs](flow-yaml.md#snapshots-and-standalone-runs) for baseline handling. +- `await: { idle: true }` as the readiness half of a navigation, named by the preceding echo. +- The documented leading Chromium `launch:` that packages the same app path and arguments used by the live `boot-device` call. + +Preserve a raw form only when conversion would change behavior: + +- Keep `tool: await-ui-element` only when its `pollIntervalMs` or `bundleId` is required. It is unrelated to the fixed `wait:` directive. +- Keep a raw swipe only for a semantic or velocity-sensitive gesture. +- For a pinch, derive `scale` from the recorded distances and target the selector under its center. +- Keep `tool: gesture-pinch` only when it is point-anchored inside a large element or deliberately pans with `endCenterX`/`endCenterY`; the directive would recenter it. +- Keep raw screenshots for useful human-reviewed before/after or diagnostic evidence. For automated visual verification, add `snapshot:` according to [Flow YAML](flow-yaml.md#snapshots-and-standalone-runs). + +See [Flow YAML](flow-yaml.md) for exact syntax. + +If polish reveals a missing action or acceptance check, the flow is incomplete. Restore its preceding state and execute the missing behavior through the recorder, or re-record; do not append remembered behavior directly to YAML. + +## Worked example + +This session records a third-party app path; the comments show what the recorder captures before polish: + +`FLOW` below abbreviates `name: "open-settings", project_root: "/Users/dev/AcmeNotes"` — every call repeats it verbatim. + +```text +flow-start-recording { name: "open-settings", project_root: "/Users/dev/AcmeNotes" } +flow-add-echo { FLOW, message: "Restart Acme Notes; expect the real Home screen" } +flow-add-step { FLOW, command: "restart-app", args: "{\"udid\":\"ABC\",\"bundleId\":\"com.acme.notes\"}" } +# captured as: - launch: com.acme.notes +flow-add-step { FLOW, command: "await-ui-element", args: "{\"udid\":\"ABC\",\"condition\":\"visible\",\"selector\":{\"identifier\":\"home-screen\"}}" } +flow-add-echo { FLOW, message: "On Home; open Settings and expect the Settings screen" } +flow-add-step { FLOW, command: "gesture-tap", args: "{\"udid\":\"ABC\",\"x\":0.91,\"y\":0.94}" } +# pre-tap capture resolves the point to: - tap: { id: settings-tab } +flow-add-step { FLOW, command: "await-ui-element", args: "{\"udid\":\"ABC\",\"condition\":\"visible\",\"selector\":{\"identifier\":\"settings-screen\"}}" } +flow-finish-recording { FLOW } +``` + +After converting the recorded `await-ui-element` tools to directives, the finished flow is: + +```yaml +steps: + - echo: Restart Acme Notes; expect the real Home screen + - launch: com.acme.notes + - await: { visible: { id: home-screen } } # identity: which screen + - await: { idle: true } # readiness: it stopped moving + - echo: On Home; open Settings and expect the Settings screen + - tap: { id: settings-tab } + - await: { visible: { id: settings-screen } } # identity: which screen + - await: { idle: true } # readiness: it stopped moving +``` + +Both screen changes carry the pair. Each `idle` gate is added during polish — it is a directive with no live tool to record. + +## Blocking audit + +Review the file before replay. Run all four greps and resolve every hit. + +```text +# 1. Coordinates and raw gestures +rg -n '(\{ *x:|^ +(x|centerX|fromX|toX):|gesture-(tap|swipe|scroll|drag|pinch|rotate|custom))' .argent/flows/.yaml + +# 2. Stored device ids +rg -n '(udid|device_id)' .argent/flows/.yaml + +# 3. Unstable gate values: positional ids, and bare (loose) selectors in a condition +rg -n '(-selector-\d+|selector-\d+\b)' .argent/flows/.yaml +rg -n '(await|assert):.*(visible|hidden|exists) *: *["'"'"'A-Za-z0-9]' .argent/flows/.yaml + +# 4. Fixed sleeps, and navigation that skipped the UI +rg -n '^\s*- wait:' .argent/flows/.yaml +rg -n 'open-url' .argent/flows/.yaml +``` + +- Convert every element-targeting point to a selector. Each coordinate `tap:` warned you when it was recorded; this grep is the second chance, not the first, so every remaining hit has to defend itself. +- Convert every raw gesture used only to find an element to `scroll-to`. +- For each remaining point/raw gesture, require the exception evidence and expected-result check from the **coordinate fallback gate** in [Reliability and recovery](reliability-and-recovery.md). +- Require zero stored device ids and zero literal credentials. +- **Reject every positional id in a gate.** Replace with a stable destination-only root or control id. +- **Rewrite every bare-selector condition into an explicit map.** Grep 3's second pattern lists them (it matches `visible: Save`, not `visible: { text: Save }`). +- **Reject data-derived gate values** — a counter, count, username, timestamp, or any number the app computes. Read every `text:` gate and confirm its value is fixed by the app's code; use an anchored `{ matches: '^…$' }` when the full value matters. +- **Every `wait:` must justify itself** — a preceding echo and a following `await:`/`assert:` that proves the state. Prefer replacing it with `await: { idle: true }`. +- **Reject every `open-url` that stands in for a navigation** — restore the source screen and record the tap path live. +- Confirm every added `snapshot:` is intentional, non-mutating, and ready for reviewed baseline creation. +- Confirm the first non-echo e2e step is `launch:` and the next functional step gates the real first screen. If either is missing on mobile/Vega, record it live; only the documented Chromium packaging launch may be inserted during polish. +- **Confirm every navigation has identity and readiness proof** — walk the file top to bottom and, for each action that changes screens, name the two gates that follow it. If either is missing, restore that screen and record it live. +- Confirm every `hidden` gate is preceded by evidence its selector is real: the same selector asserted `visible` earlier in the flow, or a proven containing screen. + +## Replay + +Run `flow-execute` on the complete polished flow with the absolute project root. For a fragment, verify its prerequisite and rerun with `prerequisiteAcknowledged: true` when requested. A replay you rescued by hand is not a pass: if you tapped, waited, or reset anything to get the run through, the pass does not count. + +An `errored` step is not a failed one: it could not be evaluated at all — an unreadable tree, focus unconfirmed with nothing to read it from. Fix the environment named in its reason and rerun; it is not a verdict about the app and never counts for or against a pass. + +The base create-flow gate is one uninterrupted full pass of the finished YAML. Return to the invoking skill for any stronger completion rule: `argent-qa-flows` requires two consecutive full passes of the unchanged flow. For CI, use `argent flow run [--platform ...]`; it exits non-zero on failure. diff --git a/packages/skills/skills/argent-create-flow/references/reliability-and-recovery.md b/packages/skills/skills/argent-create-flow/references/reliability-and-recovery.md new file mode 100644 index 000000000..28e590923 --- /dev/null +++ b/packages/skills/skills/argent-create-flow/references/reliability-and-recovery.md @@ -0,0 +1,146 @@ +# Reliability and recovery + +Read this file when selector capture warns, a finished file contains coordinates/raw scrolling, a transition or overlay can swallow an action, the platform's flow tree source is unavailable (iOS native devtools, the Android helper, a Chromium CDP session, the Vega toolkit), or a replay fails. + +[Coordinate fallback gate](#coordinate-fallback-gate) · [iOS selector recovery](#ios-selector-recovery) · [Tree source recovery on Android, Chromium, and Vega](#tree-source-recovery-on-android-chromium-and-vega) · [Strong transition gates](#strong-transition-gates) · [Obscured targets and persistent overlays](#obscured-targets-and-persistent-overlays) · [Diagnose a replay failure](#diagnose-a-replay-failure) · [Correct the smallest justified unit](#correct-the-smallest-justified-unit) + +## Coordinate fallback gate + +Use this order for every element-targeting action, applying the [stable-selector definition](../SKILL.md#stable-selectors): + +1. strict `{ id: ... }`; +2. narrow, stable `{ text: ... }` or accessibility label; +3. stable role only when it uniquely identifies the element; +4. `scroll-to` plus one of those selectors for an off-screen target; +5. raw coordinates only after completing this gate. + +An element-seeking swipe follows the same rule: if its purpose is to reveal a target, replace it with `scroll-to`. Keep a coordinate swipe only when the gesture itself is the intended UI action—for example, testing or invoking a real swipe-to-dismiss interaction—and no selector-based directive expresses it. + +### Run the gate on the warning, not at audit time + +Work this gate the moment `flow-add-step` warns that it kept a raw point, while the screen that produced it is still on the device: + +1. On iOS, make an evidenced full-tree probe before keeping the point: query each plausible id/label with `native-find-views`; when there is no useful query term, call `native-full-hierarchy` with narrow `fields` and `maxDepth: 100`. Record the relevant match or no-match result with the exception evidence. `describe` and the leaf-only `native-describe-screen` are accessibility projections and are never sufficient evidence that no flow selector exists. A recorder warning only proves automatic derivation failed; it does not rule out a sibling or child label that can safely receive the tap. +2. On other platforms, inspect the deepest available app tree: `debugger-component-tree` for React Native, otherwise `describe`. On Android no tool exposes the runner's tree at all, so step 3 is the only way to confirm a candidate there. Prefer an id on iOS and Android even when trimmed discovery omits it; on Chromium the runner's tree is a [subset of `describe`](flow-yaml.md#the-runner-tree-is-not-the-discovery-tree), so an element absent there has no selector at all. +3. Verify candidates in a scratch fragment containing `assert: { visible: }`, executed on the valid target screen. If one passes, replace the point. If it fails, inspect the exact reason and try a better id, label, target app, or container; do not assume a visible miss is depth truncation. + +A tree-unavailable error makes the candidate run **void** — on iOS an error containing `could not target a native-devtools-connected app` or `native devtools is unavailable`, on Android a failure to reach the devtools helper, on Chromium an unreachable CDP session, on Vega a missing page source. It proves the tree was absent, not that the selector failed, and never authorizes coordinates — and it is the same reason the recorder quotes back in its `selector capture failed` warning, so read that warning before treating it as a verdict about the element. + +Coordinates may remain only when the target is genuinely unlabeled (no id/text/label in available discovery) or all plausible labeled candidates failed against a working flow tree. Precede the kept point with an echo naming the target, follow it with an `await:` or `assert:` that proves the result, and report the evidence. Anything the gate did not clear gets re-recorded against a selector, not annotated. A QA flow may keep such a step only for a genuinely unlabeled target, and every kept coordinate must appear in the report with its evidence; any other kept coordinate is a blocking defect. + +## iOS selector recovery + +The full iOS flow tree exists only for an app Argent launched with instrumentation. + +1. If the app came from Metro/Expo, Xcode, its icon, or an earlier uninstrumented launch, call `restart-app`, restore the screen, and retry. `launch-app` does not terminate the app first: when the app is already running, the launch only foregrounds that existing, uninstrumented process. Only `restart-app` (terminate + relaunch) guarantees an instrumented launch. +2. Tap capture already waits up to the normal post-launch connection budget. A repeated missing-tree warning after a fresh restart is not a transient capture race. +3. Call `native-devtools-status` with the same explicit simulator UDID and bundle id. If `requiresRestart` is true, restart once and check again. +4. If the app is injectable but still disconnected, call `stop-all-simulator-servers` once, then `restart-app` and `native-devtools-status` again. This recreates the current Argent transport and devtools services; it does not change app/account data. +5. If it remains disconnected, report an Argent server/instrumentation environment blocker. Do not call the app non-injectable or replace selector actions with coordinates in a QA flow. + +More than one booted simulator is not itself an injection fault: native services are keyed by UDID. Use the same explicit UDID throughout; when standalone flow device selection reports ambiguity, pass `--device `. + +Android, Chromium, and Vega never inject anything, so none of this applies to them — their tree sources fail differently, see [Tree source recovery on Android, Chromium, and Vega](#tree-source-recovery-on-android-chromium-and-vega). + +### Terminally non-injectable iOS apps + +**Scope gate: this section applies only to `com.apple.*` system apps. A connection failure in any other app never authorizes this fallback.** + +Apple system apps (`com.apple.*`) cannot load the instrumentation. Their `launch:` step skips the impossible devtools-readiness gate, but selector directives still cannot resolve. A generic flow may use the injection-free form: + +- leading `launch:` for the system app; +- raw `tool: await-ui-element` checks against the accessibility tree; +- point `tap:` / `long-press:` actions derived from `describe`, each named by an echo; +- point focus tap plus raw `tool: keyboard` with `delayMs: 500`; +- raw `gesture-swipe` calls with `settle: true` (momentum-free, so the scroll lands where the finger lifts and the following coordinate taps stay valid) because `scroll-to` needs the missing flow tree. + +Disclose that the whole flow is injection-free. Do not pretend its coordinates are portable. + +An injection-free flow is a valid generic flow but never a QA-contract-satisfying one: report it as an injection-free artifact plus the platform blocker, not as a completed QA test. Disclose it in the final report. + +If a normally injectable app is broken in the environment, replacing `launch:` with raw `tool: restart-app` can make a self-resetting **fragment**, but the runner no longer classifies it as e2e because its first non-echo step is not `launch:`. It therefore cannot satisfy the normal `argent-qa-flows` e2e contract. Report the blocker rather than labeling that fallback a completed QA test. + +## Tree source recovery on Android, Chromium, and Vega + +No injection is involved, but each platform has one source the runner cannot do without. While it is down, selector directives fail and every recorded tap keeps its raw point — a void run, never a coordinates case. Restore the source, re-record the affected taps, and delete the points that outage produced. + +| Platform | Symptom | Cause and fix | +| -------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Android | `launch:` reports it could not reach the Android devtools helper | The helper could not be installed or started — unlock the device, allow `adb install -t`, re-run | +| Chromium | a step reports no reachable CDP session | The Electron target died or was started without remote debugging — re-boot it with `boot-device`/`electronAppPath` | +| Vega | `launch:` reports the toolkit served no page source | The toolkit attaches at app launch — re-run to relaunch; the app must be built with automation support | + +**On Android, a healthy `describe` is not evidence that the flow tree is available.** `describe` falls back to a legacy `uiautomator dump` when the helper is missing; the flow tree refuses that fallback, because a trimmed tree silently changes what selectors match instead of failing. Discovery therefore looks perfect while every flow selector fails. + +## Strong transition gates + +Every navigation carries identity then readiness ([why](flow-yaml.md#prove-a-navigation-identity-then-readiness)). **Never prove a screen** with a shared header, a persistent tab bar, a source element, a positional id (`…-selector-1` encodes how many siblings exist), or a data-derived value (a counter, a count, a username, a timestamp). + +### Leave a screen by a fixed destination, not by popping the stack + +A back button, a swipe-back, and a post-save `goBack()` all pop **one** stack entry, so where they land depends on how many entries the run happened to push. A flow that reached the same screen twice has a different stack depth than the walkthrough did, and the identity gate after the back tap then fails on the wrong destination. + +Prefer an action whose destination is fixed regardless of history: tapping the already-active bottom-tab pops to that tab's root, and a Home/Close affordance goes to a known screen. + +Reach for back only when the back navigation _is_ the behavior under test. Gate it on identity like any other navigation, and expect the destination to depend on the path taken to get there. + +## Obscured targets and persistent overlays + +A selector tap resolves an element and dispatches at its coordinates; it does not prove that element is the topmost hit-test target. A toast, snackbar, banner, sheet, or other overlay can absorb the touch while the step reports success, producing a silent misfire that surfaces later. + +- After a mutating action raises an overlay that intersects the next target, first establish its selector **while it is visible**, then dismiss it and record `await: { hidden: }` before touching that region — the trio in order, so the absence check can actually fail. Do not rely on an auto-dismiss timer; automation or backgrounded rendering can pause it. +- Prefer an app-provided e2e build affordance that disables or shortens transient overlays. Otherwise record the real dismissal interaction. +- On iOS, use the `native-user-interactable-view-at-point` tool for live diagnosis of which view would receive a candidate touch. Android and Chromium have no hit-test tool, so there the recorded `visible` → dismiss → `hidden` trio is the only proof the overlay is gone. +- Keep a dismissal swipe only when the UI really supports swipe-to-dismiss. Treat it as the intended semantic action, not as element-seeking movement; put it through the coordinate fallback gate and hard-check that the overlay became hidden. + +## Diagnose a replay failure + +Classify the result before editing: + +| Outcome | Evidence | Next step | +| -------------------- | ------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | +| Hard error | A step reports error/fail and later steps skip | Inspect that step and actual state | +| Environment, not app | The reason says the check could not run — an unreadable tree, unconfirmed focus with no tree to read it from | Fix the environment and rerun; it is not a verdict about the app and never counts toward or against a pass streak | +| Silent misfire | Run reports success but expected final state is wrong | Restore the screen where the state should have changed and record the missing gate live; do not add it in YAML | +| Partial divergence | An intermediate screenshot/tree disagrees with its echo | Find the first divergent transition | +| Acceptance failure | Navigation/actions passed but a requested check fails | Preserve the check; investigate app/data behavior | + +Then: + +1. Note the first failure/divergence index and message. +2. Call `screenshot` and `describe`; when deeper evidence is needed, call `native-find-views` / `native-describe-screen` (iOS only) or `debugger-component-tree` (React Native). +3. Compare actual state with the preceding echo and expected destination. +4. Classify the cause: wrong selector, wrong screen, missing element, timing/readiness, stale prerequisite/data, optional interstitial, or real product behavior. +5. State the diagnosis in one sentence before correcting anything. + +## Correct the smallest justified unit + +- **Parameter/selector error in one step:** edit the YAML, preferring a stable selector over a new coordinate. +- **Timing/readiness failure:** repair the transition gate, then audit every step of the same shape in the flow—especially taps after a launch, screen push, drawer/sheet open, or mutating action. A fixed delay at only the observed failure leaves the same race at the other sites. +- **Identity failure (a silent misfire, or arrival on the wrong screen):** run the same same-shape audit. Every navigation gated the same weak way has the same defect, whether or not it has surfaced yet. +- **One new/missing transition or two to three structural steps:** reset to entry state and re-record the working prefix and changed portion live. +- **Four or more broken steps, unclear state, or comparison/profiling flow:** fully re-record so every action is exercised consistently. + + Either re-record restarts under the same `name` + `project_root`, and `flow-start-recording` truncates that `.yaml` before you can read it — copy the working prefix out first. + +- **Transient manual recovery:** useful for diagnosis only; it does not fix the flow and cannot count as a replay pass. + +### A replacement gate must be strictly stronger + +A gate that let a misfire through is not repaired by a longer timeout — that keeps the same unfalsifiable check and only waits longer for it. The replacement must add the missing leg: + +| The gate that failed | Not this | This | +| -------------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Shared header / positional id | Longer `timeout` | A destination-only root or control id | +| Next tap lost to a moving screen | `wait: 2000` | Add `await: { idle: true }` after the identity gate | +| Tap absorbed by a toast | Retry the tap | `await: { hidden: }` before the tap | +| `hidden` that never matched | Longer `timeout` | Record the `visible` check first, then the `hidden` one | +| Typed value wrong or truncated | Retype it | Assert the committed value after `type:`; a bare retype hides whether the field was covered, unfocused, or not an input | + +State which leg you added before rerunning. + +### The correction budget is a stop, not a guideline + +After any correction, rerun the entire polished flow from its declared start. Count every correction cycle. **After two unsuccessful cycles, stop editing** and report the remaining failure with the recommended human decision. A flow that fails at a different index each run while its step count grows is accumulating gates around an unproven path: re-record the affected span live instead of patching. + +Never weaken, remove, hide behind `when:`, or replace a requested check merely to make the run green. If the product behavior is wrong, retain the strong test and report it as an unproven regression artifact; the invoking skill decides whether that satisfies its task. QA does not call it complete until its two-pass gate succeeds. diff --git a/packages/skills/skills/argent-device-interact/SKILL.md b/packages/skills/skills/argent-device-interact/SKILL.md index 6e9157195..e65c3dd5f 100644 --- a/packages/skills/skills/argent-device-interact/SKILL.md +++ b/packages/skills/skills/argent-device-interact/SKILL.md @@ -56,25 +56,26 @@ Common schemes: `messages://`, `settings://`, `maps://?q=`, `tel://