Skip to content

Commit 16cd5ad

Browse files
zoomote2[bot]roomoteedelauna
authored
[Chore] Show the Zoo Code panel in the Solheim smoke video and bound the apt step (#2195)
* feat(evals): show the Zoo Code panel in the Solheim smoke video Add the sidebar-runner.cjs extension-tests module so the smoke recording shows the Zoo Code chat panel, hold teardown three seconds after the completion message, and bound the display dependency install at five minutes. No production extension changes. * fix(smoke): persist sidebar focus diagnostics and cover teardown holds --------- Co-authored-by: Roomote <roomote@roomote.dev> Co-authored-by: Elliott de Launay <edelauna@gmail.com>
1 parent 1d78f74 commit 16cd5ad

15 files changed

Lines changed: 302 additions & 8 deletions

‎.github/workflows/solheim-provider-smoke.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
pnpm --filter @roo-code/vscode-webview build
4242
4343
- name: Install display dependencies
44+
timeout-minutes: 5
4445
run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64
4546

4647
- name: Run Solheim provider smoke
@@ -73,6 +74,7 @@ jobs:
7374
path: |
7475
${{ runner.temp }}/solheim-provider-smoke/verdict.json
7576
${{ runner.temp }}/solheim-provider-smoke/trace.json
77+
${{ runner.temp }}/solheim-provider-smoke/sidebar-focus.json
7678
if-no-files-found: error
7779
retention-days: 7
7880

‎scripts/agent-evals/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ Execution limits are enforced, not decorative. The controller applies the
3030
activation, task start, task run, usage drain, and teardown step limits, and
3131
the trace records the measured durations.
3232

33+
Two launch options are off by default. `testsPath` adds `--extensionTestsPath`
34+
to the VS Code launch; that module's `run()` must never settle, because VS Code
35+
exits once it does. `holdBeforeTeardownMs` holds the finished task on screen
36+
for that window after the completion message, before teardown starts.
37+
3338
## Structured trace
3439

3540
Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is

‎scripts/agent-evals/isolation.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,21 @@ export function buildChildEnvironment(
1919
}
2020
}
2121

22-
export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] {
22+
export function buildLaunchArgs(
23+
workspace: string,
24+
userData: string,
25+
extensions: string,
26+
extension: string,
27+
testsPath?: string,
28+
): string[] {
2329
return [
2430
workspace,
2531
`--user-data-dir=${userData}`,
2632
`--extensions-dir=${extensions}`,
2733
`--extensionDevelopmentPath=${extension}`,
34+
// Off by default: an --extensionTestsPath module settles into a VS Code
35+
// exit, so only a caller that owns a never-settling module may set it.
36+
...(testsPath ? [`--extensionTestsPath=${testsPath}`] : []),
2837
"--no-sandbox",
2938
"--disable-gpu-sandbox",
3039
"--password-store=basic",

‎scripts/agent-evals/launcher.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ export interface HostRuntimeOptions {
1717
artifactsDir: string
1818
now?: () => number
1919
pause?: (ms: number) => Promise<void>
20+
// Optional --extensionTestsPath module. The module's run() must never
21+
// settle, because VS Code exits when it does. Off by default.
22+
testsPath?: string
23+
// Optional bounded status artifact written by the extension test module.
24+
testsDiagnosticsPath?: string
2025
}
2126

2227
const readCommit = async (root: string): Promise<MeasuredString> => {
@@ -84,13 +89,19 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime {
8489
path.join(temp, "userdata"),
8590
path.join(temp, "extensions"),
8691
extension,
92+
options.testsPath,
8793
)
8894
const child = spawn(
8995
process.env.DISPLAY ? executable : "xvfb-run",
9096
process.env.DISPLAY ? args : ["-a", executable, ...args],
9197
{
9298
detached: true,
93-
env: buildChildEnvironment(process.env, isolatedHome, socket),
99+
env: {
100+
...buildChildEnvironment(process.env, isolatedHome, socket),
101+
...(options.testsDiagnosticsPath
102+
? { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: options.testsDiagnosticsPath }
103+
: {}),
104+
},
94105
stdio: "ignore",
95106
},
96107
)

‎scripts/agent-evals/runner.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ function harness(
252252
signals,
253253
artifacts,
254254
emit,
255+
close,
255256
interrupt: () => interrupt(),
256257
get cleaned() {
257258
return cleaned
@@ -1050,4 +1051,94 @@ describe("agent fixture controller", () => {
10501051
assert.equal(result.execution.sessionLost, false)
10511052
assert.equal(result.teardown.status, "ok")
10521053
})
1054+
1055+
for (const hostExited of [false, true]) {
1056+
it(`holds teardown only while the completed host is live (exited=${hostExited})`, async () => {
1057+
const fixtureName = `controller-hold-${hostExited}`
1058+
registerTestFixture(fixtureName)
1059+
const f = harness("no-events")
1060+
f.emit({
1061+
eventName: RooCodeEventName.Message,
1062+
payload: [
1063+
{
1064+
taskId: "root",
1065+
action: "created",
1066+
message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" },
1067+
},
1068+
],
1069+
})
1070+
f.emit({
1071+
eventName: RooCodeEventName.TaskCompleted,
1072+
payload: [
1073+
"root",
1074+
{ totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 },
1075+
{},
1076+
{
1077+
isSubtask: false,
1078+
},
1079+
],
1080+
})
1081+
const holdPauses: number[] = []
1082+
let closeTaskSeenDuringHold = false
1083+
const pause = f.runtime.pause
1084+
f.runtime.pause = async (ms) => {
1085+
if (ms === 3_000) {
1086+
holdPauses.push(ms)
1087+
closeTaskSeenDuringHold = f.commands.some(
1088+
(command) => command.commandName === TaskCommandName.CloseTask,
1089+
)
1090+
}
1091+
await pause(ms)
1092+
}
1093+
const result = await runAgentFixture(f.runtime, {
1094+
fixtureName,
1095+
credential: CREDENTIAL,
1096+
observers: {
1097+
onAcceptedEvent: (event) => {
1098+
if (hostExited && event.eventName === RooCodeEventName.TaskCompleted) f.close()
1099+
},
1100+
shouldStop: () => true,
1101+
},
1102+
holdBeforeTeardownMs: 3_000,
1103+
})
1104+
assert.equal(result.execution.code, "COMPLETED")
1105+
assert.equal(result.execution.completionMessageSeen, true)
1106+
assert.deepEqual(holdPauses, hostExited ? [] : [3_000])
1107+
// The hold runs before teardown, so CloseTask is sent only after it.
1108+
assert.equal(closeTaskSeenDuringHold, false)
1109+
assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask)
1110+
assert.equal(result.teardown.status, "ok")
1111+
assert.equal(result.teardown.hostExited, true)
1112+
const trace = writtenTrace(f)
1113+
// CloseTask exits immediately, so none of the recording hold belongs to teardown.
1114+
assert.equal(trace.execution.timings.teardownMs, 0)
1115+
})
1116+
}
1117+
it("skips the teardown hold without a completion message", async () => {
1118+
registerTestFixture("controller-hold-skipped", {
1119+
limits: {
1120+
activationTimeoutMs: 120_000,
1121+
taskStartTimeoutMs: 35_000,
1122+
taskRunTimeoutMs: 200,
1123+
usageDrainMs: 100,
1124+
teardownStepMs: 100,
1125+
},
1126+
})
1127+
const f = harness("no-events")
1128+
const holdPauses: number[] = []
1129+
const pause = f.runtime.pause
1130+
f.runtime.pause = async (ms) => {
1131+
if (ms === 3_000) holdPauses.push(ms)
1132+
await pause(ms)
1133+
}
1134+
const result = await runAgentFixture(f.runtime, {
1135+
fixtureName: "controller-hold-skipped",
1136+
credential: CREDENTIAL,
1137+
holdBeforeTeardownMs: 3_000,
1138+
})
1139+
assert.equal(result.execution.code, "TASK_RUN_TIMEOUT")
1140+
assert.equal(result.execution.completionMessageSeen, false)
1141+
assert.deepEqual(holdPauses, [])
1142+
assert.equal(result.teardown.status, "ok")
1143+
})
10531144
})

‎scripts/agent-evals/runner.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ export interface AgentRunOptions {
5757
fixtureName: string
5858
credential: string
5959
observers?: AgentRunObservers
60+
// Optional hold before teardown after the completion message, so a
61+
// recording can capture the finished task. Off by default; skipped when
62+
// no completion message was seen or the host is already gone.
63+
holdBeforeTeardownMs?: number
6064
}
6165

6266
export interface AgentRunResult {
@@ -325,6 +329,11 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp
325329
// Never expose thrown provider, filesystem or host messages.
326330
} finally {
327331
waiter?.onExit()
332+
// A recording needs the finished task on screen before teardown
333+
// clears it. The hold is opt-in and only applies after the
334+
// completion message, while a live host can still show it.
335+
if (completionMessageSeen && options.holdBeforeTeardownMs && host && !host.exited)
336+
await runtime.pause(options.holdBeforeTeardownMs)
328337
const teardownStart = runtime.now()
329338
const attempt = async (step: string, operation: () => unknown) => {
330339
try {

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ describe("Solheim provider smoke workflow policy", () => {
3434
assert.equal(checkouts[0].with["persist-credentials"], false)
3535
})
3636

37+
it("bounds the display dependency install to five minutes", () => {
38+
const apt = steps.find((step) => step.name === "Install display dependencies")
39+
assert.ok(apt, "missing display dependency step")
40+
assert.equal(apt["timeout-minutes"], 5)
41+
})
42+
3743
it("exposes the provider credential only to the bounded provider step", () => {
3844
const provider = steps.find((step) => step.id === "provider")
3945
assert.ok(provider, "missing provider step")
@@ -61,7 +67,10 @@ describe("Solheim provider smoke workflow policy", () => {
6167
.split("\n")
6268
.map((line) => line.trim()),
6369
)
64-
assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]])
70+
assert.deepEqual(paths, [
71+
[output + "/verdict.json", output + "/trace.json", output + "/sidebar-focus.json"],
72+
[output + "/smoke.mp4"],
73+
])
6574
for (const step of uploads) {
6675
assert.equal(
6776
step.if,

‎scripts/solheim-smoke/README.md‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ Activation and completion each have a 120-second limit; task-start acknowledgmen
3838
in the controller. The extension awaits startup and configuration persistence without racing shared-profile
3939
writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than
4040
trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout.
41-
Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty
41+
Teardown escalates through close, terminate, and kill within independent five-second windows. After the
42+
completion message, the driver first holds the finished task on screen for three seconds, so the recording ends
43+
on the completed chat before teardown clears it. The display dependency install step in the workflow is bounded
44+
at five minutes, so a slow apt mirror fails the job instead of consuming it. A non-empty
4245
completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass.
4346
The driver never approves dialogs or tool execution; it closes the task during teardown.
4447

@@ -53,7 +56,10 @@ It does not contradict the `OK` verdict. See
5356
The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names),
5457
`trace.json` (versioned metadata-only structured trace from the shared substrate; see
5558
[`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`,
56-
a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to
59+
a 1280×720, 12 fps screen recording of the fresh CI virtual display. The recording shows the Zoo Code chat
60+
panel: `sidebar-runner.cjs` runs inside the extension host through `--extensionTestsPath`, focuses the Zoo Code
61+
sidebar, and then keeps its `run()` pending for the whole smoke, because VS Code exits once that `run()` settles.
62+
It records up to
5763
eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video
5864
contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict
5965
and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task

‎scripts/solheim-smoke/config.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,16 @@ describe("Solheim provider smoke configuration", () => {
6262
assert.ok(args.includes("--extensionDevelopmentPath=/extension"))
6363
assert.ok(args.includes("--user-data-dir=/data"))
6464
})
65+
it("adds the extension tests path only when one is supplied", () => {
66+
const plain = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension")
67+
assert.ok(!plain.some((arg) => arg.startsWith("--extensionTestsPath=")))
68+
const withTests = buildLaunchArgs(
69+
"/empty-workspace",
70+
"/data",
71+
"/extensions",
72+
"/extension",
73+
"/runner/sidebar-runner.cjs",
74+
)
75+
assert.ok(withTests.includes("--extensionTestsPath=/runner/sidebar-runner.cjs"))
76+
})
6577
})

‎scripts/solheim-smoke/config.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation
55
export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1"
66
export const SOLHEIM_MODEL = "qwen3.8-27b"
77
export const SMOKE_TIMEOUT_MS = 120_000
8+
// Keeps the finished task on screen before teardown clears it, so the
9+
// recording ends on the completed chat.
10+
export const SMOKE_TEARDOWN_HOLD_MS = 3_000
811
export const SMOKE_PROMPT =
912
"This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code."
1013

0 commit comments

Comments
 (0)