diff --git a/03-tmcra-web-console/public/downloads/tmcra-codex-1.0.0-rc.1.zip.sha256 b/03-tmcra-web-console/public/downloads/tmcra-codex-1.0.0-rc.1.zip.sha256 index 3e0ea65..abf9ce3 100644 --- a/03-tmcra-web-console/public/downloads/tmcra-codex-1.0.0-rc.1.zip.sha256 +++ b/03-tmcra-web-console/public/downloads/tmcra-codex-1.0.0-rc.1.zip.sha256 @@ -1 +1 @@ -a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148 tmcra-codex-1.0.0-rc.1.zip +23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365 tmcra-codex-1.0.0-rc.1.zip diff --git a/03-tmcra-web-console/public/downloads/tmcra-codex-latest.zip.sha256 b/03-tmcra-web-console/public/downloads/tmcra-codex-latest.zip.sha256 index a3135f0..3211abb 100644 --- a/03-tmcra-web-console/public/downloads/tmcra-codex-latest.zip.sha256 +++ b/03-tmcra-web-console/public/downloads/tmcra-codex-latest.zip.sha256 @@ -1 +1 @@ -a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148 tmcra-codex-latest.zip +23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365 tmcra-codex-latest.zip diff --git a/03-tmcra-web-console/public/downloads/tmcra-codex-release.json b/03-tmcra-web-console/public/downloads/tmcra-codex-release.json index e79f7d4..fb5b9e3 100644 --- a/03-tmcra-web-console/public/downloads/tmcra-codex-release.json +++ b/03-tmcra-web-console/public/downloads/tmcra-codex-release.json @@ -9,8 +9,8 @@ "versioned": "tmcra-codex-1.0.0-rc.1.zip", "latestSha256": "tmcra-codex-latest.zip.sha256", "versionedSha256": "tmcra-codex-1.0.0-rc.1.zip.sha256", - "bytes": 6709140, - "sha256": "a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148", + "bytes": 6709448, + "sha256": "23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365", "entryCount": 235 }, "install": { @@ -21,5 +21,5 @@ "node": ">=18", "codexPluginCli": true }, - "generatedAtUtc": "2026-09-05T19:59:30.2754219Z" + "generatedAtUtc": "2026-09-05T20:29:02.0890665Z" } diff --git a/07-tmcra-codex-plugins/tmcra-memory/docs/outbox-wakeup-fix.md b/07-tmcra-codex-plugins/tmcra-memory/docs/outbox-wakeup-fix.md new file mode 100644 index 0000000..1b151d6 --- /dev/null +++ b/07-tmcra-codex-plugins/tmcra-memory/docs/outbox-wakeup-fix.md @@ -0,0 +1,26 @@ +# Outbox wakeup handoff + +The Windows E2E timeout revealed a genuine producer/worker race. Increasing the +test timeout from 8 to 20 seconds did not fix it, so that timeout change has been +reverted. + +A worker could finish its last empty-queue check while still holding the drain +lock. A producer then enqueued a new record, saw that lock, wrote a request marker +and returned. The old worker released its lock and exited without consuming the +marker. The durable record remained local until another host event woke a worker. + +The fix pairs two checks: + +1. The producer rechecks the worker lock after writing its request. If the worker + has already released it, the producer continues the launch path. +2. The worker releases its lock before its final request check. A pending signal + causes another drain attempt; a competing active worker stops that attempt. + +Concurrent request-marker creation is now idempotent and does not truncate or +delete another producer's signal. + +`node tests/outbox_wakeup_mock.mjs` uses child-process-only filesystem gates to +exercise both exit interleavings deterministically, without changing production +timings or source files. The previous implementation reproduces `0 !== 1` on the +first race; the fix passes both interleavings and 20 concurrent signals. The test +is part of the full Codex contract suite. Only synthetic loopback memory is used. diff --git a/07-tmcra-codex-plugins/tmcra-memory/hooks/hook_common.mjs b/07-tmcra-codex-plugins/tmcra-memory/hooks/hook_common.mjs index 91e3ef6..944ff5e 100644 --- a/07-tmcra-codex-plugins/tmcra-memory/hooks/hook_common.mjs +++ b/07-tmcra-codex-plugins/tmcra-memory/hooks/hook_common.mjs @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { spawn } from "node:child_process"; -import { mkdir, open, readFile, rename, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { controlKey, memoryPolicy, mayWrite, beginMemoryTurn, taskContext, budgetEvidence, memoryDashboard, @@ -1035,18 +1035,17 @@ export async function startOutboxDrain() { }; const signalExistingDrain = async () => { await mkdir(outboxDirectory, { recursive: true }); - try { - await utimes(drainRequestPath, new Date(), new Date()); - } catch (error) { - if (error?.code !== "ENOENT") throw error; - const handle = await open(drainRequestPath, "wx", 0o600); - await handle.close(); - } + // Create-or-open is idempotent across concurrent producers. No truncation, + // and no ENOENT -> exclusive-create race can delete another producer's signal. + const handle = await open(drainRequestPath, "a", 0o600); + await handle.close(); }; try { if (await fresh(drainLockPath, DRAIN_LOCK_STALE_MS)) { await signalExistingDrain(); - return null; + // The worker may have finished after our first observation. Its final + // request check and this second lock check form the wakeup handoff. + if (await fresh(drainLockPath, DRAIN_LOCK_STALE_MS)) return null; } if (!(await claimMarker(drainLaunchPath, DRAIN_LAUNCH_STALE_MS))) return null; await signalExistingDrain(); diff --git a/07-tmcra-codex-plugins/tmcra-memory/scripts/drain_outbox.mjs b/07-tmcra-codex-plugins/tmcra-memory/scripts/drain_outbox.mjs index 072cf8f..35ae0e1 100644 --- a/07-tmcra-codex-plugins/tmcra-memory/scripts/drain_outbox.mjs +++ b/07-tmcra-codex-plugins/tmcra-memory/scripts/drain_outbox.mjs @@ -97,7 +97,7 @@ async function acquireLock() { } async function drain() { - if (!(await acquireLock())) return; + if (!(await acquireLock())) return false; await rm(launchPath, { force: true }); await rm(requestPath, { force: true }); const startedAt = Date.now(); @@ -337,4 +337,10 @@ async function drain() { } } -await drain(); +// Release the lock before the final request check. A producer that observed +// this worker either leaves a signal we consume here, or notices the released +// lock in its own post-signal check and launches a replacement. +let ownedDrain; +do { + ownedDrain = await drain(); +} while (ownedDrain !== false && await hasDrainRequest()); diff --git a/07-tmcra-codex-plugins/tmcra-memory/scripts/test_codex_e2e.mjs b/07-tmcra-codex-plugins/tmcra-memory/scripts/test_codex_e2e.mjs index 2a6b4d4..d68ce52 100644 --- a/07-tmcra-codex-plugins/tmcra-memory/scripts/test_codex_e2e.mjs +++ b/07-tmcra-codex-plugins/tmcra-memory/scripts/test_codex_e2e.mjs @@ -57,6 +57,7 @@ if (mode === "mock" || mode === "all") { const memoryControls = await run(join(pluginRoot, "tests", "memory_controls_contract.mjs")); const chatConfirmation = await run(join(pluginRoot, "tests", "chat_confirmation_contract.mjs")); const fullLocal = await run(join(pluginRoot, "tests", "full_local_contract.mjs")); + const outboxWakeup = await run(join(pluginRoot, "tests", "outbox_wakeup_mock.mjs")); const localSetup = await run(join(pluginRoot, "tests", "local_setup_contract.mjs")); const providerSetup = await run(join(pluginRoot, "tests", "provider_setup_contract.mjs"), { timeoutMs: 60_000, @@ -77,6 +78,7 @@ if (mode === "mock" || mode === "all") { memoryControls, chatConfirmation, fullLocal, + outboxWakeup, localSetup, ...coreMock, providerSetup, diff --git a/07-tmcra-codex-plugins/tmcra-memory/tests/codex_e2e_mock.mjs b/07-tmcra-codex-plugins/tmcra-memory/tests/codex_e2e_mock.mjs index 53f48b0..3bc64dd 100644 --- a/07-tmcra-codex-plugins/tmcra-memory/tests/codex_e2e_mock.mjs +++ b/07-tmcra-codex-plugins/tmcra-memory/tests/codex_e2e_mock.mjs @@ -214,9 +214,7 @@ async function test(name, callback) { results.push({ name, ok: true, elapsedMs: Date.now() - started }); } -// Detached worker startup on shared Windows CI includes scheduler/antivirus delay. -// This is an eventual-delivery contract, not a production latency assertion. -async function waitFor(predicate, { timeoutMs = process.env.CI ? 20_000 : 8_000, intervalMs = 50 } = {}) { +async function waitFor(predicate, { timeoutMs = 8_000, intervalMs = 50 } = {}) { const deadline = Date.now() + timeoutMs; while (!(await predicate())) { if (Date.now() >= deadline) throw new Error(`condition did not become true within ${timeoutMs}ms`); diff --git a/07-tmcra-codex-plugins/tmcra-memory/tests/outbox_wakeup_mock.mjs b/07-tmcra-codex-plugins/tmcra-memory/tests/outbox_wakeup_mock.mjs new file mode 100644 index 0000000..4098ef5 --- /dev/null +++ b/07-tmcra-codex-plugins/tmcra-memory/tests/outbox_wakeup_mock.mjs @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { fork } from "node:child_process"; +import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { MockTmcraServer } from "./mock_tmcra_server.mjs"; + +const plugin = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const root = await mkdtemp(join(tmpdir(), "tmcra-wakeup-race-")); +const previous = { ...process.env }; +const workers = new Set(); +const token = randomUUID(); +const server = new MockTmcraServer({ validTokens: [token] }); +const bounded = (promise, label, ms = 8000) => new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(Error(`Timed out: ${label}`)), ms); + promise.then(value => { clearTimeout(timer); resolve(value); }, error => { clearTimeout(timer); reject(error); }); +}); + +// Pause real fs operations in a child process only. Production source and +// scheduler timings remain unchanged; IPC selects the exact race window. +const preload = join(root, "gate.mjs"); +await writeFile(preload, ` +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +const op = process.env.RACE_OPERATION; +const original = fs.promises[op]; +let paused = false; +fs.promises[op] = async (...args) => { + if (!paused && String(args[0]).endsWith('.drain.lock')) { + paused = true; + const value = op === 'stat' ? await original(...args) : undefined; + process.send({ gated: true }); + await new Promise(resolve => process.once('message', resolve)); + process.disconnect(); + return op === 'stat' ? value : original(...args); + } + return original(...args); +}; +syncBuiltinESMExports(); +`); +const producer = join(root, "producer.mjs"); +await writeFile(producer, `import { startOutboxDrain } from ${JSON.stringify(pathToFileURL(join(plugin, "hooks/hook_common.mjs")).href)}; await startOutboxDrain();`); + +function launch(script, operation) { + const child = fork(script, [], { env: { ...process.env, RACE_OPERATION: operation }, + execArgv: operation ? ["--import", pathToFileURL(preload).href] : [], + silent: true, windowsHide: true }); + workers.add(child); + let errors = ""; + child.stderr.on("data", value => { errors += value; }); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", code => { workers.delete(child); code === 0 ? resolve() : reject(Error(`Worker failed (${code}): ${errors.replaceAll(token, "[REDACTED]")}`)); }); + }); + exited.catch(() => {}); + const gated = new Promise(resolve => child.once("message", resolve)); + return { child, exited, gated: bounded(gated, `${operation} gate`), release: () => child.send({ release: true }) }; +} + +try { + for (const key of Object.keys(process.env)) if (key.startsWith("TMCRA_") || key === "PLUGIN_DATA" || key === "CLAUDE_PLUGIN_DATA") delete process.env[key]; + await server.start(); + process.env.TMCRA_CONFIG_FILE = join(root, "config.json"); + process.env.TMCRA_LOCAL_BINDING_FILE = join(root, "no-local-binding.json"); + process.env.TMCRA_PROVIDER_CONFIG_FILE = join(root, "no-providers.json"); + await writeFile(process.env.TMCRA_CONFIG_FILE, JSON.stringify({ baseUrl: server.baseUrl, apiKey: token })); + const { saveOutboxTurn } = await import("../scripts/tmcra_client.mjs"); + const { startOutboxDrain } = await import("../hooks/hook_common.mjs"); + const enqueue = async name => saveOutboxTurn({ scope: "race-project", projectId: "race-project", + sessionId: name, messages: [{ message_id: name, role: "user", content: name, timestamp: new Date().toISOString() }], + metadata: { integration: "race-test" }, consistency: "eventual", slowPolicy: "auto", idempotencyKey: name }); + + // The worker has decided the queue is empty but still owns the drain lock. + // A new entry signals it during that interval; no subsequent hook may be needed. + process.env.PLUGIN_DATA = join(root, "consumer-exit"); + let before = server.records.length; + const exiting = launch(join(plugin, "scripts/drain_outbox.mjs"), "rm"); + await exiting.gated; + await enqueue("queued-during-worker-exit"); + await startOutboxDrain(); + exiting.release(); + await bounded(exiting.exited, "consumer handoff"); + assert.equal(server.records.length, before + 1, "a signal during worker exit must drain without another host event"); + assert.equal((await readdir(join(process.env.PLUGIN_DATA, "outbox"))).filter(name => name.endsWith(".json")).length, 0); + + // The producer observed the old lock, then the worker released it and finished + // its final request check before the producer actually wrote its signal. + process.env.PLUGIN_DATA = join(root, "producer-observation"); + before = server.records.length; + const oldWorker = launch(join(plugin, "scripts/drain_outbox.mjs"), "rm"); + await oldWorker.gated; + await enqueue("queued-after-stale-observation"); + const staleProducer = launch(producer, "stat"); + await staleProducer.gated; + oldWorker.release(); + await bounded(oldWorker.exited, "old worker exit"); + staleProducer.release(); + await bounded(staleProducer.exited, "producer handoff"); + const deadline = Date.now() + 8000; + while (server.records.length !== before + 1 || (await readdir(join(process.env.PLUGIN_DATA, "outbox"))).some(name => name.endsWith(".json") || name === ".drain.lock")) { + assert(Date.now() < deadline, "producer must launch a replacement after observing a released lock"); + await new Promise(resolve => setTimeout(resolve, 25)); + } + + process.env.PLUGIN_DATA = join(root, "concurrent-signals"); + const outbox = join(process.env.PLUGIN_DATA, "outbox"); + await mkdir(outbox, { recursive: true }); + await writeFile(join(outbox, ".drain.lock"), "test-owned-lock"); + await Promise.all(Array.from({ length: 20 }, () => startOutboxDrain())); + assert((await readdir(outbox)).includes(".drain.request")); + const events = await readFile(join(process.env.PLUGIN_DATA, "logs/events.jsonl"), "utf8").catch(error => { + if (error.code === "ENOENT") return ""; throw error; + }); + assert(!events.includes("outbox_drain_launch_failed"), "concurrent signals must be idempotent"); + console.log(JSON.stringify({ ok: true, consumerExitWakeup: true, staleProducerObservation: true, concurrentSignals: 20 })); +} finally { + for (const child of workers) child.kill(); + await server.stop(); + for (const key of Object.keys(process.env)) if (!(key in previous)) delete process.env[key]; + Object.assign(process.env, previous); + // Only this test's explicit mkdtemp directory is eligible for cleanup. + if (dirname(root) === resolve(tmpdir()) && root.startsWith(join(resolve(tmpdir()), "tmcra-wakeup-race-"))) + await rm(root, { recursive: true, force: true }); +}