fix: preserve Codex background delivery wakeups during worker exit

03-tmcra-web-console/public/downloads/tmcra-codex-1.0.0-rc.1.zip.sha256

@@ -1 +1 @@
-a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148  tmcra-codex-1.0.0-rc.1.zip
+23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365  tmcra-codex-1.0.0-rc.1.zip

03-tmcra-web-console/public/downloads/tmcra-codex-latest.zip.sha256

@@ -1 +1 @@
-a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148  tmcra-codex-latest.zip
+23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365  tmcra-codex-latest.zip

03-tmcra-web-console/public/downloads/tmcra-codex-release.json

@@ -9,8 +9,8 @@
     "versioned": "tmcra-codex-1.0.0-rc.1.zip",
     "latestSha256": "tmcra-codex-latest.zip.sha256",
     "versionedSha256": "tmcra-codex-1.0.0-rc.1.zip.sha256",
-    "bytes": 6709140,
-    "sha256": "a329fe3063023fba0068882b60fcfd9f7c393571c1bffd9e226e05148442c148",
+    "bytes": 6709448,
+    "sha256": "23e99f097a9169a8d80321116fd24614758fe855f5fd3b03f5a70f7ef1432365",
     "entryCount": 235
   },
   "install": {
@@ -21,5 +21,5 @@
     "node": ">=18",
     "codexPluginCli": true
   },
-  "generatedAtUtc": "2026-09-05T19:59:30.2754219Z"
+  "generatedAtUtc": "2026-09-05T20:29:02.0890665Z"
 }

07-tmcra-codex-plugins/tmcra-memory/docs/outbox-wakeup-fix.md

@@ -0,0 +1,26 @@
+# Outbox wakeup handoff
+
+The Windows E2E timeout revealed a genuine producer/worker race. Increasing the
+test timeout from 8 to 20 seconds did not fix it, so that timeout change has been
+reverted.
+
+A worker could finish its last empty-queue check while still holding the drain
+lock. A producer then enqueued a new record, saw that lock, wrote a request marker
+and returned. The old worker released its lock and exited without consuming the
+marker. The durable record remained local until another host event woke a worker.
+
+The fix pairs two checks:
+
+1. The producer rechecks the worker lock after writing its request. If the worker
+   has already released it, the producer continues the launch path.
+2. The worker releases its lock before its final request check. A pending signal
+   causes another drain attempt; a competing active worker stops that attempt.
+
+Concurrent request-marker creation is now idempotent and does not truncate or
+delete another producer's signal.
+
+`node tests/outbox_wakeup_mock.mjs` uses child-process-only filesystem gates to
+exercise both exit interleavings deterministically, without changing production
+timings or source files. The previous implementation reproduces `0 !== 1` on the
+first race; the fix passes both interleavings and 20 concurrent signals. The test
+is part of the full Codex contract suite. Only synthetic loopback memory is used.

07-tmcra-codex-plugins/tmcra-memory/hooks/hook_common.mjs

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
-import { mkdir, open, readFile, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
+import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { controlKey, memoryPolicy, mayWrite, beginMemoryTurn, taskContext, budgetEvidence, memoryDashboard,
@@ -1035,18 +1035,17 @@ export async function startOutboxDrain() {
   };
   const signalExistingDrain = async () => {
     await mkdir(outboxDirectory, { recursive: true });
-    try {
-      await utimes(drainRequestPath, new Date(), new Date());
-    } catch (error) {
-      if (error?.code !== "ENOENT") throw error;
-      const handle = await open(drainRequestPath, "wx", 0o600);
-      await handle.close();
-    }
+    // Create-or-open is idempotent across concurrent producers. No truncation,
+    // and no ENOENT -> exclusive-create race can delete another producer's signal.
+    const handle = await open(drainRequestPath, "a", 0o600);
+    await handle.close();
   };
   try {
     if (await fresh(drainLockPath, DRAIN_LOCK_STALE_MS)) {
       await signalExistingDrain();
-      return null;
+      // The worker may have finished after our first observation. Its final
+      // request check and this second lock check form the wakeup handoff.
+      if (await fresh(drainLockPath, DRAIN_LOCK_STALE_MS)) return null;
     }
     if (!(await claimMarker(drainLaunchPath, DRAIN_LAUNCH_STALE_MS))) return null;
     await signalExistingDrain();

07-tmcra-codex-plugins/tmcra-memory/scripts/drain_outbox.mjs

@@ -97,7 +97,7 @@ async function acquireLock() {
 }
 
 async function drain() {
-  if (!(await acquireLock())) return;
+  if (!(await acquireLock())) return false;
   await rm(launchPath, { force: true });
   await rm(requestPath, { force: true });
   const startedAt = Date.now();
@@ -337,4 +337,10 @@ async function drain() {
   }
 }
 
-await drain();
+// Release the lock before the final request check. A producer that observed
+// this worker either leaves a signal we consume here, or notices the released
+// lock in its own post-signal check and launches a replacement.
+let ownedDrain;
+do {
+  ownedDrain = await drain();
+} while (ownedDrain !== false && await hasDrainRequest());

07-tmcra-codex-plugins/tmcra-memory/scripts/test_codex_e2e.mjs

@@ -57,6 +57,7 @@ if (mode === "mock" || mode === "all") {
   const memoryControls = await run(join(pluginRoot, "tests", "memory_controls_contract.mjs"));
   const chatConfirmation = await run(join(pluginRoot, "tests", "chat_confirmation_contract.mjs"));
   const fullLocal = await run(join(pluginRoot, "tests", "full_local_contract.mjs"));
+  const outboxWakeup = await run(join(pluginRoot, "tests", "outbox_wakeup_mock.mjs"));
   const localSetup = await run(join(pluginRoot, "tests", "local_setup_contract.mjs"));
   const providerSetup = await run(join(pluginRoot, "tests", "provider_setup_contract.mjs"), {
     timeoutMs: 60_000,
@@ -77,6 +78,7 @@ if (mode === "mock" || mode === "all") {
     memoryControls,
     chatConfirmation,
     fullLocal,
+    outboxWakeup,
     localSetup,
     ...coreMock,
     providerSetup,

07-tmcra-codex-plugins/tmcra-memory/tests/codex_e2e_mock.mjs

@@ -214,9 +214,7 @@ async function test(name, callback) {
   results.push({ name, ok: true, elapsedMs: Date.now() - started });
 }
 
-// Detached worker startup on shared Windows CI includes scheduler/antivirus delay.
-// This is an eventual-delivery contract, not a production latency assertion.
-async function waitFor(predicate, { timeoutMs = process.env.CI ? 20_000 : 8_000, intervalMs = 50 } = {}) {
+async function waitFor(predicate, { timeoutMs = 8_000, intervalMs = 50 } = {}) {
   const deadline = Date.now() + timeoutMs;
   while (!(await predicate())) {
     if (Date.now() >= deadline) throw new Error(`condition did not become true within ${timeoutMs}ms`);

07-tmcra-codex-plugins/tmcra-memory/tests/outbox_wakeup_mock.mjs

@@ -0,0 +1,126 @@
+import assert from "node:assert/strict";
+import { fork } from "node:child_process";
+import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { MockTmcraServer } from "./mock_tmcra_server.mjs";
+
+const plugin = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const root = await mkdtemp(join(tmpdir(), "tmcra-wakeup-race-"));
+const previous = { ...process.env };
+const workers = new Set();
+const token = randomUUID();
+const server = new MockTmcraServer({ validTokens: [token] });
+const bounded = (promise, label, ms = 8000) => new Promise((resolve, reject) => {
+  const timer = setTimeout(() => reject(Error(`Timed out: ${label}`)), ms);
+  promise.then(value => { clearTimeout(timer); resolve(value); }, error => { clearTimeout(timer); reject(error); });
+});
+
+// Pause real fs operations in a child process only. Production source and
+// scheduler timings remain unchanged; IPC selects the exact race window.
+const preload = join(root, "gate.mjs");
+await writeFile(preload, `
+import fs from 'node:fs';
+import { syncBuiltinESMExports } from 'node:module';
+const op = process.env.RACE_OPERATION;
+const original = fs.promises[op];
+let paused = false;
+fs.promises[op] = async (...args) => {
+  if (!paused && String(args[0]).endsWith('.drain.lock')) {
+    paused = true;
+    const value = op === 'stat' ? await original(...args) : undefined;
+    process.send({ gated: true });
+    await new Promise(resolve => process.once('message', resolve));
+    process.disconnect();
+    return op === 'stat' ? value : original(...args);
+  }
+  return original(...args);
+};
+syncBuiltinESMExports();
+`);
+const producer = join(root, "producer.mjs");
+await writeFile(producer, `import { startOutboxDrain } from ${JSON.stringify(pathToFileURL(join(plugin, "hooks/hook_common.mjs")).href)}; await startOutboxDrain();`);
+
+function launch(script, operation) {
+  const child = fork(script, [], { env: { ...process.env, RACE_OPERATION: operation },
+    execArgv: operation ? ["--import", pathToFileURL(preload).href] : [],
+    silent: true, windowsHide: true });
+  workers.add(child);
+  let errors = "";
+  child.stderr.on("data", value => { errors += value; });
+  const exited = new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("exit", code => { workers.delete(child); code === 0 ? resolve() : reject(Error(`Worker failed (${code}): ${errors.replaceAll(token, "[REDACTED]")}`)); });
+  });
+  exited.catch(() => {});
+  const gated = new Promise(resolve => child.once("message", resolve));
+  return { child, exited, gated: bounded(gated, `${operation} gate`), release: () => child.send({ release: true }) };
+}
+
+try {
+  for (const key of Object.keys(process.env)) if (key.startsWith("TMCRA_") || key === "PLUGIN_DATA" || key === "CLAUDE_PLUGIN_DATA") delete process.env[key];
+  await server.start();
+  process.env.TMCRA_CONFIG_FILE = join(root, "config.json");
+  process.env.TMCRA_LOCAL_BINDING_FILE = join(root, "no-local-binding.json");
+  process.env.TMCRA_PROVIDER_CONFIG_FILE = join(root, "no-providers.json");
+  await writeFile(process.env.TMCRA_CONFIG_FILE, JSON.stringify({ baseUrl: server.baseUrl, apiKey: token }));
+  const { saveOutboxTurn } = await import("../scripts/tmcra_client.mjs");
+  const { startOutboxDrain } = await import("../hooks/hook_common.mjs");
+  const enqueue = async name => saveOutboxTurn({ scope: "race-project", projectId: "race-project",
+    sessionId: name, messages: [{ message_id: name, role: "user", content: name, timestamp: new Date().toISOString() }],
+    metadata: { integration: "race-test" }, consistency: "eventual", slowPolicy: "auto", idempotencyKey: name });
+
+  // The worker has decided the queue is empty but still owns the drain lock.
+  // A new entry signals it during that interval; no subsequent hook may be needed.
+  process.env.PLUGIN_DATA = join(root, "consumer-exit");
+  let before = server.records.length;
+  const exiting = launch(join(plugin, "scripts/drain_outbox.mjs"), "rm");
+  await exiting.gated;
+  await enqueue("queued-during-worker-exit");
+  await startOutboxDrain();
+  exiting.release();
+  await bounded(exiting.exited, "consumer handoff");
+  assert.equal(server.records.length, before + 1, "a signal during worker exit must drain without another host event");
+  assert.equal((await readdir(join(process.env.PLUGIN_DATA, "outbox"))).filter(name => name.endsWith(".json")).length, 0);
+
+  // The producer observed the old lock, then the worker released it and finished
+  // its final request check before the producer actually wrote its signal.
+  process.env.PLUGIN_DATA = join(root, "producer-observation");
+  before = server.records.length;
+  const oldWorker = launch(join(plugin, "scripts/drain_outbox.mjs"), "rm");
+  await oldWorker.gated;
+  await enqueue("queued-after-stale-observation");
+  const staleProducer = launch(producer, "stat");
+  await staleProducer.gated;
+  oldWorker.release();
+  await bounded(oldWorker.exited, "old worker exit");
+  staleProducer.release();
+  await bounded(staleProducer.exited, "producer handoff");
+  const deadline = Date.now() + 8000;
+  while (server.records.length !== before + 1 || (await readdir(join(process.env.PLUGIN_DATA, "outbox"))).some(name => name.endsWith(".json") || name === ".drain.lock")) {
+    assert(Date.now() < deadline, "producer must launch a replacement after observing a released lock");
+    await new Promise(resolve => setTimeout(resolve, 25));
+  }
+
+  process.env.PLUGIN_DATA = join(root, "concurrent-signals");
+  const outbox = join(process.env.PLUGIN_DATA, "outbox");
+  await mkdir(outbox, { recursive: true });
+  await writeFile(join(outbox, ".drain.lock"), "test-owned-lock");
+  await Promise.all(Array.from({ length: 20 }, () => startOutboxDrain()));
+  assert((await readdir(outbox)).includes(".drain.request"));
+  const events = await readFile(join(process.env.PLUGIN_DATA, "logs/events.jsonl"), "utf8").catch(error => {
+    if (error.code === "ENOENT") return ""; throw error;
+  });
+  assert(!events.includes("outbox_drain_launch_failed"), "concurrent signals must be idempotent");
+  console.log(JSON.stringify({ ok: true, consumerExitWakeup: true, staleProducerObservation: true, concurrentSignals: 20 }));
+} finally {
+  for (const child of workers) child.kill();
+  await server.stop();
+  for (const key of Object.keys(process.env)) if (!(key in previous)) delete process.env[key];
+  Object.assign(process.env, previous);
+  // Only this test's explicit mkdtemp directory is eligible for cleanup.
+  if (dirname(root) === resolve(tmpdir()) && root.startsWith(join(resolve(tmpdir()), "tmcra-wakeup-race-")))
+    await rm(root, { recursive: true, force: true });
+}