From 39b42ecc13628d5cb2834394d9a96ee310d8a78d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 06:54:23 +0000 Subject: [PATCH] fix: stop a job claiming to run after the process that ran it died MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Analysis runs inside the web process, so a deploy replaces the container and takes the work with it. The row keeps saying running for ever. Observed tonight: a thorough detection pass killed at frame 8050 of 9000 by a deploy, still reporting running twenty minutes later, indistinguishable in the panel from one still going — the progress simply stops advancing, which is also what a slow pass looks like. At startup this process owns no running work by definition, so anything the database still calls running was interrupted and nothing is coming to finish it. Those rows are failed with a reason, and the reason is written to the job log where the person who started it is looking. Queued rows go the same way: nothing picks them up either. Finished work is left alone, including jobs that already failed for their own reasons — the sweep must not overwrite why something actually broke. Writing the test found the bug in the first version of this: it updated updated_at, which the jobs table does not have. That would have thrown on every boot with any interrupted job present. Co-Authored-By: Claude Opus 5 --- apps/web/src/index.ts | 28 +++++++++ packages/core/src/interrupted.test.ts | 88 +++++++++++++++++++++++++++ packages/core/src/jobs.ts | 38 ++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 packages/core/src/interrupted.test.ts diff --git a/apps/web/src/index.ts b/apps/web/src/index.ts index 6bf0590..7c33137 100644 --- a/apps/web/src/index.ts +++ b/apps/web/src/index.ts @@ -1,6 +1,7 @@ import { serve } from '@hono/node-server'; import { AuthConfigError, assertAuthConfigured, isAuthEnabled } from '@reeleel/api'; +import { failInterruptedJobs, listProjects } from '@reeleel/core'; import { clientBundleExists, createWebApp } from './server.js'; @@ -43,6 +44,33 @@ if (!clientBundleExists()) { ); } +/** + * Nothing survives a restart, so nothing should claim to. + * + * Analysis runs in this process. A deploy replaces the container mid-run and + * the job row keeps saying `running` for ever — a detection pass killed at + * frame 7350 of 9000 is indistinguishable, in the UI, from one still going. + * This process owns no running work at the moment it starts, so anything the + * database still calls running was interrupted. + */ +void (async () => { + try { + const projects = await listProjects(); + let failed = 0; + for (const project of projects) { + // A registered directory that is no longer on disk has no database to open. + if (!project.exists) continue; + failed += await failInterruptedJobs(project.root); + } + if (failed > 0) { + process.stderr.write(`marked ${failed} interrupted job(s) as failed after restart\n`); + } + } catch (error) { + // Never block startup on housekeeping. + process.stderr.write(`job recovery skipped: ${String(error)}\n`); + } +})(); + serve( { fetch: createWebApp().fetch, diff --git a/packages/core/src/interrupted.test.ts b/packages/core/src/interrupted.test.ts new file mode 100644 index 0000000..ec106c4 --- /dev/null +++ b/packages/core/src/interrupted.test.ts @@ -0,0 +1,88 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +/** + * Analysis runs inside the web process, so a deploy takes it with it. The job + * row kept saying `running` for ever, and a detection pass killed at frame 7350 + * of 9000 was indistinguishable in the UI from one still going — the progress + * simply stopped advancing, which is also what a slow pass looks like. + * + * At startup this process owns no running work by definition, so anything still + * marked running was interrupted and nothing is coming to finish it. + */ + +let home: string; + +beforeAll(() => { + home = mkdtempSync(path.join(tmpdir(), 'reeleel-interrupted-')); + process.env['REELEEL_HOME'] = home; +}); + +afterAll(async () => { + const { resetDbCache } = await import('./db.js'); + resetDbCache(); + rmSync(home, { recursive: true, force: true }); + delete process.env['REELEEL_HOME']; +}); + +const project = async (name: string): Promise => { + const { createProject } = await import('./projects.js'); + const created = await createProject({ + name, + path: path.join(home, 'projects', `${name}-${process.hrtime.bigint()}`), + }); + return created.root; +}; + +describe('jobs left behind by a restart', () => { + it('fails a running job, and says why on the job itself', async () => { + const root = await project('running'); + const { createJob, updateJob, getJob, failInterruptedJobs, listJobLogsSince } = await import( + './jobs.js' + ); + + const job = await createJob(root, 'detection', {}); + await updateJob(root, job.id, { status: 'running', stage: 'detection', progress: 0.8 }); + + expect(await failInterruptedJobs(root)).toBe(1); + const after = await getJob(root, job.id); + expect(after.status).toBe('failed'); + expect(after.error).toContain('restart'); + + // The reason has to reach the log, which is where the user is looking. + const logs = (await listJobLogsSince(root, 0)).filter((entry) => entry.jobId === job.id); + expect(logs.some((entry) => entry.message.includes('interrupted'))).toBe(true); + }); + + it('fails a queued job too, since nothing will pick it up', async () => { + const root = await project('queued'); + const { createJob, getJob, failInterruptedJobs } = await import('./jobs.js'); + const job = await createJob(root, 'render', {}); + expect(await failInterruptedJobs(root)).toBe(1); + expect((await getJob(root, job.id)).status).toBe('failed'); + }); + + it('leaves finished work alone', async () => { + const root = await project('finished'); + const { createJob, updateJob, getJob, failInterruptedJobs } = await import('./jobs.js'); + + const done = await createJob(root, 'detection', {}); + await updateJob(root, done.id, { status: 'completed', stage: 'done', progress: 1 }); + const failedJob = await createJob(root, 'detection', {}); + await updateJob(root, failedJob.id, { status: 'failed', error: 'something else' }); + + expect(await failInterruptedJobs(root)).toBe(0); + expect((await getJob(root, done.id)).status).toBe('completed'); + // The original reason must survive, not be overwritten by the sweep. + expect((await getJob(root, failedJob.id)).error).toBe('something else'); + }); + + it('is a no-op on a project with no jobs', async () => { + const root = await project('empty'); + const { failInterruptedJobs } = await import('./jobs.js'); + expect(await failInterruptedJobs(root)).toBe(0); + }); +}); diff --git a/packages/core/src/jobs.ts b/packages/core/src/jobs.ts index ade07e7..17f6d88 100644 --- a/packages/core/src/jobs.ts +++ b/packages/core/src/jobs.ts @@ -258,6 +258,44 @@ export const removeJob = async (root: string, jobId: string): Promise => { return job; }; +/** + * Fails any job left mid-flight by a restart. + * + * Analysis runs inside the web process, so a deploy — or a crash, or an OOM — + * takes the work with it and leaves the row saying `running` forever. Nothing + * was ever going to continue it, and nothing said so: the panel showed a live + * job whose progress had quietly stopped advancing, which is indistinguishable + * from a slow one. A ten-minute detection pass killed at frame 7350 of 9000 + * looked exactly like a ten-minute detection pass still going. + * + * Called on startup, when by definition this process owns no running work. + */ +export const failInterruptedJobs = async (root: string): Promise => { + const db = await projectDb(root); + const orphans = await all<{ id: string }>( + db, + "SELECT id FROM jobs WHERE status IN ('running', 'queued')", + ); + if (orphans.length === 0) return 0; + + for (const orphan of orphans) { + await logJob( + root, + orphan.id, + 'interrupted: the server restarted while this was running. Nothing continues it — start it again.', + 'error', + ); + } + // `finished_at`, not `updated_at`: this table records when work stopped. + await execute( + db, + `UPDATE jobs SET status = 'failed', error = ?, finished_at = ? + WHERE status IN ('running', 'queued')`, + ['Interrupted by a server restart.', nowIso()], + ); + return orphans.length; +}; + /** Bulk cleanup for `reeleel jobs prune`. */ export const pruneJobs = async (root: string, statuses: JobStatus[]): Promise => { if (statuses.length === 0) return 0;