A job that says running after its process died

apps/web/src/index.ts

@@ -1,6 +1,7 @@
 import { serve } from '@hono/node-server';
 
 import { AuthConfigError, assertAuthConfigured, isAuthEnabled } from '@reeleel/api';
+import { failInterruptedJobs, listProjects } from '@reeleel/core';
 
 import { clientBundleExists, createWebApp } from './server.js';
 
@@ -43,6 +44,33 @@ if (!clientBundleExists()) {
   );
 }
 
+/**
+ * Nothing survives a restart, so nothing should claim to.
+ *
+ * Analysis runs in this process. A deploy replaces the container mid-run and
+ * the job row keeps saying `running` for ever — a detection pass killed at
+ * frame 7350 of 9000 is indistinguishable, in the UI, from one still going.
+ * This process owns no running work at the moment it starts, so anything the
+ * database still calls running was interrupted.
+ */
+void (async () => {
+  try {
+    const projects = await listProjects();
+    let failed = 0;
+    for (const project of projects) {
+      // A registered directory that is no longer on disk has no database to open.
+      if (!project.exists) continue;
+      failed += await failInterruptedJobs(project.root);
+    }
+    if (failed > 0) {
+      process.stderr.write(`marked ${failed} interrupted job(s) as failed after restart\n`);
+    }
+  } catch (error) {
+    // Never block startup on housekeeping.
+    process.stderr.write(`job recovery skipped: ${String(error)}\n`);
+  }
+})();
+
 serve(
   {
     fetch: createWebApp().fetch,

packages/core/src/interrupted.test.ts

@@ -0,0 +1,88 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+/**
+ * Analysis runs inside the web process, so a deploy takes it with it. The job
+ * row kept saying `running` for ever, and a detection pass killed at frame 7350
+ * of 9000 was indistinguishable in the UI from one still going — the progress
+ * simply stopped advancing, which is also what a slow pass looks like.
+ *
+ * At startup this process owns no running work by definition, so anything still
+ * marked running was interrupted and nothing is coming to finish it.
+ */
+
+let home: string;
+
+beforeAll(() => {
+  home = mkdtempSync(path.join(tmpdir(), 'reeleel-interrupted-'));
+  process.env['REELEEL_HOME'] = home;
+});
+
+afterAll(async () => {
+  const { resetDbCache } = await import('./db.js');
+  resetDbCache();
+  rmSync(home, { recursive: true, force: true });
+  delete process.env['REELEEL_HOME'];
+});
+
+const project = async (name: string): Promise<string> => {
+  const { createProject } = await import('./projects.js');
+  const created = await createProject({
+    name,
+    path: path.join(home, 'projects', `${name}-${process.hrtime.bigint()}`),
+  });
+  return created.root;
+};
+
+describe('jobs left behind by a restart', () => {
+  it('fails a running job, and says why on the job itself', async () => {
+    const root = await project('running');
+    const { createJob, updateJob, getJob, failInterruptedJobs, listJobLogsSince } = await import(
+      './jobs.js'
+    );
+
+    const job = await createJob(root, 'detection', {});
+    await updateJob(root, job.id, { status: 'running', stage: 'detection', progress: 0.8 });
+
+    expect(await failInterruptedJobs(root)).toBe(1);
+    const after = await getJob(root, job.id);
+    expect(after.status).toBe('failed');
+    expect(after.error).toContain('restart');
+
+    // The reason has to reach the log, which is where the user is looking.
+    const logs = (await listJobLogsSince(root, 0)).filter((entry) => entry.jobId === job.id);
+    expect(logs.some((entry) => entry.message.includes('interrupted'))).toBe(true);
+  });
+
+  it('fails a queued job too, since nothing will pick it up', async () => {
+    const root = await project('queued');
+    const { createJob, getJob, failInterruptedJobs } = await import('./jobs.js');
+    const job = await createJob(root, 'render', {});
+    expect(await failInterruptedJobs(root)).toBe(1);
+    expect((await getJob(root, job.id)).status).toBe('failed');
+  });
+
+  it('leaves finished work alone', async () => {
+    const root = await project('finished');
+    const { createJob, updateJob, getJob, failInterruptedJobs } = await import('./jobs.js');
+
+    const done = await createJob(root, 'detection', {});
+    await updateJob(root, done.id, { status: 'completed', stage: 'done', progress: 1 });
+    const failedJob = await createJob(root, 'detection', {});
+    await updateJob(root, failedJob.id, { status: 'failed', error: 'something else' });
+
+    expect(await failInterruptedJobs(root)).toBe(0);
+    expect((await getJob(root, done.id)).status).toBe('completed');
+    // The original reason must survive, not be overwritten by the sweep.
+    expect((await getJob(root, failedJob.id)).error).toBe('something else');
+  });
+
+  it('is a no-op on a project with no jobs', async () => {
+    const root = await project('empty');
+    const { failInterruptedJobs } = await import('./jobs.js');
+    expect(await failInterruptedJobs(root)).toBe(0);
+  });
+});

packages/core/src/jobs.ts

@@ -258,6 +258,44 @@ export const removeJob = async (root: string, jobId: string): Promise<Job> => {
   return job;
 };
 
+/**
+ * Fails any job left mid-flight by a restart.
+ *
+ * Analysis runs inside the web process, so a deploy — or a crash, or an OOM —
+ * takes the work with it and leaves the row saying `running` forever. Nothing
+ * was ever going to continue it, and nothing said so: the panel showed a live
+ * job whose progress had quietly stopped advancing, which is indistinguishable
+ * from a slow one. A ten-minute detection pass killed at frame 7350 of 9000
+ * looked exactly like a ten-minute detection pass still going.
+ *
+ * Called on startup, when by definition this process owns no running work.
+ */
+export const failInterruptedJobs = async (root: string): Promise<number> => {
+  const db = await projectDb(root);
+  const orphans = await all<{ id: string }>(
+    db,
+    "SELECT id FROM jobs WHERE status IN ('running', 'queued')",
+  );
+  if (orphans.length === 0) return 0;
+
+  for (const orphan of orphans) {
+    await logJob(
+      root,
+      orphan.id,
+      'interrupted: the server restarted while this was running. Nothing continues it — start it again.',
+      'error',
+    );
+  }
+  // `finished_at`, not `updated_at`: this table records when work stopped.
+  await execute(
+    db,
+    `UPDATE jobs SET status = 'failed', error = ?, finished_at = ?
+      WHERE status IN ('running', 'queued')`,
+    ['Interrupted by a server restart.', nowIso()],
+  );
+  return orphans.length;
+};
+
 /** Bulk cleanup for `reeleel jobs prune`. */
 export const pruneJobs = async (root: string, statuses: JobStatus[]): Promise<number> => {
   if (statuses.length === 0) return 0;