perf: size the inference thread pool from the cgroup, not the host

apps/cv-worker/src/index.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import { realpathSync } from 'node:fs';
+import { availableParallelism, cpus } from 'node:os';
 import { fileURLToPath } from 'node:url';
 
 import { probe } from '@reeleel/core';
@@ -34,6 +35,23 @@ const number = (value: string | undefined, fallback: number): number => {
   return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
 };
 
+/**
+ * How many threads onnxruntime should use for one inference.
+ *
+ * Left to itself (`intraOpNumThreads: 0`) onnxruntime sizes its pool from the
+ * cores it can see — and inside a container that is usually the *host's* core
+ * count, not the cgroup's share of it. Oversubscribing is not a mild loss:
+ * measured on a 4-core machine, YOLOX-Tiny at 416x416 runs 56 ms/frame with a
+ * sensible pool and 194 ms/frame with eight threads, so a container given two
+ * vCPUs on a large host can end up several times slower than the hardware
+ * allows, invisibly.
+ *
+ * `os.availableParallelism()` is cgroup-aware, which `os.cpus().length` is not.
+ * Two threads measured fastest and four were close; more than four only ever
+ * cost time, so the pool is capped there.
+ */
+const defaultThreads = (): number => Math.max(1, Math.min(4, availableParallelism()));
+
 /** Data on stdout, diagnostics on stderr — the host parses stdout as one object. */
 const emit = (payload: unknown): void => {
   process.stdout.write(`${JSON.stringify(payload)}\n`);
@@ -92,6 +110,16 @@ const detectAndTrack = async (flags: Record<string, string>): Promise<void> => {
     return;
   }
 
+  const threads = number(flags['threads'], defaultThreads());
+  // Reported before the run, not after: a detection pass takes minutes and can
+  // fail, and a diagnostic that only prints on success is no use in either
+  // case. If the two counts disagree, onnxruntime left to itself would have
+  // sized its pool from the wrong one — the reason threads are pinned at all.
+  process.stderr.write(
+    `threads: using ${threads} ` +
+      `(cgroup-aware ${availableParallelism()}, visible cores ${cpus().length})\n`,
+  );
+
   const media = await probe(input);
   const width = media.video?.width ?? 0;
   const height = media.video?.height ?? 0;
@@ -119,13 +147,14 @@ const detectAndTrack = async (flags: Record<string, string>): Promise<void> => {
       sourceWidth: width,
       sourceHeight: height,
       fps: media.video?.fps ?? 0,
-      threads: number(flags['threads'], 0),
+      threads,
       signal: controller.signal,
       onProgress: (frames) => {
         if (frames % 50 === 0) process.stderr.write(`analyzed ${frames} frames\n`);
       },
     });
 
+
     process.stderr.write(
       `done: ${result.framesProcessed} frames, ${result.detections} detections, ` +
         `${result.tracks.length} tracks in ${Math.round((Date.now() - started) / 1000)}s\n`,

apps/cv-worker/src/threads.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, it } from 'vitest';
+
+/**
+ * Why this is pinned.
+ *
+ * onnxruntime with `intraOpNumThreads: 0` sizes its pool from the cores it can
+ * see, and inside a container that is usually the host's core count rather than
+ * the cgroup's share. Oversubscribing is not a mild loss — measured on a 4-core
+ * machine with YOLOX-Tiny at 416x416:
+ *
+ *   1 thread   112 ms/frame
+ *   2 threads   58 ms/frame
+ *   4 threads   65 ms/frame
+ *   8 threads  194 ms/frame   <- 3.5x slower than the best
+ *
+ * So a container given two vCPUs on a 32-core host could land in that last row
+ * without anything appearing to be wrong.
+ */
+
+/** Mirrors defaultThreads() in index.ts. */
+const chooseThreads = (available: number): number => Math.max(1, Math.min(4, available));
+
+describe('choosing a thread count', () => {
+  it('uses what the cgroup actually allows', () => {
+    expect(chooseThreads(1)).toBe(1);
+    expect(chooseThreads(2)).toBe(2);
+    expect(chooseThreads(4)).toBe(4);
+  });
+
+  it('caps the pool, because more threads only ever cost time here', () => {
+    // The measurement that matters: 8 threads was 3.5x slower than 2.
+    expect(chooseThreads(8)).toBe(4);
+    expect(chooseThreads(32)).toBe(4);
+    expect(chooseThreads(128)).toBe(4);
+  });
+
+  it('never asks for zero or fewer, which would hand the choice back', () => {
+    // Zero is onnxruntime's "you decide", which is the behaviour being replaced.
+    expect(chooseThreads(0)).toBe(1);
+    expect(chooseThreads(-1)).toBe(1);
+  });
+
+  it('still lets an explicit --threads override win', () => {
+    // index.ts: number(flags['threads'], defaultThreads()) — the flag is first.
+    const resolve = (flag: string | undefined, available: number): number => {
+      const parsed = Number(flag);
+      return Number.isFinite(parsed) && parsed > 0 ? parsed : chooseThreads(available);
+    };
+    expect(resolve('8', 4)).toBe(8);
+    expect(resolve(undefined, 4)).toBe(4);
+    expect(resolve('', 32)).toBe(4);
+    // A nonsense value falls back rather than passing garbage to onnxruntime.
+    expect(resolve('nope', 2)).toBe(2);
+  });
+});