From 34f07a2e1143e41439c7803e0cff1278ef86aac9 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 06:35:07 +0000 Subject: [PATCH] fix: find the athlete again after re-detection instead of asking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replacing a video's tracks assigns new ids, so the one piece of work only a human can do was thrown away every run. Three re-identifications in one evening, the last after a ten-minute detection pass, for a project with one athlete in it. Ids do not survive a re-detection but positions do: the athlete was in the same place on the same frames whichever pass observed them. Bindings are snapshotted as geometry before the delete and re-attached afterwards by mean box overlap across shared frames, which is zero for anyone who was never on screen at the same time and near-zero for anyone standing elsewhere. A match built on almost no shared frames is discounted rather than trusted, because a re-bind that silently picks the wrong child is worse than being asked again — and anything below the threshold still says so plainly. Multi-track selections are preserved too: every fragment the user picked is matched independently, so an athlete stitched from six tracks comes back stitched. Co-Authored-By: Claude Opus 5 --- packages/core/src/analyze.ts | 37 ++++++++-- packages/core/src/rebind.test.ts | 75 ++++++++++++++++++++ packages/core/src/tracks.ts | 113 +++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/rebind.test.ts diff --git a/packages/core/src/analyze.ts b/packages/core/src/analyze.ts index f36cf63..2c59d8d 100644 --- a/packages/core/src/analyze.ts +++ b/packages/core/src/analyze.ts @@ -11,7 +11,7 @@ import { getFocalAthlete } from './athletes.js'; import { generateMoments } from './moments.js'; import { generateProxy, generateThumbnails } from './media.js'; import { readManifest } from './projects.js'; -import { clearTracks, createTrack } from './tracks.js'; +import { clearTracks, createTrack, rebindAthletes, snapshotAthleteBindings } from './tracks.js'; import type { Job, Preset } from './types.js'; import { findMissingSources, listVideos } from './videos.js'; @@ -189,6 +189,8 @@ export const analyzeProject = async ( const settings = settingsForPreset(preset); const warnings: string[] = []; const stagesRun: string[] = []; + /** Athlete bindings captured before a re-detection wipes their tracks. */ + let rememberedBindings: Awaited> = []; const missing = await findMissingSources(root); if (missing.length > 0) { @@ -435,6 +437,9 @@ export const analyzeProject = async ( * be broken. A project analysed six times carried six overlapping copies * of every player. */ + // Remembered before the delete, so the athlete can be found again in + // the new tracks rather than re-identified by hand every single run. + const bindings = await snapshotAthleteBindings(root, video.id); const cleared = await clearTracks(root, video.id); if (cleared.removed > 0) { await logJob( @@ -443,12 +448,7 @@ export const analyzeProject = async ( `replacing ${cleared.removed} track(s) from earlier runs of this video`, ); } - if (cleared.unboundAthletes.length > 0) { - // Track ids do not survive re-detection, so the binding cannot either. - warnings.push( - 'Re-detection replaced the tracks your athlete was bound to. Open "Identify your athlete" and pick them again.', - ); - } + rememberedBindings = bindings; for (const track of parsed.tracks ?? []) { await createTrack(root, { @@ -468,6 +468,29 @@ export const analyzeProject = async ( tracksCreated += 1; } } + /** + * Find the athlete again in the new tracks. Positions survive a + * re-detection even though ids do not, so the person standing where the + * athlete stood, on the frames they stood there, is them. + */ + if (rememberedBindings.length > 0) { + const restored = await rebindAthletes(root, videos[0]?.id ?? '', rememberedBindings); + const lost = rememberedBindings.length - restored.length; + if (restored.length > 0) { + const tracks = restored.reduce((sum, entry) => sum + entry.trackIds.length, 0); + await logJob( + root, + job.id, + `re-identified ${restored.length} athlete(s) across ${tracks} new track(s)`, + ); + } + if (lost > 0) { + warnings.push( + `${lost} athlete(s) could not be matched to the new tracks. Open "Identify your athlete" and pick them again.`, + ); + } + } + stagesRun.push('detection', 'tracking'); } diff --git a/packages/core/src/rebind.test.ts b/packages/core/src/rebind.test.ts new file mode 100644 index 0000000..82c617d --- /dev/null +++ b/packages/core/src/rebind.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; + +import { trackSimilarity } from './tracks.js'; +import type { TrackSeries } from './scoring.js'; + +/** + * Re-detection assigns new track ids, so replacing a video's tracks used to + * discard the one piece of work only a human can do — three re-identifications + * in one evening, the last after a ten-minute detection pass. + * + * Positions survive even though ids do not: the athlete was in the same place + * on the same frames whichever run observed them. These pin the matching rule, + * because a re-bind that picks the wrong child is worse than asking again. + */ + +const series = ( + id: string, + from: number, + to: number, + at: (ts: number) => { x: number; y: number }, +): TrackSeries => ({ + id, + className: 'player', + samples: Array.from({ length: Math.round((to - from) * 4) + 1 }, (_unused, i) => { + const ts = from + i / 4; + return { ts, ...at(ts), w: 40, h: 100, confidence: 0.9 }; + }), +}); + +const walking = (offset = 0) => (ts: number) => ({ x: 100 + ts * 10 + offset, y: 300 }); + +describe('finding the same athlete in a fresh set of tracks', () => { + it('matches a track to its own re-detection', () => { + const before = series('old', 0, 20, walking()); + // The same person, detected again with slightly different boxes. + const after = series('new', 0, 20, walking(3)); + expect(trackSimilarity(before, after)).toBeGreaterThan(0.7); + }); + + it('scores zero against someone who is never on screen at the same time', () => { + const before = series('old', 0, 20, walking()); + const later = series('other', 120, 140, walking()); + expect(trackSimilarity(before, later)).toBe(0); + }); + + it('scores low against someone elsewhere on the court at the same moment', () => { + const before = series('old', 0, 20, walking()); + const across = series('other', 0, 20, walking(600)); + expect(trackSimilarity(before, across)).toBe(0); + }); + + /** + * The guard against a confident wrong answer: a single frame of overlap is + * not evidence, however well the boxes happen to line up on it. + */ + it('discounts a match built on almost no shared frames', () => { + const before = series('old', 0, 20, walking()); + const glimpse = series('brief', 10, 10.25, walking()); + expect(trackSimilarity(before, glimpse)).toBeLessThan(0.3); + }); + + it('prefers the better of two overlapping candidates', () => { + const before = series('old', 0, 20, walking()); + const close = series('close', 0, 20, walking(5)); + const further = series('further', 0, 20, walking(35)); + expect(trackSimilarity(before, close)).toBeGreaterThan(trackSimilarity(before, further)); + }); + + it('handles an empty track without dividing by zero', () => { + const before = series('old', 0, 20, walking()); + const empty: TrackSeries = { id: 'empty', className: 'player', samples: [] }; + expect(trackSimilarity(before, empty)).toBe(0); + expect(trackSimilarity(empty, before)).toBe(0); + }); +}); diff --git a/packages/core/src/tracks.ts b/packages/core/src/tracks.ts index aaea734..4ad43c8 100644 --- a/packages/core/src/tracks.ts +++ b/packages/core/src/tracks.ts @@ -261,6 +261,119 @@ export const clearTracks = async (root: string, videoId: string): Promise => { + const db = await projectDb(root); + const rows = await all<{ id: string; focal_track_id: string | null }>( + db, + 'SELECT id, focal_track_id FROM athletes', + ); + const series = await loadTrackSeries(root, videoId); + const byId = new Map(series.map((track) => [track.id, track])); + + const bindings: AthleteBinding[] = []; + for (const row of rows) { + const assigned = await tracksForAthlete(root, row.id); + const ids = new Set([...assigned, ...(row.focal_track_id === null ? [] : [row.focal_track_id])]); + const kept = [...ids].map((id) => byId.get(id)).filter((t): t is TrackSeries => t !== undefined); + if (kept.length > 0) bindings.push({ athleteId: row.id, series: kept }); + } + return bindings; +}; + +/** Intersection-over-union of two boxes. */ +const boxIou = (a: TrackSample, b: TrackSample): number => { + const x1 = Math.max(a.x, b.x); + const y1 = Math.max(a.y, b.y); + const x2 = Math.min(a.x + a.w, b.x + b.w); + const y2 = Math.min(a.y + a.h, b.y + b.h); + const overlap = Math.max(0, x2 - x1) * Math.max(0, y2 - y1); + if (overlap <= 0) return 0; + return overlap / (a.w * a.h + b.w * b.h - overlap); +}; + +/** + * How much two tracks look like the same person: mean box overlap across the + * frames they share, times nothing else. Tracks that never coexist score zero, + * which is what stops a re-bind from picking a stranger who happened to stand + * where the athlete used to be an hour of footage later. + */ +export const trackSimilarity = (a: TrackSeries, b: TrackSeries): number => { + const other = new Map(b.samples.map((sample) => [Math.round(sample.ts * 4), sample])); + let total = 0; + let shared = 0; + for (const sample of a.samples) { + const match = other.get(Math.round(sample.ts * 4)); + if (match === undefined) continue; + shared += 1; + total += boxIou(sample, match); + } + return shared === 0 ? 0 : (total / shared) * Math.min(1, shared / 10); +}; + +/** Below this, a candidate is a different person and the binding is dropped. */ +const REBIND_THRESHOLD = 0.3; + +/** + * Re-attaches each athlete to whichever new tracks occupy the same space and + * time as the ones they were bound to. Returns the athletes that could be + * restored, so a caller can say plainly which ones still need a human. + */ +export const rebindAthletes = async ( + root: string, + videoId: string, + bindings: AthleteBinding[], +): Promise<{ athleteId: string; trackIds: string[] }[]> => { + if (bindings.length === 0) return []; + const fresh = await loadTrackSeries(root, videoId); + if (fresh.length === 0) return []; + + const restored: { athleteId: string; trackIds: string[] }[] = []; + for (const binding of bindings) { + const matched = new Set(); + for (const old of binding.series) { + let best: { id: string; score: number } | null = null; + for (const candidate of fresh) { + if (candidate.className !== old.className) continue; + const score = trackSimilarity(old, candidate); + if (score > (best?.score ?? 0)) best = { id: candidate.id, score }; + } + if (best !== null && best.score >= REBIND_THRESHOLD) matched.add(best.id); + } + if (matched.size === 0) continue; + + const trackIds = [...matched]; + const primary = trackIds[0]; + if (primary === undefined) continue; + await assignTracksToAthlete(root, binding.athleteId, trackIds); + const db = await projectDb(root); + await execute(db, 'UPDATE athletes SET focal_track_id = ? WHERE id = ?', [ + primary, + binding.athleteId, + ]); + restored.push({ athleteId: binding.athleteId, trackIds }); + } + return restored; +}; + /** Fuses `sourceId` into `targetId` — the annotator's "merge tracks" action. */ export const mergeTracks = async ( root: string,