Skip to content

Commit c7ae2f5

Browse files
Merge pull request #202 from CodeForPhilly/feat/incremental-reload
feat(api): hot reload applies only the records that changed
2 parents 76633f9 + 044fe1f commit c7ae2f5

7 files changed

Lines changed: 563 additions & 12 deletions

File tree

‎apps/api/src/routes/internal.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import { promisify } from 'node:util';
3434
import type { FastifyInstance } from 'fastify';
3535

3636
import { errorResponse, ok } from '../lib/response.js';
37-
import { reloadInMemoryStateAndFts } from '../store/memory/reload.js';
37+
import { reloadInMemoryStateAndFts, type ReloadMode } from '../store/memory/reload.js';
3838
import type { ReconcileOutcome, ReconcileResult } from '../store/reconcile.js';
3939

4040
const exec = promisify(execFile);
@@ -197,9 +197,12 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {
197197

198198
// ---- Reconcile + rebuild under the repository write lock ----
199199
let result: ReconcileResult;
200+
let reloadMode: ReloadMode | undefined;
200201
try {
201202
result = await fastify.reconcileDataRepo({
202-
onMoved: () => reloadInMemoryStateAndFts(fastify),
203+
onMoved: async (moved) => {
204+
reloadMode = await reloadInMemoryStateAndFts(fastify, { fromCommit: moved.oldCommit });
205+
},
203206
});
204207
} catch (err) {
205208
// A throw from the rebuild leaves the in-memory state + FTS index
@@ -242,6 +245,7 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {
242245
oldCommit: result.oldCommit,
243246
newCommit: result.newCommit,
244247
conflictBranch: result.conflictBranch,
248+
reloadMode,
245249
durationMs,
246250
},
247251
'hot-reload: in-memory state + FTS rebuilt',
@@ -251,6 +255,7 @@ export async function internalRoutes(fastify: FastifyInstance): Promise<void> {
251255
ok({
252256
noChanges: false,
253257
rebuilt: true,
258+
reloadMode,
254259
outcome: result.outcome,
255260
oldCommit: result.oldCommit,
256261
newCommit: result.newCommit,
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
/**
2+
* Incremental hot reload — specs/behaviors/storage.md → "Hot reload".
3+
*
4+
* Instead of re-reading every sheet, diff each in-memory sheet between the
5+
* commit the live state reflects and the new HEAD (gitsheets
6+
* `Sheet.diffFrom`), then unindex each changed record's old version and
7+
* index its new one. `unindex*` is the exact inverse of `index*`, so the
8+
* result equals a full load of the new tree.
9+
*/
10+
import { RECORD_PATH_KEY, RECORD_SHEET_KEY, type DiffChange } from 'gitsheets';
11+
12+
import type { FtsEngine } from '../fts.js';
13+
import type { PublicStore } from '../public.js';
14+
import {
15+
indexBlogPost,
16+
indexHelpWantedInterest,
17+
indexHelpWantedRole,
18+
indexMembership,
19+
indexPerson,
20+
indexPersonEvaluation,
21+
indexProject,
22+
indexProjectBuzz,
23+
indexProjectUpdate,
24+
indexSlugHistory,
25+
indexTag,
26+
indexTagAssignment,
27+
unindexBlogPost,
28+
unindexHelpWantedInterest,
29+
unindexHelpWantedRole,
30+
unindexMembership,
31+
unindexPerson,
32+
unindexPersonEvaluation,
33+
unindexProject,
34+
unindexProjectBuzz,
35+
unindexProjectUpdate,
36+
unindexSlugHistory,
37+
unindexTag,
38+
unindexTagAssignment,
39+
type InMemoryState,
40+
} from './state.js';
41+
42+
/** More changed records than this and a full rebuild is cheaper and safer. */
43+
export const INCREMENTAL_RECORD_LIMIT = 5_000;
44+
45+
type Sheets = PublicStore;
46+
type RecordOf<K extends keyof Sheets> = Sheets[K] extends { queryAll(): Promise<(infer T)[]> }
47+
? T
48+
: never;
49+
50+
interface SheetHandler<K extends keyof Sheets> {
51+
readonly sheet: K;
52+
readonly unindex: (state: InMemoryState, record: RecordOf<K>) => void;
53+
readonly index: (state: InMemoryState, record: RecordOf<K>) => void;
54+
readonly ftsRemove?: (fts: FtsEngine, record: RecordOf<K>) => void;
55+
readonly ftsAdd?: (fts: FtsEngine, record: RecordOf<K>) => void;
56+
}
57+
58+
function handler<K extends keyof Sheets>(h: SheetHandler<K>): SheetHandler<K> {
59+
return h;
60+
}
61+
62+
/** Every sheet the loader reads into InMemoryState, in loader order. */
63+
const HANDLERS = [
64+
handler({
65+
sheet: 'projects',
66+
unindex: unindexProject,
67+
index: indexProject,
68+
ftsRemove: (fts, p) => fts.removeProject(p.slug),
69+
ftsAdd: (fts, p) => {
70+
if (!p.deletedAt) fts.upsertProject(p.slug, p.title, p.summary ?? '', p.overview ?? '');
71+
},
72+
}),
73+
handler({
74+
sheet: 'people',
75+
unindex: unindexPerson,
76+
index: indexPerson,
77+
ftsRemove: (fts, p) => fts.removePerson(p.slug),
78+
ftsAdd: (fts, p) => {
79+
if (!p.deletedAt) fts.upsertPerson(p.slug, p.fullName, p.bio ?? '');
80+
},
81+
}),
82+
handler({ sheet: 'tags', unindex: unindexTag, index: indexTag }),
83+
handler({ sheet: 'tag-assignments', unindex: unindexTagAssignment, index: indexTagAssignment }),
84+
handler({ sheet: 'project-memberships', unindex: unindexMembership, index: indexMembership }),
85+
handler({ sheet: 'project-updates', unindex: unindexProjectUpdate, index: indexProjectUpdate }),
86+
handler({ sheet: 'project-buzz', unindex: unindexProjectBuzz, index: indexProjectBuzz }),
87+
handler({ sheet: 'blog-posts', unindex: unindexBlogPost, index: indexBlogPost }),
88+
handler({
89+
sheet: 'help-wanted-roles',
90+
unindex: unindexHelpWantedRole,
91+
index: indexHelpWantedRole,
92+
ftsRemove: (fts, r) => fts.removeHelpWanted(r.id),
93+
ftsAdd: (fts, r) => fts.upsertHelpWanted(r.id, r.title, r.description),
94+
}),
95+
handler({
96+
sheet: 'help-wanted-interest',
97+
unindex: unindexHelpWantedInterest,
98+
index: indexHelpWantedInterest,
99+
}),
100+
handler({
101+
sheet: 'slug-history',
102+
unindex: unindexSlugHistory,
103+
index: (state, r) => indexSlugHistory(state, r),
104+
}),
105+
handler({
106+
sheet: 'person-evaluations',
107+
unindex: unindexPersonEvaluation,
108+
index: indexPersonEvaluation,
109+
}),
110+
] as const;
111+
112+
interface CollectedChanges {
113+
readonly handler: SheetHandler<keyof Sheets>;
114+
readonly changes: readonly DiffChange[];
115+
}
116+
117+
/**
118+
* Collect every changed record between `fromCommit` and each sheet's current
119+
* read snapshot. Returns null when more than {@link INCREMENTAL_RECORD_LIMIT}
120+
* records changed, meaning the caller should do a full rebuild.
121+
*/
122+
export async function collectChanges(
123+
store: PublicStore,
124+
fromCommit: string,
125+
): Promise<CollectedChanges[] | null> {
126+
const collected: CollectedChanges[] = [];
127+
let total = 0;
128+
for (const h of HANDLERS) {
129+
const changes: DiffChange[] = [];
130+
for await (const change of store[h.sheet].diffFrom(fromCommit, { records: true })) {
131+
changes.push(change);
132+
if (++total > INCREMENTAL_RECORD_LIMIT) return null;
133+
}
134+
if (changes.length > 0) {
135+
collected.push({ handler: h as unknown as SheetHandler<keyof Sheets>, changes });
136+
}
137+
}
138+
return collected;
139+
}
140+
141+
/**
142+
* Apply collected changes to the live state and FTS index. Synchronous, so
143+
* no request handler observes a half-applied diff.
144+
*/
145+
export function applyChanges(
146+
state: InMemoryState,
147+
fts: FtsEngine,
148+
collected: readonly CollectedChanges[],
149+
): number {
150+
let applied = 0;
151+
for (const { handler: h, changes } of collected) {
152+
for (const change of changes) {
153+
const src = change.src as RecordOf<keyof Sheets> | undefined;
154+
const dst = change.dst as RecordOf<keyof Sheets> | undefined;
155+
if (src) {
156+
h.unindex(state, src);
157+
h.ftsRemove?.(fts, src);
158+
}
159+
if (dst) {
160+
// Annotate like `queryAll()` does: gitsheets locates a record's
161+
// current path through these on a later upsert/delete.
162+
Object.assign(dst, { [RECORD_SHEET_KEY]: h.sheet, [RECORD_PATH_KEY]: change.path });
163+
h.index(state, dst);
164+
h.ftsAdd?.(fts, dst);
165+
}
166+
applied++;
167+
}
168+
}
169+
return applied;
170+
}

‎apps/api/src/store/memory/reload.ts‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,36 @@
2020
* Used by `apps/api/src/routes/internal.ts` (the `POST
2121
* /api/_internal/reload-data` webhook).
2222
*/
23+
import { execFile } from 'node:child_process';
24+
import { promisify } from 'node:util';
25+
2326
import type { FastifyInstance } from 'fastify';
2427
import { invalidateFacets } from './facets.js';
28+
import { applyChanges, collectChanges } from './incremental.js';
2529
import { loadInMemoryState } from './loader.js';
2630
import type { InMemoryState } from './state.js';
2731

32+
const exec = promisify(execFile);
33+
34+
export interface ReloadOptions {
35+
/**
36+
* The commit the live state currently reflects. When given, the reload
37+
* applies only the records that changed since, unless a sheet config
38+
* changed or the diff is too large.
39+
*/
40+
readonly fromCommit?: string;
41+
}
42+
43+
export type ReloadMode = 'incremental' | 'full';
44+
45+
/** Whether `.gitsheets/` (the sheet configs) differs between two commits. */
46+
async function sheetConfigsChanged(repoPath: string, from: string, to: string): Promise<boolean> {
47+
const tree = async (commit: string): Promise<string> =>
48+
(await exec('git', ['rev-parse', `${commit}:.gitsheets`], { cwd: repoPath })).stdout.trim();
49+
const [a, b] = await Promise.all([tree(from), tree(to)]);
50+
return a !== b;
51+
}
52+
2853
/**
2954
* Replace the contents of `target`'s Maps with `source`'s Maps. The
3055
* `target` object identity is preserved so services that captured a
@@ -42,16 +67,34 @@ function replaceMapContents<K, V>(target: Map<K, V>, source: Map<K, V>): void {
4267
}
4368

4469
/**
45-
* Build a fresh `InMemoryState` from the public store and swap it into
46-
* the running Fastify instance, then reload the revocation set and the FTS
47-
* index from the same snapshot and invalidate the facet cache.
70+
* Bring the live in-memory state, revocation set and FTS index up to the
71+
* public store's current snapshot, and invalidate the facet cache.
72+
*
73+
* With `fromCommit`, applies only the records that changed since (see
74+
* `incremental.ts`); otherwise, or when configs changed or the diff is
75+
* large, rebuilds everything. Returns which path ran.
4876
*
4977
* The caller must already have refreshed the public store to the new HEAD
5078
* (`fastify.reconcileDataRepo` does, under the repository write lock).
5179
*/
5280
export async function reloadInMemoryStateAndFts(
5381
fastify: FastifyInstance,
54-
): Promise<void> {
82+
opts: ReloadOptions = {},
83+
): Promise<ReloadMode> {
84+
const { fromCommit } = opts;
85+
if (fromCommit) {
86+
const repoPath = fastify.config.CFP_DATA_REPO_PATH;
87+
if (!(await sheetConfigsChanged(repoPath, fromCommit, 'HEAD'))) {
88+
const changes = await collectChanges(fastify.store.public, fromCommit);
89+
if (changes) {
90+
applyChanges(fastify.inMemoryState, fastify.fts, changes);
91+
await fastify.revocations.loadFromSheet(fastify.store.public);
92+
invalidateFacets();
93+
return 'incremental';
94+
}
95+
}
96+
}
97+
5598
// Step 1: build the fresh in-memory state. If this throws (validator
5699
// failure on a new record, etc.), the live state is untouched and the
57100
// caller surfaces the error.
@@ -74,6 +117,7 @@ export async function reloadInMemoryStateAndFts(
74117

75118
// Step 5: drop the cached facet roll-ups so the next request recomputes.
76119
invalidateFacets();
120+
return 'full';
77121
}
78122

79123
/**

‎apps/api/src/store/memory/state.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,3 +324,100 @@ export function indexPersonEvaluation(state: InMemoryState, record: PersonEvalua
324324
}
325325
set.add(key);
326326
}
327+
328+
// ---------------------------------------------------------------------------
329+
// Unindexing — the exact inverse of each index* above, so applying a commit
330+
// diff (unindex the old version, index the new one) leaves the state equal to
331+
// a full load of the new tree. Empty secondary Sets are deleted, since a full
332+
// load never creates them. Used by the incremental hot reload.
333+
// ---------------------------------------------------------------------------
334+
335+
function removeFromSet<K>(map: Map<K, Set<string>>, key: K, id: string): void {
336+
const set = map.get(key);
337+
if (!set) return;
338+
set.delete(id);
339+
if (set.size === 0) map.delete(key);
340+
}
341+
342+
/** Delete `key` from `map` only if it still points at `id`. */
343+
function deleteIfOwned<K>(map: Map<K, string>, key: K, id: string): void {
344+
if (map.get(key) === id) map.delete(key);
345+
}
346+
347+
export function unindexSlugHistory(state: InMemoryState, record: SlugHistory): void {
348+
const key = slugHistoryKey(record.entityType, record.oldSlug);
349+
if (state.slugHistory.get(key)?.newSlug === record.newSlug) state.slugHistory.delete(key);
350+
}
351+
352+
export function unindexProject(state: InMemoryState, project: Project): void {
353+
state.projects.delete(project.id);
354+
state.projectSlugById.delete(project.id);
355+
deleteIfOwned(state.projectIdBySlug, project.slug, project.id);
356+
if (typeof project.legacyId === 'number') {
357+
deleteIfOwned(state.projectIdByLegacyId, project.legacyId, project.id);
358+
}
359+
}
360+
361+
export function unindexPerson(state: InMemoryState, person: Person): void {
362+
state.people.delete(person.id);
363+
state.personSlugById.delete(person.id);
364+
deleteIfOwned(state.personIdBySlug, person.slug, person.id);
365+
}
366+
367+
export function unindexTag(state: InMemoryState, tag: Tag): void {
368+
state.tags.delete(tag.id);
369+
deleteIfOwned(state.tagIdByHandle, `${tag.namespace}.${tag.slug}`, tag.id);
370+
}
371+
372+
export function unindexTagAssignment(state: InMemoryState, ta: TagAssignment): void {
373+
state.tagAssignments.delete(ta.id);
374+
removeFromSet(state.tagAssignmentsByTaggable, ta.taggableId, ta.id);
375+
removeFromSet(state.tagAssignmentsByTag, ta.tagId, ta.id);
376+
}
377+
378+
export function unindexMembership(state: InMemoryState, m: ProjectMembership): void {
379+
state.projectMemberships.delete(m.id);
380+
removeFromSet(state.membershipsByProject, m.projectId, m.id);
381+
removeFromSet(state.membershipsByPerson, m.personId, m.id);
382+
}
383+
384+
export function unindexProjectUpdate(state: InMemoryState, update: ProjectUpdate): void {
385+
state.projectUpdates.delete(update.id);
386+
removeFromSet(state.updatesByProject, update.projectId, update.id);
387+
deleteIfOwned(state.updateByProjectAndNumber, `${update.projectId}:${update.number}`, update.id);
388+
}
389+
390+
export function unindexProjectBuzz(state: InMemoryState, buzz: ProjectBuzz): void {
391+
state.projectBuzz.delete(buzz.id);
392+
removeFromSet(state.buzzByProject, buzz.projectId, buzz.id);
393+
deleteIfOwned(state.buzzByProjectAndSlug, `${buzz.projectId}:${buzz.slug}`, buzz.id);
394+
deleteIfOwned(state.buzzIdBySlug, buzz.slug, buzz.id);
395+
}
396+
397+
export function unindexBlogPost(state: InMemoryState, post: BlogPost): void {
398+
state.blogPosts.delete(post.id);
399+
deleteIfOwned(state.blogPostIdBySlug, post.slug, post.id);
400+
if (typeof post.legacyId === 'number') {
401+
deleteIfOwned(state.blogPostIdByLegacyId, post.legacyId, post.id);
402+
}
403+
}
404+
405+
export function unindexHelpWantedRole(state: InMemoryState, role: HelpWantedRole): void {
406+
state.helpWantedRoles.delete(role.id);
407+
removeFromSet(state.helpWantedByProject, role.projectId, role.id);
408+
}
409+
410+
export function unindexHelpWantedInterest(
411+
state: InMemoryState,
412+
expr: HelpWantedInterestExpression,
413+
): void {
414+
state.helpWantedInterest.delete(expr.id);
415+
removeFromSet(state.interestByRole, expr.roleId, expr.id);
416+
deleteIfOwned(state.interestByRoleAndPerson, `${expr.roleId}:${expr.personId}`, expr.id);
417+
}
418+
419+
export function unindexPersonEvaluation(state: InMemoryState, record: PersonEvaluation): void {
420+
const key = `${record.personSlug}/${record.evaluator}`;
421+
state.personEvaluations.delete(key);
422+
removeFromSet(state.evaluationsByPerson, record.personSlug, key);
423+
}

0 commit comments

Comments
 (0)