Merge pull request #256 from CodeForPhilly/develop

Release: v0.10.3

Author: themightychris

apps/api/src/app.ts

@@ -158,12 +158,13 @@ export async function buildApp(opts: BuildAppOptions = {}): Promise<FastifyInsta
   await fastify.register(slugRedirectPlugin);
   await fastify.register(legacyRedirectPlugin);
   await fastify.register(eventRemindersPlugin);
-  await fastify.register(spamPurgePlugin);
 
   // ----- 7. Session middleware (JWT auth) -----
   // Before rate-limit and idempotency: all three use onRequest hooks, which run
   // in registration order, and the other two key on request.session.person.
   await fastify.register(sessionMiddlewarePlugin);
+  // After the session middleware: the sweep clears purged people's session metadata.
+  await fastify.register(spamPurgePlugin);
 
   // ----- 8. Rate limiting -----
   await fastify.register(rateLimitPlugin);

apps/api/src/auth/session-metadata.ts

@@ -54,6 +54,19 @@ export class SessionMetadataStore {
     await this.flush(privateStore);
   }
 
+  /** Forget every session of a purged person. */
+  async removeAllFor(personId: string, privateStore: PrivateStore): Promise<number> {
+    let removed = 0;
+    for (const [jti, meta] of this.#map) {
+      if (meta.personId === personId) {
+        this.#map.delete(jti);
+        removed++;
+      }
+    }
+    if (removed > 0) await this.flush(privateStore);
+    return removed;
+  }
+
   getAll(personId: string): SessionMeta[] {
     const result: SessionMeta[] = [];
     for (const meta of this.#map.values()) {

apps/api/src/env.ts

@@ -74,6 +74,13 @@ export const EnvSchema = z.object({
    * image; unset in dev (Vite owns 5173).
    */
   CFP_WEB_DIST_PATH: z.string().optional(),
+  /**
+   * Whether this pod runs the daily scheduled spam purge
+   * (specs/behaviors/person-lifecycle.md#scheduled-spam-purge). Off by default:
+   * sandbox and production share `published`, so exactly one deployment —
+   * production — may run a destructive sweep against it.
+   */
+  CFP_SCHEDULED_PURGE: z.enum(['true', 'false']).default('false'),
   /**
    * Host of the public-facing site (e.g. `codeforphilly.org` in prod,
    * `next-v2.codeforphilly.org` in sandbox). Used by the server-side
@@ -147,6 +154,7 @@ export const envJsonSchema = {
     SAML_ENTITY_ID: { type: 'string', default: SAML_ENTITY_ID_DEFAULT },
     SLACK_TEAM_HOST: { type: 'string', default: 'codeforphilly.slack.com' },
     CFP_WEB_DIST_PATH: { type: 'string' },
+    CFP_SCHEDULED_PURGE: { type: 'string', enum: ['true', 'false'], default: 'false' },
     CFP_SITE_HOST: { type: 'string', default: 'codeforphilly.org' },
     POSTMARK_SERVER_TOKEN: { type: 'string' },
     POSTMARK_MESSAGE_STREAM: { type: 'string', default: 'outbound' },

apps/api/src/plugins/spam-purge.ts

@@ -8,6 +8,14 @@ import fp from 'fastify-plugin';
 
 import { SPAM_PURGE_INTERVAL_MS, SpamPurgeService } from '../services/spam-purge.js';
 
+/**
+ * Whether this pod runs the sweep: only when CFP_SCHEDULED_PURGE is "true",
+ * and never under tests (they call the sweep directly).
+ */
+export function scheduledPurgeEnabled(config: { NODE_ENV: string; CFP_SCHEDULED_PURGE: string }): boolean {
+  return config.NODE_ENV !== 'test' && config.CFP_SCHEDULED_PURGE === 'true';
+}
+
 declare module 'fastify' {
   interface FastifyInstance {
     spamPurge: SpamPurgeService;
@@ -20,6 +28,7 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {
     fts: fastify.fts,
     store: fastify.store,
     log: fastify.log,
+    sessionMetadata: fastify.sessionMetadata,
   });
   fastify.decorate('spamPurge', service);
 
@@ -31,8 +40,10 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {
 
   let interval: ReturnType<typeof setInterval> | undefined;
   fastify.addHook('onReady', async () => {
-    // Tests boot many apps; the sweep is exercised directly there.
-    if (fastify.config.NODE_ENV === 'test') return;
+    if (!scheduledPurgeEnabled(fastify.config)) {
+      fastify.log.info('scheduled spam purge disabled (CFP_SCHEDULED_PURGE is not "true")');
+      return;
+    }
     run();
     interval = setInterval(run, SPAM_PURGE_INTERVAL_MS);
   });
@@ -44,5 +55,5 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {
 export default fp(spamPurgePlugin, {
   name: 'spam-purge',
   fastify: '5.x',
-  dependencies: ['services'],
+  dependencies: ['services', 'session-middleware'],
 });

apps/api/src/routes/moderation.ts

@@ -14,6 +14,7 @@ import { ApiNotFoundError, ApiValidationError } from '../lib/errors.js';
 import { getCallerSession } from '../services/permissions.js';
 import { buildTransactionOptions } from '../store/commit-meta.js';
 import type { MemberOrigin, VoteFilter } from '../services/moderation.js';
+import { findOrphanPrivateData, purgeOrphanPrivateData } from '../services/private-orphans.js';
 
 function requireStaffOr404(request: FastifyRequest): void {
   const level = request.session.accountLevel;
@@ -139,4 +140,39 @@ export async function moderationRoutes(fastify: FastifyInstance): Promise<void>
       });
     },
   );
+
+  // POST /api/admin/private-data/purge-orphans (administrator only)
+  // Spec: specs/api/moderation.md → POST /api/admin/private-data/purge-orphans
+  fastify.post(
+    '/api/admin/private-data/purge-orphans',
+    {
+      schema: {
+        tags: ['admin'],
+        summary: 'Delete private data of people with no public record (admin only)',
+        body: {
+          type: 'object',
+          properties: { apply: { type: 'boolean' } },
+          required: ['apply'],
+          additionalProperties: false,
+        },
+      },
+    },
+    async (request) => {
+      if (request.session.accountLevel !== 'administrator') throw new ApiNotFoundError('Not found');
+      const { apply } = request.body as { apply: boolean };
+      // Under the write lock: account creation writes the private profile
+      // before the public record, and must never look orphaned mid-write.
+      const found = await fastify.publicRepo.withLock(async () => {
+        const publicIds = new Set(fastify.inMemoryState.people.keys());
+        const result = await findOrphanPrivateData(publicIds, fastify.store.private, fastify.sessionMetadata);
+        if (apply && result.personIds.length > 0) {
+          await purgeOrphanPrivateData(result, fastify.store.private, fastify.sessionMetadata);
+        }
+        return result;
+      });
+      const { personIds, ...counts } = found;
+      request.log.info({ apply, orphanPeople: personIds.length, ...counts }, 'private orphan purge');
+      return ok({ applied: apply, orphanPeople: personIds.length, ...counts });
+    },
+  );
 }

apps/api/src/routes/people.ts

@@ -239,6 +239,7 @@ export async function peopleRoutes(fastify: FastifyInstance): Promise<void> {
     },
   }, async (request, reply) => {
     const { slug } = request.params as { slug: string };
+    const personId = fastify.inMemoryState.personIdBySlug.get(slug);
     const result = await fastify.store.transact(
       buildTransactionOptions({
         request,
@@ -250,6 +251,7 @@ export async function peopleRoutes(fastify: FastifyInstance): Promise<void> {
       async (tx) => fastify.services.peopleWrite.purge(tx, slug, request.session),
     );
     result.value.stateApply.apply(fastify.inMemoryState, fastify.fts);
+    if (personId) await fastify.sessionMetadata.removeAllFor(personId, fastify.store.private);
     return reply.code(204).send();
   });
 

apps/api/src/services/person-purge.ts

@@ -43,6 +43,9 @@ async function purgePersonRecords(
   // 1. Delete the person record.
   await tx.public.people.delete(existing);
   stateApply.removePerson(personId, existing.slug);
+  // Their private data goes in the same transaction (profile, legacy password,
+  // reset tokens, claim requests).
+  tx.private.deletePersonData(personId);
 
   // 2. Cascade-delete project-memberships.
   const membershipIds = state.membershipsByPerson.get(existing.slug) ?? new Set<string>();

apps/api/src/services/private-orphans.ts

@@ -0,0 +1,78 @@
+/**
+ * Private data left by people who no longer exist publicly — the cleanup
+ * behind `POST /api/admin/private-data/purge-orphans`
+ * (specs/api/moderation.md). Purge deletes private data in the same
+ * transaction now; everyone purged or pruned before that left a private
+ * profile, legacy password, claim requests and session metadata behind.
+ *
+ * Runs inside the API on purpose: the API holds the private store in memory
+ * and rewrites whole files on each private write, so an out-of-process
+ * cleanup would be undone by the next write.
+ */
+import type { SessionMetadataStore } from '../auth/session-metadata.js';
+import type { PrivateStore } from '../store/private/index.js';
+
+export interface OrphanPrivateData {
+  /** Person ids with private data and no public record. */
+  readonly personIds: string[];
+  readonly profiles: number;
+  readonly legacyPasswords: number;
+  readonly claimRequests: number;
+  readonly sessions: number;
+}
+
+/** Find private data whose person is not in `publicIds`. */
+export async function findOrphanPrivateData(
+  publicIds: ReadonlySet<string>,
+  privateStore: PrivateStore,
+  sessions: SessionMetadataStore,
+): Promise<OrphanPrivateData> {
+  const orphans = new Set<string>();
+  let profiles = 0;
+  let legacyPasswords = 0;
+  let claimRequests = 0;
+  let sessionCount = 0;
+
+  for await (const profile of privateStore.listAllProfiles()) {
+    if (!publicIds.has(profile.personId)) {
+      orphans.add(profile.personId);
+      profiles++;
+    }
+  }
+  for (const personId of privateStore.indices.legacyPasswordByPersonId.keys()) {
+    if (!publicIds.has(personId)) {
+      orphans.add(personId);
+      legacyPasswords++;
+    }
+  }
+  for (const req of await privateStore.listAllClaimRequests()) {
+    const ids = [req.claimedPersonId, req.requesterPersonId].filter((id): id is string => !!id);
+    const gone = ids.filter((id) => !publicIds.has(id));
+    if (gone.length > 0) {
+      gone.forEach((id) => orphans.add(id));
+      claimRequests++;
+    }
+  }
+  for (const personId of orphans) sessionCount += sessions.getAll(personId).length;
+
+  return {
+    personIds: [...orphans].sort(),
+    profiles,
+    legacyPasswords,
+    claimRequests,
+    sessions: sessionCount,
+  };
+}
+
+/** Delete the private data of `found.personIds`. */
+export async function purgeOrphanPrivateData(
+  found: OrphanPrivateData,
+  privateStore: PrivateStore,
+  sessions: SessionMetadataStore,
+): Promise<void> {
+  await privateStore.transact(async (tx) => {
+    for (const personId of found.personIds) tx.deletePersonData(personId);
+  });
+  for (const personId of found.personIds) await sessions.removeAllFor(personId, privateStore);
+}
+

apps/api/src/services/spam-purge.ts

@@ -11,6 +11,7 @@ import type { FtsEngine } from '../store/fts.js';
 import type { InMemoryState } from '../store/memory/state.js';
 import { StateApply } from '../store/state-apply.js';
 import type { Store } from '../store/store.js';
+import type { SessionMetadataStore } from '../auth/session-metadata.js';
 import { purgePeople } from './person-purge.js';
 
 /** How long a spam vote stands before the sweep acts on it. */
@@ -48,6 +49,7 @@ export interface SpamPurgeDeps {
   readonly fts: FtsEngine;
   readonly store: Store;
   readonly log: FastifyBaseLogger;
+  readonly sessionMetadata: SessionMetadataStore;
 }
 
 export class SpamPurgeService {
@@ -86,6 +88,9 @@ export class SpamPurgeService {
       throw err;
     }
     stateApply.apply(this.#d.state, this.#d.fts);
+    for (const person of queued) {
+      await this.#d.sessionMetadata.removeAllFor(person.id, this.#d.store.private);
+    }
     this.#d.log.info({ purged: slugs }, 'scheduled spam purge');
     return slugs;
   }

apps/api/src/store/private/base.ts

@@ -174,13 +174,15 @@ export abstract class BasePrivateStore implements PrivateStore {
     const profilesSnapshot = new Map(this.profiles);
     const legacySnapshot = new Map(this.legacyPasswords);
     const claimRequestsSnapshot = new Map(this.claimRequests);
+    const passwordTokensSnapshot = new Map(this.passwordTokens);
 
     // Staged mutations applied only in-memory during the handler
     const stagedProfilePuts: Map<string, PrivateProfile> = new Map();
     const stagedProfileDeletes: Set<string> = new Set();
     const stagedLegacyPuts: Map<string, LegacyPasswordCredential> = new Map();
     const stagedLegacyDeletes: Set<string> = new Set();
     const stagedClaimRequestPuts: Map<string, AccountClaimRequest> = new Map();
+    const stagedPersonPurges: Set<string> = new Set();
 
     const tx: PrivateStoreTx = {
       putProfile: (profile) => {
@@ -205,6 +207,11 @@ export abstract class BasePrivateStore implements PrivateStore {
         const parsed = AccountClaimRequestSchema.parse(req);
         stagedClaimRequestPuts.set(parsed.id, parsed);
       },
+      deletePersonData: (personId) => {
+        stagedPersonPurges.add(personId);
+        stagedProfilePuts.delete(personId);
+        stagedLegacyPuts.delete(personId);
+      },
     };
 
     let result: T;
@@ -215,6 +222,7 @@ export abstract class BasePrivateStore implements PrivateStore {
       this.profiles = profilesSnapshot;
       this.legacyPasswords = legacySnapshot;
       this.claimRequests = claimRequestsSnapshot;
+      this.passwordTokens = passwordTokensSnapshot;
       this.rebuildIndices();
       throw err;
     }
@@ -235,19 +243,33 @@ export abstract class BasePrivateStore implements PrivateStore {
     for (const [id, req] of stagedClaimRequestPuts) {
       this.claimRequests.set(id, req);
     }
+    for (const personId of stagedPersonPurges) {
+      this.profiles.delete(personId);
+      this.legacyPasswords.delete(personId);
+      for (const [id, req] of this.claimRequests) {
+        if (req.claimedPersonId === personId || req.requesterPersonId === personId) this.claimRequests.delete(id);
+      }
+      for (const [hash, token] of this.passwordTokens) {
+        if (token.personId === personId) this.passwordTokens.delete(hash);
+      }
+    }
     this.rebuildIndices();
 
     // Flush to backend
     const flushOps: Promise<void>[] = [];
-    if (stagedProfilePuts.size > 0 || stagedProfileDeletes.size > 0) {
+    const purging = stagedPersonPurges.size > 0;
+    if (purging || stagedProfilePuts.size > 0 || stagedProfileDeletes.size > 0) {
       flushOps.push(this.flushProfiles());
     }
-    if (stagedLegacyPuts.size > 0 || stagedLegacyDeletes.size > 0) {
+    if (purging || stagedLegacyPuts.size > 0 || stagedLegacyDeletes.size > 0) {
       flushOps.push(this.flushLegacyPasswords());
     }
-    if (stagedClaimRequestPuts.size > 0) {
+    if (purging || stagedClaimRequestPuts.size > 0) {
       flushOps.push(this.flushClaimRequests());
     }
+    if (purging) {
+      flushOps.push(this.flushPasswordTokens());
+    }
 
     try {
       await Promise.all(flushOps);
@@ -257,6 +279,7 @@ export abstract class BasePrivateStore implements PrivateStore {
       this.profiles = profilesSnapshot;
       this.legacyPasswords = legacySnapshot;
       this.claimRequests = claimRequestsSnapshot;
+      this.passwordTokens = passwordTokensSnapshot;
       this.rebuildIndices();
       throw new PrivateStoreError(
         'private_store_unavailable',