From 7fa7242f7c3938dd1c0dc65c47e44b66d9cbd7c1 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:46:02 -0700 Subject: [PATCH 01/16] server: add the vote convention module (src/votes/convention.ts) One file names the vote sign. It owns the wire convention (frozen at agree = -1; named WIRE_AGREE/WIRE_DISAGREE/WIRE_PASS), the export convention (fixed at agree = +1), and the storage convention, read from an injectable ConventionSource whose only implementation today is the constant STORAGE_AGREE_VALUE = -1. The pure conversions map between the three numeric forms and a typed Vote ('agree' | 'disagree' | 'pass'). NULL in is NULL out for every conversion; the CSV exports' NULL cell is the named EXPORT_NULL_VALUE (0, as today). Out-of-range integers throw unless the caller opts into "skip" (semantic reads) or "keep" (keep today's bytes), so a legacy tolerance is visible at the call site. Unit tests cover every function, NULL, out-of-range values, round trips under both storage conventions, and an injected source. No call site changes in this commit. No schema change. No API change. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- server/__tests__/unit/voteConvention.test.ts | 269 ++++++++++++++ server/src/votes/convention.ts | 369 +++++++++++++++++++ 2 files changed, 638 insertions(+) create mode 100644 server/__tests__/unit/voteConvention.test.ts create mode 100644 server/src/votes/convention.ts diff --git a/server/__tests__/unit/voteConvention.test.ts b/server/__tests__/unit/voteConvention.test.ts new file mode 100644 index 0000000000..7bfc01e00f --- /dev/null +++ b/server/__tests__/unit/voteConvention.test.ts @@ -0,0 +1,269 @@ +import { afterEach, describe, expect, test } from "@jest/globals"; + +import { + CONSTANT_CONVENTION_SOURCE, + EXPORT_AGREE, + EXPORT_DISAGREE, + EXPORT_NULL_VALUE, + EXPORT_PASS, + STORAGE_AGREE_VALUE, + StorageConvention, + VOTES, + Vote, + VoteConventionError, + WIRE_AGREE, + WIRE_AGREE_VALUE, + WIRE_DISAGREE, + WIRE_PASS, + WIRE_VOTE_MAX, + WIRE_VOTE_MIN, + addToTally, + currentStorageConvention, + emptyTally, + exportToSemantic, + exportToStorage, + isVote, + resetConventionSource, + semanticToExport, + semanticToStorage, + semanticToWire, + setConventionSource, + storageSqlValue, + storageToExport, + storageToSemantic, + storageToWire, + wireToSemantic, + wireToStorage, +} from "../../src/votes/convention"; + +// The storage convention after the planned un-flip, used to prove that the +// wire and the export do not move when storage does. +const FLIPPED: StorageConvention = { agreeValue: 1, version: 1 }; +const TODAY: StorageConvention = { agreeValue: -1, version: 0 }; + +afterEach(() => resetConventionSource()); + +describe("the constants", () => { + test("wire is frozen at agree = -1", () => { + expect(WIRE_AGREE_VALUE).toBe(-1); + expect([WIRE_AGREE, WIRE_DISAGREE, WIRE_PASS]).toEqual([-1, 1, 0]); + expect([WIRE_VOTE_MIN, WIRE_VOTE_MAX]).toEqual([-1, 1]); + }); + + test("export is fixed at agree = +1", () => { + expect([EXPORT_AGREE, EXPORT_DISAGREE, EXPORT_PASS]).toEqual([1, -1, 0]); + expect(EXPORT_NULL_VALUE).toBe(0); + }); + + test("storage today is agree = -1 from the constant source", () => { + expect(STORAGE_AGREE_VALUE).toBe(-1); + expect(currentStorageConvention()).toEqual({ + agreeValue: -1, + version: null, + }); + expect(CONSTANT_CONVENTION_SOURCE.current().agreeValue).toBe(-1); + }); + + test("VOTES lists the three semantic votes and isVote recognises only them", () => { + expect([...VOTES]).toEqual(["agree", "disagree", "pass"]); + for (const v of VOTES) expect(isVote(v)).toBe(true); + for (const v of ["Agree", "skip", -1, 0, 1, null, undefined]) { + expect(isVote(v)).toBe(false); + } + }); +}); + +describe("the convention source", () => { + test("an injected source is used by every storage conversion", () => { + setConventionSource({ current: () => FLIPPED }); + expect(currentStorageConvention()).toBe(FLIPPED); + expect(storageToSemantic(1)).toBe("agree"); + expect(semanticToStorage("agree")).toBe(1); + expect(wireToStorage(-1)).toBe(1); + expect(storageToWire(1)).toBe(-1); + expect(exportToStorage(1)).toBe(1); + expect(storageToExport(1)).toBe(1); + expect(storageSqlValue("agree")).toBe(1); + }); + + test("reset restores the constant", () => { + setConventionSource({ current: () => FLIPPED }); + resetConventionSource(); + expect(currentStorageConvention().agreeValue).toBe(-1); + }); + + test("a source returning an impossible agree value is refused", () => { + setConventionSource({ + current: () => ({ agreeValue: 0 as unknown as -1, version: 9 }), + }); + expect(() => currentStorageConvention()).toThrow(VoteConventionError); + expect(() => storageToSemantic(-1)).toThrow(VoteConventionError); + }); + + test("a per-call convention overrides the source", () => { + expect(storageToSemantic(1, { convention: FLIPPED })).toBe("agree"); + expect(storageToSemantic(1)).toBe("disagree"); + }); +}); + +describe("each conversion, today's convention", () => { + const table: Array<[Vote, number, number, number]> = [ + // vote, wire, storage, export + ["agree", -1, -1, 1], + ["disagree", 1, 1, -1], + ["pass", 0, 0, 0], + ]; + + test.each(table)( + "%s: wire %d, storage %d, export %d", + (vote, wire, storage, exp) => { + expect(wireToSemantic(wire)).toBe(vote); + expect(semanticToWire(vote)).toBe(wire); + expect(storageToSemantic(storage)).toBe(vote); + expect(semanticToStorage(vote)).toBe(storage); + expect(exportToSemantic(exp)).toBe(vote); + expect(semanticToExport(vote)).toBe(exp); + expect(wireToStorage(wire)).toBe(storage); + expect(storageToWire(storage)).toBe(wire); + expect(exportToStorage(exp)).toBe(storage); + expect(storageToExport(storage)).toBe(exp); + expect(storageSqlValue(vote)).toBe(storage); + } + ); + + test("pass never becomes negative zero", () => { + expect(Object.is(storageToExport(0), 0)).toBe(true); + expect(Object.is(wireToStorage(0), 0)).toBe(true); + expect(Object.is(exportToStorage(0), 0)).toBe(true); + expect(Object.is(semanticToExport("pass"), 0)).toBe(true); + }); +}); + +describe("each conversion, after the un-flip (agree = +1 in storage)", () => { + test("the wire and export do not move; only storage does", () => { + const o = { convention: FLIPPED }; + expect(wireToStorage(WIRE_AGREE, o)).toBe(1); + expect(wireToStorage(WIRE_DISAGREE, o)).toBe(-1); + expect(storageToWire(1, o)).toBe(WIRE_AGREE); + expect(storageToExport(1, o)).toBe(EXPORT_AGREE); + expect(exportToStorage(EXPORT_AGREE, o)).toBe(1); + expect(storageToSemantic(-1, o)).toBe("disagree"); + expect(storageSqlValue("disagree", o)).toBe(-1); + }); + + test("a wire agree exports as +1 under both storage conventions", () => { + for (const convention of [TODAY, FLIPPED]) { + const o = { convention }; + expect(storageToExport(wireToStorage(WIRE_AGREE, o), o)).toBe( + EXPORT_AGREE + ); + } + }); +}); + +describe("NULL: in null, out null (export writes EXPORT_NULL_VALUE)", () => { + for (const nullish of [null, undefined]) { + test(`${String(nullish)}`, () => { + expect(wireToSemantic(nullish)).toBeNull(); + expect(storageToSemantic(nullish)).toBeNull(); + expect(exportToSemantic(nullish)).toBeNull(); + expect(wireToStorage(nullish)).toBeNull(); + expect(storageToWire(nullish)).toBeNull(); + expect(exportToStorage(nullish)).toBeNull(); + expect(storageToExport(nullish)).toBe(EXPORT_NULL_VALUE); + // Even under "throw", NULL is not an error. + expect(storageToSemantic(nullish, { onInvalid: "throw" })).toBeNull(); + }); + } + + test("the export NULL cell is byte-identical to the old String(-null)", () => { + expect(String(storageToExport(null))).toBe(String(-null)); + }); +}); + +describe("out-of-range values", () => { + const invalid = [2, -2, 7, 0.5, NaN]; + + test.each(invalid)("%p throws by default", (raw) => { + expect(() => wireToSemantic(raw)).toThrow(VoteConventionError); + expect(() => storageToSemantic(raw)).toThrow(VoteConventionError); + expect(() => exportToSemantic(raw)).toThrow(VoteConventionError); + expect(() => wireToStorage(raw)).toThrow(VoteConventionError); + expect(() => storageToWire(raw)).toThrow(VoteConventionError); + expect(() => exportToStorage(raw)).toThrow(VoteConventionError); + expect(() => storageToExport(raw)).toThrow(VoteConventionError); + }); + + test.each(invalid)("%p is null under skip for semantic reads", (raw) => { + expect(storageToSemantic(raw, { onInvalid: "skip" })).toBeNull(); + expect(wireToSemantic(raw, { onInvalid: "skip" })).toBeNull(); + expect(exportToSemantic(raw, { onInvalid: "skip" })).toBeNull(); + }); + + test.each(invalid)( + "%p is unchanged under keep for numeric conversions", + (raw) => { + expect(wireToStorage(raw, { onInvalid: "keep" })).toBe(raw); + expect(storageToWire(raw, { onInvalid: "keep" })).toBe(raw); + expect(exportToStorage(raw, { onInvalid: "keep" })).toBe(raw); + } + ); + + test.each([2, -2, 7])( + "%p exports under keep exactly as the old String(-row.vote)", + (raw) => { + expect(String(storageToExport(raw, { onInvalid: "keep" }))).toBe( + String(-raw) + ); + } + ); + + test("the error names the representation and the value", () => { + expect(() => storageToSemantic(7)).toThrow(/7 is not a storage vote/); + }); + + test("an unknown semantic value is refused", () => { + expect(() => semanticToWire("Agree" as Vote)).toThrow(VoteConventionError); + expect(() => semanticToStorage("skip" as Vote)).toThrow( + VoteConventionError + ); + expect(() => storageSqlValue("" as Vote)).toThrow(VoteConventionError); + }); +}); + +describe("round trips", () => { + for (const convention of [TODAY, FLIPPED]) { + const o = { convention }; + describe(`storage agree = ${convention.agreeValue}`, () => { + test.each([...VOTES])("%s survives every representation", (vote) => { + expect(wireToSemantic(semanticToWire(vote))).toBe(vote); + expect(storageToSemantic(semanticToStorage(vote, o), o)).toBe(vote); + expect(exportToSemantic(semanticToExport(vote))).toBe(vote); + }); + + test.each([-1, 0, 1])("wire %d → storage → wire", (wire) => { + expect(storageToWire(wireToStorage(wire, o), o)).toBe(wire); + }); + + test.each([-1, 0, 1])("export %d → storage → export", (exp) => { + expect(storageToExport(exportToStorage(exp, o), o)).toBe(exp); + }); + + test.each([-1, 0, 1])("storage %d → wire → storage", (raw) => { + expect(wireToStorage(storageToWire(raw, o), o)).toBe(raw); + }); + }); + } +}); + +describe("tally", () => { + test("counts semantic votes and NULL separately", () => { + const t = emptyTally(); + for (const raw of [-1, -1, 1, 0, null, 7]) { + addToTally(t, storageToSemantic(raw, { onInvalid: "skip" })); + } + expect(t).toEqual({ agree: 2, disagree: 1, pass: 1, none: 2 }); + addToTally(t, "pass", 3); + expect(t.pass).toBe(4); + }); +}); diff --git a/server/src/votes/convention.ts b/server/src/votes/convention.ts new file mode 100644 index 0000000000..daa23810be --- /dev/null +++ b/server/src/votes/convention.ts @@ -0,0 +1,369 @@ +/** + * The vote sign convention. Every vote value the server reads, writes, serves, + * exports or imports crosses one of the functions in this file; no other server + * file may compare a vote with -1/1 or negate one (see + * __tests__/unit/voteSignLiterals.test.ts, which fails on a new one). + * + * Four representations exist, and only this file knows how they relate: + * + * semantic 'agree' | 'disagree' | 'pass' (the typed `Vote`) + * wire the number clients POST and the read routes return. + * FROZEN at agree = -1, disagree = +1, pass = 0. Cached client + * bundles post these numbers, so the wire never changes sign; a + * cleaner wire later is a new string field, not a flip of this one. + * storage the number in votes.vote / votes_latest_unique.vote. Today + * agree = -1, the same as the wire. The value comes from a + * ConventionSource; today that is the constant + * STORAGE_AGREE_VALUE, and the database-resident convention row + * can replace it later through setConventionSource() without + * touching any call site. + * export the number in the CSV exports (votes.csv, participant-votes.csv) + * and the votes-bulk import. FIXED at agree = +1, disagree = -1, + * pass = 0, independent of storage. + * + * NULL. votes.vote is nullable with no CHECK. Every conversion maps NULL (and + * undefined) to null: NULL in, NULL out, never a pass and never an error. What a + * NULL means for a count or a CSV cell is the caller's policy; the one export + * policy (a NULL cell is written "0", as it always has been) is named below as + * EXPORT_NULL_VALUE. + * + * Out-of-range integers (2, 7, NaN from an unparsed import cell). The column has + * no CHECK, so they can exist. By default every conversion throws + * VoteConventionError. Paths that must keep today's bytes for such rows opt in + * explicitly with { onInvalid: "skip" } (semantic reads: treated like NULL) or + * { onInvalid: "keep" } (numeric conversions: the value is returned unchanged). + */ + +export type Vote = "agree" | "disagree" | "pass"; + +export const VOTES: readonly Vote[] = Object.freeze([ + "agree", + "disagree", + "pass", +]); + +/** The sign the agree vote takes in a numeric representation. */ +export type AgreeValue = -1 | 1; + +/** A raw numeric vote as it comes out of a row, a request or a CSV cell. */ +export type RawVote = number | null | undefined; + +export type InvalidPolicy = "throw" | "skip" | "keep"; + +export interface ConversionOptions { + /** + * What to do with an integer outside {-1, 0, 1}: + * "throw" (default) raise VoteConventionError; + * "skip" return null (semantic reads only); + * "keep" return the value unchanged (numeric conversions only). + */ + onInvalid?: InvalidPolicy; + /** Override the storage convention for this call (tests, migrations). */ + convention?: StorageConvention; +} + +export class VoteConventionError extends Error { + readonly raw: unknown; + readonly representation: string; + constructor(representation: string, raw: unknown) { + super( + `polis_err_vote_convention: ${String( + raw + )} is not a ${representation} vote (expected -1, 0, 1 or null)` + ); + this.name = "VoteConventionError"; + this.raw = raw; + this.representation = representation; + } +} + +// --------------------------------------------------------------------------- +// The numeric conventions +// --------------------------------------------------------------------------- + +/** Wire: frozen. Clients and cached bundles post these. */ +export const WIRE_AGREE_VALUE = -1 as const; +export const WIRE_AGREE = WIRE_AGREE_VALUE; +export const WIRE_DISAGREE = -WIRE_AGREE_VALUE as 1; +export const WIRE_PASS = 0 as const; +/** The inclusive range the POST /votes and POST /comments parsers accept. */ +export const WIRE_VOTE_MIN = Math.min(WIRE_AGREE, WIRE_DISAGREE, WIRE_PASS); +export const WIRE_VOTE_MAX = Math.max(WIRE_AGREE, WIRE_DISAGREE, WIRE_PASS); + +/** Export: fixed. The CSV exports and the votes-bulk import use agree = +1. */ +export const EXPORT_AGREE_VALUE = 1 as const; +export const EXPORT_AGREE = EXPORT_AGREE_VALUE; +export const EXPORT_DISAGREE = -EXPORT_AGREE_VALUE as -1; +export const EXPORT_PASS = 0 as const; +/** + * A NULL stored vote is written to the vote CSV exports as this value. This is + * the exports' behaviour since they were written (`String(-null)` is "0"); it is + * kept byte-identical here and named so a later change is one line. + */ +export const EXPORT_NULL_VALUE = 0 as const; + +/** + * Storage: the convention every row in votes / votes_latest_unique was written + * under since 2012. The one TypeScript statement of it. + */ +export const STORAGE_AGREE_VALUE: AgreeValue = -1; + +export interface StorageConvention { + /** The value agree takes in storage. Disagree is its negation; pass is 0. */ + readonly agreeValue: AgreeValue; + /** The convention row's version when it came from the database; null for the constant. */ + readonly version: number | null; +} + +/** + * Where the storage convention comes from. Today: the constant. Later: the + * database row (P-078 PR-A), supplied through setConventionSource() at startup. + * current() is synchronous on purpose so call sites never change shape; a + * database-backed source caches the row it last read. + */ +export interface ConventionSource { + current(): StorageConvention; +} + +export const CONSTANT_CONVENTION_SOURCE: ConventionSource = Object.freeze({ + current: (): StorageConvention => + Object.freeze({ agreeValue: STORAGE_AGREE_VALUE, version: null }), +}); + +let activeSource: ConventionSource = CONSTANT_CONVENTION_SOURCE; + +export function setConventionSource(source: ConventionSource): void { + activeSource = source; +} + +export function resetConventionSource(): void { + activeSource = CONSTANT_CONVENTION_SOURCE; +} + +export function currentStorageConvention(): StorageConvention { + const convention = activeSource.current(); + if (convention.agreeValue !== 1 && convention.agreeValue !== -1) { + throw new VoteConventionError( + "storage convention agree", + convention.agreeValue + ); + } + return convention; +} + +// --------------------------------------------------------------------------- +// Internal: one table per numeric representation +// --------------------------------------------------------------------------- + +function isNullish(raw: unknown): raw is null | undefined { + return raw === null || raw === undefined; +} + +function toSemantic( + raw: RawVote, + agree: AgreeValue, + representation: string, + onInvalid: InvalidPolicy +): Vote | null { + if (isNullish(raw)) return null; + if (raw === agree) return "agree"; + if (raw === -agree) return "disagree"; + if (raw === 0) return "pass"; + if (onInvalid === "skip") return null; + throw new VoteConventionError(representation, raw); +} + +function fromSemantic(vote: Vote, agree: AgreeValue): number { + switch (vote) { + case "agree": + return agree; + case "disagree": + return -agree; + case "pass": + return 0; + default: + throw new VoteConventionError("semantic", vote); + } +} + +function convert( + raw: RawVote, + fromAgree: AgreeValue, + toAgree: AgreeValue, + representation: string, + onInvalid: InvalidPolicy +): number | null { + if (isNullish(raw)) return null; + const vote = toSemantic(raw, fromAgree, representation, "skip"); + if (vote === null) { + if (onInvalid === "keep") return raw; + throw new VoteConventionError(representation, raw); + } + return fromSemantic(vote, toAgree); +} + +function storageAgree(options?: ConversionOptions): AgreeValue { + return (options?.convention ?? currentStorageConvention()).agreeValue; +} + +// --------------------------------------------------------------------------- +// Public conversions +// --------------------------------------------------------------------------- + +export function isVote(value: unknown): value is Vote { + return value === "agree" || value === "disagree" || value === "pass"; +} + +/** Wire number → Vote. NULL → null. */ +export function wireToSemantic( + raw: RawVote, + options?: ConversionOptions +): Vote | null { + return toSemantic( + raw, + WIRE_AGREE_VALUE, + "wire", + options?.onInvalid ?? "throw" + ); +} + +export function semanticToWire(vote: Vote): number { + return fromSemantic(vote, WIRE_AGREE_VALUE); +} + +/** Stored number → Vote. NULL → null. */ +export function storageToSemantic( + raw: RawVote, + options?: ConversionOptions +): Vote | null { + return toSemantic( + raw, + storageAgree(options), + "storage", + options?.onInvalid ?? "throw" + ); +} + +export function semanticToStorage( + vote: Vote, + options?: ConversionOptions +): number { + return fromSemantic(vote, storageAgree(options)); +} + +/** Export number (agree = +1) → Vote. NULL → null. */ +export function exportToSemantic( + raw: RawVote, + options?: ConversionOptions +): Vote | null { + return toSemantic( + raw, + EXPORT_AGREE_VALUE, + "export", + options?.onInvalid ?? "throw" + ); +} + +export function semanticToExport(vote: Vote): number { + return fromSemantic(vote, EXPORT_AGREE_VALUE); +} + +/** The write path: a wire value from POST /votes or POST /comments → the value to INSERT. */ +export function wireToStorage( + raw: RawVote, + options?: ConversionOptions +): number | null { + return convert( + raw, + WIRE_AGREE_VALUE, + storageAgree(options), + "wire", + options?.onInvalid ?? "throw" + ); +} + +/** The read routes: a stored value → the value GET /votes, /votes/me and participationInit return. */ +export function storageToWire( + raw: RawVote, + options?: ConversionOptions +): number | null { + return convert( + raw, + storageAgree(options), + WIRE_AGREE_VALUE, + "storage", + options?.onInvalid ?? "throw" + ); +} + +/** The votes-bulk import: a CSV vote_value (agree = +1) → the value to INSERT. */ +export function exportToStorage( + raw: RawVote, + options?: ConversionOptions +): number | null { + return convert( + raw, + EXPORT_AGREE_VALUE, + storageAgree(options), + "export", + options?.onInvalid ?? "throw" + ); +} + +/** + * The vote CSV exports: a stored value → the number written to the cell. + * NULL → EXPORT_NULL_VALUE. An out-of-range stored integer throws by default; + * with { onInvalid: "keep" } it is written as the exports have always written it + * (`String(-row.vote)`: the value scaled by the storage and export agree signs), + * so the export bytes do not change until the census (P-078 §1e.4) shows no + * such rows exist. + */ +export function storageToExport( + raw: RawVote, + options?: ConversionOptions +): number { + if (isNullish(raw)) return EXPORT_NULL_VALUE; + const agree = storageAgree(options); + const vote = toSemantic(raw, agree, "storage", "skip"); + if (vote === null) { + if (options?.onInvalid === "keep") return raw * agree * EXPORT_AGREE_VALUE; + throw new VoteConventionError("storage", raw); + } + return semanticToExport(vote); +} + +/** + * The integer to put in a SQL predicate that selects a stored vote, e.g. + * `CASE WHEN v.vote = ${storageSqlValue("agree")} THEN 1 ...`. Always an + * integer, so interpolation is safe. + */ +export function storageSqlValue( + vote: Vote, + options?: ConversionOptions +): number { + const value = semanticToStorage(vote, options); + if (!Number.isInteger(value)) throw new VoteConventionError("storage", value); + return value; +} + +/** Counts of semantic votes; NULL and skipped values are counted in `none`. */ +export interface VoteTally { + agree: number; + disagree: number; + pass: number; + none: number; +} + +export function emptyTally(): VoteTally { + return { agree: 0, disagree: 0, pass: 0, none: 0 }; +} + +export function addToTally( + tally: VoteTally, + vote: Vote | null, + n = 1 +): VoteTally { + if (vote === null) tally.none += n; + else tally[vote] += n; + return tally; +} From 114a385a23bbd4c7262b2a5cac894cc3db4bc3f5 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:47:39 -0700 Subject: [PATCH 02/16] server: route the CSV exports through the vote convention module report.ts had three hand-written flips (votes.csv String(-row.vote), participant-votes.csv and participant-importance.csv -row.vote) and two opposite literal tables in one file: comments.csv counted -1 as agree, participant-votes.csv counted 1 as agree, each right only because of where the flip sat. Both now classify the stored value once through storageToSemantic and count by name; the cells are written by storageToExport (agree = +1). The importance export never used the flipped value, so its map is gone and n-votes counts the voted tids. The bytes do not change: a new test pins every vote column of the four exports for agree, disagree, pass, NULL (written 0, as before) and an out-of-range stored value (written negated, as before), and passes on both the old and the new report.ts. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../unit/exportVoteSignBytes.test.ts | 181 ++++++++++++++++++ server/src/report.ts | 44 +++-- 2 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 server/__tests__/unit/exportVoteSignBytes.test.ts diff --git a/server/__tests__/unit/exportVoteSignBytes.test.ts b/server/__tests__/unit/exportVoteSignBytes.test.ts new file mode 100644 index 0000000000..afa3b66d70 --- /dev/null +++ b/server/__tests__/unit/exportVoteSignBytes.test.ts @@ -0,0 +1,181 @@ +/** + * Byte pins for the vote columns of the CSV exports. The expected strings were + * produced by the export code as it stood before the vote convention module + * (report.ts with its hand-written `-row.vote` flips) and must not change: + * agree exports as 1, disagree as -1, pass as 0, a NULL vote as 0, and an + * out-of-range stored value (no CHECK constraint exists) as its negation. + */ +import { beforeEach, describe, expect, jest, test } from "@jest/globals"; + +import pg from "../../src/db/pg-query"; +import { getPcaFromBundle } from "../../src/utils/pca"; +import { + formatDatetime, + sendCommentSummary, + sendParticipantImportance, + sendParticipantVotesSummary, + sendVotesSummary, +} from "../../src/report"; + +jest.mock("../../src/db/pg-query", () => ({ + __esModule: true, + default: { queryP_readOnly: jest.fn(), stream_queryP_readOnly: jest.fn() }, +})); +jest.mock("../../src/utils/zinvite", () => ({ + getZinvite: jest.fn(), + getZidForRid: jest.fn(), +})); +jest.mock("../../src/routes/xids", () => ({ getXids: jest.fn() })); +jest.mock("../../src/utils/pca"); +jest.mock("../../src/utils/logger"); +jest.mock("../../src/utils/fail"); + +type Row = Record; + +function streamRows(rows: Row[]) { + (pg.stream_queryP_readOnly as jest.Mock).mockImplementation( + (...args: any[]) => { + rows.forEach((row) => args[2](row)); + args[3](); + } + ); +} + +function capture() { + const out: string[] = []; + const res = { + setHeader: jest.fn(), + write: jest.fn((s: string) => out.push(s)), + end: jest.fn(), + send: jest.fn((s: string) => out.push(s)), + }; + return { res, text: () => out.join("") }; +} + +const pcaData = { + "in-conv": [1, 2], + "base-clusters": { + members: [[1], [2]], + x: [0, 1], + y: [0, 1], + id: [0, 1], + count: [1, 1], + }, + "group-clusters": [ + { id: 0, center: [0, 0], members: [0] }, + { id: 1, center: [1, 1], members: [1] }, + ], + "user-vote-counts": { 1: 3, 2: 3 }, +}; + +// Stored values today: agree = -1, disagree = 1, pass = 0, plus NULL and an +// out-of-range 2 that the column admits. +const T = 1700000000000; + +beforeEach(() => jest.clearAllMocks()); + +describe("vote CSV exports keep their bytes", () => { + test("votes.csv", async () => { + (pg.queryP_readOnly as jest.Mock).mockResolvedValueOnce([ + { importance_enabled: false }, + ] as never); + streamRows([ + { timestamp: T, tid: 1, pid: 1, vote: -1 }, + { timestamp: T, tid: 1, pid: 2, vote: 1 }, + { timestamp: T, tid: 2, pid: 1, vote: 0 }, + { timestamp: T, tid: 2, pid: 2, vote: null }, + { timestamp: T, tid: 3, pid: 1, vote: 2 }, + ]); + const { res, text } = capture(); + await sendVotesSummary(7, res as any); + const d = formatDatetime(T); + expect(text()).toBe( + "timestamp,datetime,comment-id,voter-id,vote\n" + + `1700000000,${d},1,1,1\n` + + `1700000000,${d},1,2,-1\n` + + `1700000000,${d},2,1,0\n` + + `1700000000,${d},2,2,0\n` + + `1700000000,${d},3,1,-2\n` + ); + }); + + test("participant-votes.csv", async () => { + (pg.queryP_readOnly as jest.Mock).mockResolvedValueOnce([ + { tid: 1, pid: 1 }, + { tid: 2, pid: 1 }, + { tid: 3, pid: 2 }, + ] as never); + (getPcaFromBundle as jest.Mock).mockResolvedValue({ + asPOJO: pcaData, + } as never); + streamRows([ + { pid: 1, tid: 1, vote: -1 }, + { pid: 1, tid: 2, vote: 1 }, + { pid: 1, tid: 3, vote: null }, + { pid: 2, tid: 1, vote: 0 }, + { pid: 2, tid: 2, vote: 2 }, + { pid: 2, tid: 4, vote: -1 }, // a tid outside the comment columns + ]); + const { res, text } = capture(); + await sendParticipantVotesSummary(7, res as any); + expect(text()).toBe( + "participant,group-id,n-comments,n-votes,n-agree,n-disagree,1,2,3\n" + + "1,0,2,3,1,1,1,-1,0\n" + + "2,1,1,3,1,0,0,-2,\n" + ); + }); + + test("participant-importance.csv", async () => { + (pg.queryP_readOnly as jest.Mock).mockResolvedValueOnce([ + { tid: 1, pid: 1 }, + { tid: 2, pid: 1 }, + { tid: 3, pid: 2 }, + ] as never); + (getPcaFromBundle as jest.Mock).mockResolvedValue({ + asPOJO: pcaData, + } as never); + streamRows([ + { pid: 1, tid: 1, vote: -1, high_priority: true }, + { pid: 1, tid: 2, vote: null, high_priority: false }, + { pid: 2, tid: 3, vote: 2, high_priority: true }, + { pid: 2, tid: 4, vote: 1, high_priority: true }, + ]); + const { res, text } = capture(); + await sendParticipantImportance(7, res as any); + expect(text()).toBe( + "participant,group-id,n-comments,n-votes,n-important,1,2,3\n" + + "1,0,2,2,1,1,0,\n" + + "2,1,1,1,1,,,1\n" + ); + }); + + test("comments.csv counts", async () => { + (pg.queryP_readOnly as jest.Mock) + .mockResolvedValueOnce([{ importance_enabled: false }] as never) + .mockResolvedValueOnce([ + { + tid: 1, + pid: 5, + created: String(T), + txt: "a", + mod: 1, + velocity: 1, + active: true, + }, + ] as never); + streamRows([ + { tid: 1, vote: -1, high_priority: false }, + { tid: 1, vote: -1, high_priority: false }, + { tid: 1, vote: 1, high_priority: false }, + { tid: 1, vote: 0, high_priority: false }, + { tid: 1, vote: null, high_priority: false }, + { tid: 1, vote: 2, high_priority: false }, + ]); + const { res, text } = capture(); + await sendCommentSummary(7, res as any); + expect(text()).toBe( + "timestamp,datetime,comment-id,author-id,agrees,disagrees,moderated,comment-body\n" + + `1700000000,${formatDatetime(String(T) as any)},1,5,2,1,1,"a"\n` + ); + }); +}); diff --git a/server/src/report.ts b/server/src/report.ts index 1175299a74..aef128b505 100644 --- a/server/src/report.ts +++ b/server/src/report.ts @@ -7,6 +7,11 @@ import { getXids } from "./routes/xids"; import { getPcaFromBundle } from "./utils/pca"; import { failJson } from "./utils/fail"; import logger from "./utils/logger"; +import { + RawVote, + storageToExport, + storageToSemantic, +} from "./votes/convention"; import { getCommentsWithClusters } from "./utils/commentClusters"; import { presentPca } from "./utils/pcaPresentation"; import type { XidRecord } from "./d"; @@ -334,10 +339,11 @@ export async function sendCommentSummary(zid: number, res: ResponseLike) { (row) => { const comment = comments.get(row.tid); if (comment) { - // note that -1 means agree and 1 means disagree - if (row.vote === -1) comment.agrees += 1; - else if (row.vote === 1) comment.disagrees += 1; - else if (row.vote === 0) comment.pass += 1; + // NULL and out-of-range stored values are counted nowhere, as before. + const vote = storageToSemantic(row.vote, { onInvalid: "skip" }); + if (vote === "agree") comment.agrees += 1; + else if (vote === "disagree") comment.disagrees += 1; + else if (vote === "pass") comment.pass += 1; // Count high priority votes if enabled if (importanceEnabled && row.high_priority) { @@ -400,7 +406,8 @@ export async function sendVotesSummary(zid: number, res: ResponseLike) { datetime: (row) => formatDatetime(row.timestamp), "comment-id": (row) => String(row.tid), "voter-id": (row) => String(row.pid), - vote: (row) => String(-row.vote), // have to flip -1 to 1 and vice versa + // The export's own convention (agree = +1); NULL is EXPORT_NULL_VALUE. + vote: (row) => String(storageToExport(row.vote, { onInvalid: "keep" })), }; // Add important column if enabled @@ -455,15 +462,17 @@ export async function sendParticipantVotesSummary( // Query the votes in participant order so that we can summarize them in a streaming pass let currentParticipantId = -1; - const currentParticipantVotes = new Map(); + // tid -> the stored vote value, converted only when the row is written + const currentParticipantVotes = new Map(); const sendCurrentParticipantRow = () => { const totalVotes = currentParticipantVotes.size; let agrees = 0; let disagrees = 0; - for (const vote of currentParticipantVotes.values()) { - if (vote === 1) agrees += 1; - else if (vote === -1) disagrees += 1; + for (const raw of currentParticipantVotes.values()) { + const vote = storageToSemantic(raw, { onInvalid: "skip" }); + if (vote === "agree") agrees += 1; + else if (vote === "disagree") disagrees += 1; } const values = [ currentParticipantId, @@ -472,7 +481,13 @@ export async function sendParticipantVotesSummary( totalVotes, agrees, disagrees, - ...commentIds.map((tid) => currentParticipantVotes.get(tid)), + ...commentIds.map((tid) => + currentParticipantVotes.has(tid) + ? storageToExport(currentParticipantVotes.get(tid), { + onInvalid: "keep", + }) + : undefined + ), ]; res.write( values @@ -493,8 +508,7 @@ export async function sendParticipantVotesSummary( currentParticipantId = pid; currentParticipantVotes.clear(); } - // have to flip vote from -1 to 1 and vice versa - currentParticipantVotes.set(row.tid, -row.vote); + currentParticipantVotes.set(row.tid, row.vote); }, () => { if (currentParticipantId != -1) { @@ -536,12 +550,12 @@ export async function sendParticipantImportance( // Query the votes in participant order so that we can summarize them in a streaming pass let currentParticipantId = -1; - const currentParticipantVotes = new Map(); const currentParticipantImportance = new Map(); const currentParticipantVotedComments = new Set(); const sendCurrentParticipantRow = () => { - const totalVotes = currentParticipantVotes.size; + // The vote value plays no part in this export; only which tids were voted. + const totalVotes = currentParticipantVotedComments.size; let importantVotes = 0; for (const tid of currentParticipantVotedComments) { if (currentParticipantImportance.get(tid)) { @@ -581,14 +595,12 @@ export async function sendParticipantImportance( sendCurrentParticipantRow(); } currentParticipantId = pid; - currentParticipantVotes.clear(); currentParticipantImportance.clear(); currentParticipantVotedComments.clear(); } if (!commentIdSet.has(row.tid)) { return; } - currentParticipantVotes.set(row.tid, -row.vote); currentParticipantImportance.set(row.tid, row.high_priority || false); currentParticipantVotedComments.add(row.tid); }, From 8e4b6f775b799c6e7dba0ce9ff470a6f5032c9b5 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:48:12 -0700 Subject: [PATCH 03/16] server: route the votes-bulk import through exportToStorage The import swapped exact +1/-1 by hand ("INTENTIONAL VOTE FLIPPING, REMOVE AFTER VOTES REFACTOR"). vote_value is in the export convention (agree = +1, as the admin import screen documents), so the reader now converts it with exportToStorage. Values other than -1/0/1 are still stored as given ({ onInvalid: "keep" }); a new test pins that, and it passes on the old and new code. Refusing them belongs to the declared import format change, not here. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../unit/importProcessorBoundaries.test.ts | 7 +++++++ server/src/workers/import-processor.ts | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/server/__tests__/unit/importProcessorBoundaries.test.ts b/server/__tests__/unit/importProcessorBoundaries.test.ts index 07ef8f2b2f..72496e4a1b 100644 --- a/server/__tests__/unit/importProcessorBoundaries.test.ts +++ b/server/__tests__/unit/importProcessorBoundaries.test.ts @@ -95,6 +95,13 @@ beforeEach(() => { }); describe("CSV import transaction and lifecycle boundaries", () => { + test("stores an out-of-range vote_value unchanged, as before the convention module", async () => { + csv += "v2,public-one,2,2024-01-02T00:00:00Z,comment-one\n"; + csv += "v3,public-two,-7,2024-01-03T00:00:00Z,comment-one\n"; + await processImportJob(payload); + expect(voteInserts()[0][1][3]).toEqual([-1, 2, -7]); + }); + test("maps public CSV signs and comment IDs, retains participant zero, and skips unknown comments", async () => { csv += "v2,public-one,-1,2024-01-02T00:00:00Z,comment-one\n"; csv += "v3,public-two,0,2024-01-03T00:00:00Z,comment-one\n"; diff --git a/server/src/workers/import-processor.ts b/server/src/workers/import-processor.ts index 09cfdcb227..ae1a6daf8f 100644 --- a/server/src/workers/import-processor.ts +++ b/server/src/workers/import-processor.ts @@ -6,6 +6,7 @@ import pg from "../db/pg-query"; import logger from "../utils/logger"; import Config from "../config"; import { sendTextEmail } from "../email/senders"; +import { exportToStorage } from "../votes/convention"; const customEndpoint = Config.AWS_S3_ENDPOINT; const config: S3ClientConfig = { @@ -215,13 +216,12 @@ function mapRowData( const parsed = Date.parse(row.timestamp); if (!isNaN(parsed)) ts = parsed; } - // INTETNIONAL VOTE FLIPPING, REMOVE AFTER VOTES REFACTOR - let voteValue = parseInt(row.vote_value, 10); - if (voteValue === 1) { - voteValue = -1; - } else if (voteValue === -1) { - voteValue = 1; - } + // vote_value is in the export convention (agree = +1, as the admin import + // screen documents). Values other than -1/0/1 are stored as given, as before; + // refusing them is a separate change (P-078 PR-E). + const voteValue = exportToStorage(parseInt(row.vote_value, 10), { + onInvalid: "keep", + }); return [zid, internalTid, row.user_id, voteValue, ts]; } From cfac05cc5c75a2cf66b4bea87bccad4916482bf2 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:49:14 -0700 Subject: [PATCH 04/16] server: route the vote write path through the vote convention module POST /votes and POST /comments parse the vote with getIntInRange(WIRE_VOTE_MIN, WIRE_VOTE_MAX) instead of a bare (-1, 1). doVotesPost, the single INSERT reached from votes, comments and the bulk seed upload, converts the wire value with wireToStorage before writing; today wire and storage agree (-1), so the stored value is unchanged, and when the storage convention changes only the source changes. The seed comments' default author vote is WIRE_PASS rather than a bare 0. The (unreached) modOptions branch reads the vote by name rather than through the push/pull reaction literals. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- server/app.ts | 5 +++-- server/src/routes/comments.ts | 5 +++-- server/src/routes/votes.ts | 15 ++++++++++----- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/server/app.ts b/server/app.ts index 04e5b45576..8faec77e9c 100644 --- a/server/app.ts +++ b/server/app.ts @@ -228,6 +228,7 @@ import { want, wantHeader, } from "./src/utils/parameter"; +import { WIRE_VOTE_MAX, WIRE_VOTE_MIN } from "./src/votes/convention"; const app = express(); const devMode = Config.isDevMode; @@ -810,7 +811,7 @@ export const appReady = helpersInitialized.then( want("xid", getStringLimitLength(1, 999), assignToP), ensureParticipant({ createIfMissing: true, issueJWT: true }), need("txt", getStringLimitLength(1, 997), assignToP), - want("vote", getIntInRange(-1, 1), assignToP), + want("vote", getIntInRange(WIRE_VOTE_MIN, WIRE_VOTE_MAX), assignToP), want("is_seed", getBool, assignToP), attachAuthToken(), handle_POST_comments @@ -1255,7 +1256,7 @@ export const appReady = helpersInitialized.then( want("xid", getStringLimitLength(1, 999), assignToP), ensureParticipant({ createIfMissing: true, issueJWT: true }), need("tid", getInt, assignToP), - need("vote", getIntInRange(-1, 1), assignToP), + need("vote", getIntInRange(WIRE_VOTE_MIN, WIRE_VOTE_MAX), assignToP), want("starred", getBool, assignToP), want("high_priority", getBool, assignToP, false), want("lang", getStringLimitLength(1, 10), assignToP), diff --git a/server/src/routes/comments.ts b/server/src/routes/comments.ts index ad930e2708..6ad7c37ecf 100644 --- a/server/src/routes/comments.ts +++ b/server/src/routes/comments.ts @@ -13,6 +13,7 @@ import { getZinvite } from "../utils/zinvite"; import { isModerator, polisTypes } from "../utils/common"; import { MPromise } from "../utils/metered"; import { votesPost } from "./votes"; +import { WIRE_PASS } from "../votes/convention"; import analyzeComment from "../utils/moderation"; import Config from "../config"; import logger from "../utils/logger"; @@ -515,7 +516,7 @@ async function handle_POST_comments(req: RequestWithP, res: any) { // 7. Handle voting on the comment if specified const shouldDefaultVote = req.p.is_seed && _.isUndefined(vote); - const finalVote = shouldDefaultVote ? 0 : vote; + const finalVote = shouldDefaultVote ? WIRE_PASS : vote; if (!_.isUndefined(finalVote)) { await votesPost(uid, pid, zid, tid, finalVote, 0, false); @@ -948,7 +949,7 @@ async function handle_POST_comments_bulk( // Handle default vote for seed comments (matching handle_POST_comments behavior) if (is_seed) { - await votesPost(uid!, finalPid, zid!, tid, 0, 0, false); + await votesPost(uid!, finalPid, zid!, tid, WIRE_PASS, 0, false); // Schedule vote count update setTimeout(() => { updateVoteCount(zid!, finalPid); diff --git a/server/src/routes/votes.ts b/server/src/routes/votes.ts index 79250f091b..24f3ad5041 100644 --- a/server/src/routes/votes.ts +++ b/server/src/routes/votes.ts @@ -4,7 +4,7 @@ import { ConversationInfo, PidReadyResult, RequestWithP } from "../d"; import { failJson } from "../utils/fail"; import { getNextComment } from "../nextComment"; import { getPid } from "../user"; -import { isDuplicateKey, polisTypes } from "../utils/common"; +import { isDuplicateKey } from "../utils/common"; import logger from "../utils/logger"; import pg from "../db/pg-query"; import SQL from "../db/sql"; @@ -23,6 +23,7 @@ import { import Config from "../config"; import { SendMessageCommand } from "@aws-sdk/client-sqs"; import { sqsClient } from "../utils/sqs"; +import { wireToSemantic, wireToStorage } from "../votes/convention"; const sql_votes_latest_unique = SQL.sql_votes_latest_unique; @@ -63,13 +64,16 @@ async function doVotesPost( high_priority?: boolean ): Promise { const zid = conv?.zid; + // voteType is a wire value (range-checked by the route's parser); the row + // stores the storage convention's value. + const storageVote = wireToStorage(voteType); weight = weight || 0; const weight_x_32767 = Math.trunc(weight * 32767); // weight is stored as a SMALLINT, so convert from a [-1,1] float to [-32767,32767] int return new Promise((resolve, reject) => { const query = "INSERT INTO votes (pid, zid, tid, vote, weight_x_32767, high_priority, created) VALUES ($1, $2, $3, $4, $5, $6, default) RETURNING *;"; - const params = [pid, zid, tid, voteType, weight_x_32767, high_priority]; + const params = [pid, zid, tid, storageVote, weight_x_32767, high_priority]; pg.query(query, params, function (err: any, result: { rows: any[] }) { if (err) { @@ -246,15 +250,16 @@ async function handle_POST_votes(req: RequestWithP, res: any) { // 6. Handle moderation options if (result.shouldMod) { result.modOptions = {}; - if (req.p.vote === polisTypes.reactions.pull) { + const vote = wireToSemantic(req.p.vote, { onInvalid: "skip" }); + if (vote === "agree") { result.modOptions.as_important = true; result.modOptions.as_factual = true; result.modOptions.as_feeling = true; - } else if (req.p.vote === polisTypes.reactions.push) { + } else if (vote === "disagree") { result.modOptions.as_notmyfeeling = true; result.modOptions.as_notgoodidea = true; result.modOptions.as_abusive = true; - } else if (req.p.vote === polisTypes.reactions.pass) { + } else if (vote === "pass") { result.modOptions.as_unsure = true; result.modOptions.as_spam = true; result.modOptions.as_abusive = true; From 7d37f25134db8847c32011c09a7ae70d8af53f8d Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:53:08 -0700 Subject: [PATCH 05/16] server: route the vote read sites through the vote convention module - GET /votes, GET /votes/me and participationInit's votes return each row through storageRowToWire (wire = storage today, so the value and the JSON key order are unchanged; an out-of-range stored value is returned as stored, as before). - The moderation list counts (comment.ts) and the famous/ptptoi letter vectors (server-helpers.ts) classify the stored value by name through storageToSemantic instead of the push/pull reaction literals; NULL and unknown values keep their old treatment (counted nowhere, left 'u'). - The collective statement query builds its CASE predicates from storageSqlValue; the SQL text is the same as before. - polisTypes.reactions (push/pull/see/pass), which never said agree or disagree, is retired; nothing reads it any more. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- server/__tests__/unit/voteConvention.test.ts | 26 ++++++++++++++++++++ server/src/comment.ts | 8 +++--- server/src/routes/collectiveStatement.ts | 13 +++++++--- server/src/routes/votes.ts | 13 +++++++--- server/src/server-helpers.ts | 11 +++++---- server/src/utils/common.ts | 17 +------------ server/src/votes/convention.ts | 20 +++++++++++++++ 7 files changed, 77 insertions(+), 31 deletions(-) diff --git a/server/__tests__/unit/voteConvention.test.ts b/server/__tests__/unit/voteConvention.test.ts index 7bfc01e00f..d96584fefa 100644 --- a/server/__tests__/unit/voteConvention.test.ts +++ b/server/__tests__/unit/voteConvention.test.ts @@ -31,6 +31,7 @@ import { storageSqlValue, storageToExport, storageToSemantic, + storageRowToWire, storageToWire, wireToSemantic, wireToStorage, @@ -256,6 +257,31 @@ describe("round trips", () => { } }); +describe("storageRowToWire (the vote read routes)", () => { + test("keeps key order and every other field; maps the vote", () => { + const row = { zid: 1, pid: 2, tid: 3, vote: -1, weight_x_32767: 0 }; + const before = JSON.stringify(row); + expect(JSON.stringify(storageRowToWire(row))).toBe(before); + expect( + storageRowToWire({ pid: 2, vote: 1 }, { convention: FLIPPED }) + ).toEqual({ pid: 2, vote: -1 }); + }); + + test("NULL stays null, an out-of-range value is returned as stored", () => { + expect(storageRowToWire({ vote: null }).vote).toBeNull(); + expect(storageRowToWire({ vote: 7 }).vote).toBe(7); + expect(() => storageRowToWire({ vote: 7 }, { onInvalid: "throw" })).toThrow( + VoteConventionError + ); + }); + + test("a row without a vote field is untouched", () => { + const row = { pid: 2 }; + expect(storageRowToWire(row)).toEqual({ pid: 2 }); + expect("vote" in row).toBe(false); + }); +}); + describe("tally", () => { test("counts semantic votes and NULL separately", () => { const t = emptyTally(); diff --git a/server/src/comment.ts b/server/src/comment.ts index 5ab1b66b1f..c430574e7f 100644 --- a/server/src/comment.ts +++ b/server/src/comment.ts @@ -12,6 +12,7 @@ import Utils from "./utils/common"; import { isProConvo } from "./routes/comments"; import { UUID } from "crypto"; import logger from "./utils/logger"; +import { storageToSemantic } from "./votes/convention"; export type CommentRow = { tid: number; @@ -305,11 +306,12 @@ function _getCommentsForModerationList(o: { disagree_count: 0, pass_count: 0, }); - if (row.vote === Utils.polisTypes.reactions.pull) { + const vote = storageToSemantic(row.vote, { onInvalid: "skip" }); + if (vote === "agree") { o.agree_count = Number(row.count); - } else if (row.vote === Utils.polisTypes.reactions.push) { + } else if (vote === "disagree") { o.disagree_count = Number(row.count); - } else if (row.vote === Utils.polisTypes.reactions.pass) { + } else if (vote === "pass") { o.pass_count = Number(row.count); } } diff --git a/server/src/routes/collectiveStatement.ts b/server/src/routes/collectiveStatement.ts index a876114125..d88f7e9056 100644 --- a/server/src/routes/collectiveStatement.ts +++ b/server/src/routes/collectiveStatement.ts @@ -13,6 +13,7 @@ import Anthropic from "@anthropic-ai/sdk"; import { v4 as uuidv4 } from "uuid"; import pgQuery from "../db/pg-query"; import { getCommentIdsForCluster } from "../utils/commentClusters"; +import { storageSqlValue } from "../votes/convention"; const dynamoDBConfig: any = { region: Config.AWS_REGION || "us-east-1", @@ -578,15 +579,19 @@ async function getCommentsForTopic( return []; } - // Get full comment data with voting information + // Get full comment data with voting information. The stored values come + // from the vote convention module (integers, safe to interpolate). + const agree = storageSqlValue("agree"); + const disagree = storageSqlValue("disagree"); + const pass = storageSqlValue("pass"); const commentsQuery = ` SELECT c.tid as comment_id, c.txt as comment_text, COALESCE(COUNT(DISTINCT v.pid), 0) as total_votes, - COALESCE(SUM(CASE WHEN v.vote = 1 THEN 1 ELSE 0 END), 0) as disagrees, - COALESCE(SUM(CASE WHEN v.vote = -1 THEN 1 ELSE 0 END), 0) as agrees, - COALESCE(SUM(CASE WHEN v.vote = 0 THEN 1 ELSE 0 END), 0) as passes + COALESCE(SUM(CASE WHEN v.vote = ${disagree} THEN 1 ELSE 0 END), 0) as disagrees, + COALESCE(SUM(CASE WHEN v.vote = ${agree} THEN 1 ELSE 0 END), 0) as agrees, + COALESCE(SUM(CASE WHEN v.vote = ${pass} THEN 1 ELSE 0 END), 0) as passes FROM comments c LEFT JOIN votes_latest_unique v ON c.tid = v.tid AND c.zid = v.zid WHERE c.zid = $1 AND c.tid = ANY($2::int[]) diff --git a/server/src/routes/votes.ts b/server/src/routes/votes.ts index 24f3ad5041..be7cd05a20 100644 --- a/server/src/routes/votes.ts +++ b/server/src/routes/votes.ts @@ -23,7 +23,11 @@ import { import Config from "../config"; import { SendMessageCommand } from "@aws-sdk/client-sqs"; import { sqsClient } from "../utils/sqs"; -import { wireToSemantic, wireToStorage } from "../votes/convention"; +import { + storageRowToWire, + wireToSemantic, + wireToStorage, +} from "../votes/convention"; const sql_votes_latest_unique = SQL.sql_votes_latest_unique; @@ -157,7 +161,7 @@ async function votesGet(p: { if (err) { reject(err); } else { - resolve(results.rows); + resolve(results.rows.map((row) => storageRowToWire(row))); } } ); @@ -184,7 +188,10 @@ function handle_GET_votes_me( for (let i = 0; i < docs.rows.length; i++) { docs.rows[i].weight = docs.rows[i].weight / 32767; } - finishArray(res, docs.rows); + finishArray( + res, + docs.rows.map((row) => storageRowToWire(row)) + ); } ); }); diff --git a/server/src/server-helpers.ts b/server/src/server-helpers.ts index 73acfc0b94..a96004940f 100644 --- a/server/src/server-helpers.ts +++ b/server/src/server-helpers.ts @@ -10,7 +10,7 @@ import { getMathBundle } from "./utils/mathBundle"; import { getSocialParticipants } from "./participant"; import { getUserInfoForUid2 } from "./user"; import { getZinvite, getZinvites } from "./utils/zinvite"; -import { ifDefinedFirstElseSecond, polisTypes } from "./utils/common"; +import { ifDefinedFirstElseSecond } from "./utils/common"; import { MPromise } from "./utils/metered"; import { PcaCacheItem } from "./utils/pca"; import { sendTextEmail } from "./email/senders"; @@ -18,6 +18,7 @@ import { UserInfo } from "./d"; import Config from "./config"; import logger from "./utils/logger"; import pg from "./db/pg-query"; +import { storageToSemantic } from "./votes/convention"; // TODO consider "p2a24a2dadadu15" format const votesForZidPidCache = new LruCache({ @@ -606,12 +607,12 @@ function aggregateVotesToPidVotesObj(votes: any[]): { [key: string]: string } { // set up a vector for the participant, if not there already vectors[v.pid] = vectors[v.pid] || createEmptyVoteVector(greatestTid); // assign a vote value at that location - const vote = v.vote; - if (polisTypes.reactions.push === vote) { + const vote = storageToSemantic(v.vote, { onInvalid: "skip" }); + if (vote === "disagree") { vectors[v.pid][v.tid] = "d"; - } else if (polisTypes.reactions.pull === vote) { + } else if (vote === "agree") { vectors[v.pid][v.tid] = "a"; - } else if (polisTypes.reactions.pass === vote) { + } else if (vote === "pass") { vectors[v.pid][v.tid] = "p"; } else { logger.error("unknown vote value"); diff --git a/server/src/utils/common.ts b/server/src/utils/common.ts index 73a192c271..197b376d08 100644 --- a/server/src/utils/common.ts +++ b/server/src/utils/common.ts @@ -4,21 +4,13 @@ import pg from "../db/pg-query"; import { getConversationInfo } from "../conversation"; import Config from "../config"; +// Vote values are not here: they live in src/votes/convention.ts. type PolisTypes = { - reactions: Reactions; staractions: StarActions; mod: Mod; - reactionValues?: any; starValues?: any; }; -type Reactions = { - push: number; - pull: number; - see: number; - pass: number; -}; - type StarActions = { unstar: number; star: number; @@ -58,12 +50,6 @@ function hexToStr(hexString: string) { } const polisTypes: PolisTypes = { - reactions: { - push: 1, - pull: -1, - see: 0, - pass: 0, - }, staractions: { unstar: 0, star: 1, @@ -74,7 +60,6 @@ const polisTypes: PolisTypes = { ok: 1, }, }; -polisTypes.reactionValues = _.values(polisTypes.reactions); polisTypes.starValues = _.values(polisTypes.staractions); function isConversationOwner( diff --git a/server/src/votes/convention.ts b/server/src/votes/convention.ts index daa23810be..a46f162ae8 100644 --- a/server/src/votes/convention.ts +++ b/server/src/votes/convention.ts @@ -332,6 +332,26 @@ export function storageToExport( return semanticToExport(vote); } +/** + * A votes / votes_latest_unique row as the read routes return it (GET /votes, + * GET /votes/me, participationInit's votes): the vote in the wire convention. + * Today wire equals storage, so the value is unchanged. An out-of-range stored + * value is returned as stored, as the routes always have. The row is updated in + * place so the JSON key order does not change. + */ +export function storageRowToWire( + row: T, + options?: ConversionOptions +): T { + if (row && "vote" in row) { + row.vote = storageToWire(row.vote, { + ...options, + onInvalid: options?.onInvalid ?? "keep", + }); + } + return row; +} + /** * The integer to put in a SQL predicate that selects a stored vote, e.g. * `CASE WHEN v.vote = ${storageSqlValue("agree")} THEN 1 ...`. Always an From f605f586ef0b6b8580a06d141fb5614925f7340a Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:53:41 -0700 Subject: [PATCH 06/16] server: fail on a new vote sign literal outside the convention module A unit test reads every server source file (src/, app.ts, index.ts, bin/, scripts/) except src/votes/convention.ts and fails on a line that names a vote and carries a sign literal or a negation: -1, === 1, * -1, -row.vote, vote < 0, vote = 1 in SQL, or the retired push/pull reaction names. It also checks that the patterns catch the shapes the audit found (and do not catch group-1 or an index sentinel), and that every allowlist entry still matches, so the list can only shrink. After the routing commits the scan finds nothing and the allowlist is empty. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../__tests__/unit/voteSignLiterals.test.ts | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 server/__tests__/unit/voteSignLiterals.test.ts diff --git a/server/__tests__/unit/voteSignLiterals.test.ts b/server/__tests__/unit/voteSignLiterals.test.ts new file mode 100644 index 0000000000..e028eb797c --- /dev/null +++ b/server/__tests__/unit/voteSignLiterals.test.ts @@ -0,0 +1,145 @@ +/** + * Vote-sign lint. The vote sign is named in exactly one server file, + * src/votes/convention.ts. This test reads every other server source file and + * fails on a line that mentions a vote and also carries a sign literal or a + * negation: `-1`, `=== 1`, `* -1`, `-row.vote`, `vote < 0`, `vote = 1` in SQL, + * or the legacy `polisTypes.reactions` push/pull names. + * + * A new hit means a vote sign is being decided outside the convention module: + * route it through src/votes/convention.ts instead. A line that genuinely is not + * a vote sign goes in ALLOWLIST below with its reason. An allowlist entry that + * no longer matches anything fails too, so the list only shrinks. + */ +import { describe, expect, test } from "@jest/globals"; +import fs from "node:fs"; +import path from "node:path"; + +const SERVER_ROOT = path.resolve(__dirname, "../.."); +const CONVENTION_MODULE = "src/votes/convention.ts"; +const ROOTS = ["src", "app.ts", "index.ts", "bin", "scripts"]; + +/** A line is in scope when it names a vote. */ +const NAMES_VOTE = /vote/i; + +/** Sign literals and negations that decide a vote's meaning. */ +const SIGN_PATTERNS: Array<[string, RegExp]> = [ + ["literal -1", /(?|!=)\s*[-+]?1\b/i], + ["multiplication by a sign", /\*\s*[-+]?1\b/], + ["unary minus on a vote", /(?]=?\s*0\b/i], + ["legacy reaction names", /reactions\s*\.\s*(push|pull|see|pass)\b/], +]; + +interface AllowlistEntry { + file: string; + /** Exact substring of the offending line. */ + line: string; + reason: string; +} + +const ALLOWLIST: AllowlistEntry[] = []; + +function listSources(): string[] { + const out: string[] = []; + const walk = (rel: string) => { + const abs = path.join(SERVER_ROOT, rel); + const stat = fs.statSync(abs); + if (stat.isDirectory()) { + for (const entry of fs.readdirSync(abs).sort()) { + walk(path.posix.join(rel, entry)); + } + } else if (/\.(ts|js|cjs|mjs)$/.test(rel) && !rel.endsWith(".d.ts")) { + out.push(rel); + } + }; + for (const root of ROOTS) walk(root); + return out.filter((rel) => rel !== CONVENTION_MODULE); +} + +interface Hit { + file: string; + lineNumber: number; + line: string; + rule: string; +} + +function scan(): Hit[] { + const hits: Hit[] = []; + for (const file of listSources()) { + const lines = fs + .readFileSync(path.join(SERVER_ROOT, file), "utf8") + .split("\n"); + lines.forEach((line, i) => { + if (!NAMES_VOTE.test(line)) return; + for (const [rule, pattern] of SIGN_PATTERNS) { + if (pattern.test(line)) { + hits.push({ file, lineNumber: i + 1, line: line.trim(), rule }); + break; + } + } + }); + } + return hits; +} + +function allowedBy(hit: Hit): AllowlistEntry | undefined { + return ALLOWLIST.find( + (a) => a.file === hit.file && hit.line.includes(a.line) + ); +} + +describe("vote sign literals outside src/votes/convention.ts", () => { + const hits = scan(); + + test("the scan reads the server sources", () => { + expect(listSources().length).toBeGreaterThan(50); + expect(listSources()).toContain("app.ts"); + expect(listSources()).not.toContain(CONVENTION_MODULE); + }); + + test("the patterns catch the shapes they are meant to catch", () => { + const shapes = [ + "vote: (row) => String(-row.vote),", + "currentParticipantVotes.set(row.tid, -row.vote);", + "if (row.vote === -1) comment.agrees += 1;", + "if (vote === 1) agrees += 1;", + "COALESCE(SUM(CASE WHEN v.vote = 1 THEN 1 ELSE 0 END), 0) as disagrees,", + 'want("vote", getIntInRange(-1, 1), assignToP),', + "voteValue = -1;", + "voteValue = 1;", + "const flipped = vote * -1;", + "if (row.vote < 0) agrees++;", + "if (row.vote === Utils.polisTypes.reactions.pull) {", + ]; + for (const shape of shapes) { + expect(SIGN_PATTERNS.some(([, p]) => p.test(shape))).toBe(true); + } + const clean = [ + "const vote = storageToSemantic(row.vote);", + 'if (vote === "agree") agrees += 1;', + "group-1-agree-count", + "let currentParticipantId = -1;", + ]; + for (const line of clean) { + expect( + NAMES_VOTE.test(line) && SIGN_PATTERNS.some(([, p]) => p.test(line)) + ).toBe(false); + } + }); + + test("no vote sign literal outside the convention module (allowlist aside)", () => { + const offending = hits + .filter((hit) => !allowedBy(hit)) + .map((h) => `${h.file}:${h.lineNumber} [${h.rule}] ${h.line}`); + expect(offending).toEqual([]); + }); + + test("every allowlist entry still matches a line (the list only shrinks)", () => { + const stale = ALLOWLIST.filter( + (entry) => !hits.some((hit) => allowedBy(hit) === entry) + ).map((e) => `${e.file}: ${e.line}`); + expect(stale).toEqual([]); + }); +}); From 108a5e26223506107bb2c7bb8df9e7b09a69cb4a Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:56:01 -0700 Subject: [PATCH 07/16] server: the projection-gate witness runs the real vote convention module The witness executes the real votesGet / handle_GET_votes_me source in a vm context. Those routes now call storageRowToWire, so the witness transpiles src/votes/convention.ts as-is and exposes its exports in the same context (nothing is copied). A source tree that predates the module gets none, since its routes do not reference it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- server/scripts/projection-gate-witness.mjs | 28 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/server/scripts/projection-gate-witness.mjs b/server/scripts/projection-gate-witness.mjs index fcde3a2325..cdbce2dcfe 100644 --- a/server/scripts/projection-gate-witness.mjs +++ b/server/scripts/projection-gate-witness.mjs @@ -5,7 +5,8 @@ // `sql_votes_latest_unique` builder DEFINITION from server/src/db/sql.ts, the real // `votesGet` / `getVotesForSingleParticipant` / `handle_GET_votes` / // `handle_GET_votes_me` from server/src/routes/votes.ts, and the real -// `addConversationIds` / `finishArray` from server/src/server-helpers.ts (via the +// `addConversationIds` / `finishArray` from server/src/server-helpers.ts, and the +// vote convention module server/src/votes/convention.ts (via the // TypeScript compiler), and runs them in a `vm` wired to the installed `pg`/`sql`/ // `underscore`. So a change to the real route, serializer, OR the builder definition // changes the SERVED output. Nothing is a copy. @@ -84,8 +85,24 @@ function pickVariableInitializer(file, name) { throw new Error(`variable ${name} not found in ${file}`); } -function buildRealApp(defText, routeText, serializerText, client, servedPidForGetPid, sqlLog) { +// The real vote convention module (server/src/votes/convention.ts), which the +// read routes call to put each row's vote in the wire convention. Transpiled and +// run as-is, like the route code; a source tree that predates the module gets +// none (its routes do not reference it). +function loadConvention(srcRoot) { + const file = path.join(srcRoot, "votes", "convention.ts"); + if (!fs.existsSync(file)) return {}; + const compiled = ts.transpileModule(fs.readFileSync(file, "utf8"), { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS }, + }).outputText; + const mod = { exports: {} }; + vm.runInNewContext(compiled, { module: mod, exports: mod.exports }); + return mod.exports; +} + +function buildRealApp(defText, routeText, serializerText, client, servedPidForGetPid, sqlLog, convention) { const context = { + ...convention, _, sql, // the real builder definition runs against the real library pg: { @@ -170,10 +187,10 @@ async function frozenExpected(client, site, zid, pid, tid) { return rows; } -async function captureSite(client, site, a, defText, routeText, serializerText) { +async function captureSite(client, site, a, defText, routeText, serializerText, convention) { const pid = a.pid; const sqlLog = { last: null }; - const context = buildRealApp(defText, routeText, serializerText, client, pid === null ? 0 : pid, sqlLog); + const context = buildRealApp(defText, routeText, serializerText, client, pid === null ? 0 : pid, sqlLog, convention); let served; if (site === "votesGet") { @@ -210,6 +227,7 @@ async function main() { "addConversationIds", "finishArray", ]); + const convention = loadConvention(srcRoot); const client = new Client({ connectionString: a.dsn }); await client.connect(); @@ -218,7 +236,7 @@ async function main() { for (const site of a.sites) { await client.query("BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"); try { - out[site] = await captureSite(client, site, a, defText, routeText, serializerText); + out[site] = await captureSite(client, site, a, defText, routeText, serializerText, convention); await client.query("ROLLBACK"); } catch (e) { await client.query("ROLLBACK"); From c72038bc0412a175cc308142ee5e814bd5fc083e Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 00:59:06 -0700 Subject: [PATCH 08/16] delphi: the projection-gate source-bound test copies the vote convention module The wire witness now runs votes.ts together with the real src/votes/convention.ts, so the test's copied source tree includes it, and mutation A targets the routed read (storageRowToWire) instead of the old bare resolve(results.rows). With a local passfile DSN all eight wire tests pass; in CI they fail the same way before and after this change (the fixture's password DSN is refused by the DSN admission check). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- delphi/tests/projgate/projection_gate_test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/delphi/tests/projgate/projection_gate_test.py b/delphi/tests/projgate/projection_gate_test.py index 8133abed79..ce71cca50b 100644 --- a/delphi/tests/projgate/projection_gate_test.py +++ b/delphi/tests/projgate/projection_gate_test.py @@ -560,7 +560,7 @@ def test_wire_gate_negative_control_extra_field(dsn: str) -> None: def _copy_server_src(dst_root: str) -> str: - """Copy the two real source files the witness reads into /src.""" + """Copy the real source files the witness reads into /src.""" import shutil server_dir = pg._resolve_server_dir() @@ -569,6 +569,9 @@ def _copy_server_src(dst_root: str) -> str: src = os.path.join(dst_root, "src") os.makedirs(os.path.join(src, "routes"), exist_ok=True) os.makedirs(os.path.join(src, "db"), exist_ok=True) + os.makedirs(os.path.join(src, "votes"), exist_ok=True) + shutil.copy(os.path.join(server_dir, "src", "votes", "convention.ts"), + os.path.join(src, "votes", "convention.ts")) shutil.copy(os.path.join(server_dir, "src", "routes", "votes.ts"), os.path.join(src, "routes", "votes.ts")) shutil.copy(os.path.join(server_dir, "src", "server-helpers.ts"), @@ -606,9 +609,10 @@ def _findings(**flt): return {r.site.name: r for r in _wire_or_skip(dsn, {"zid": PUBLIC_FIXTURE_ZID, "pid": 0}, **kw)} # Mutation A: flip the served vote sign in the ACTUAL route. - assert "resolve(results.rows);" in votes_orig + served_rows = "resolve(results.rows.map((row) => storageRowToWire(row)));" + assert served_rows in votes_orig _write(votes_ts, votes_orig.replace( - "resolve(results.rows);", + served_rows, "resolve(results.rows.map((r) => ({ ...r, vote: -r.vote })));")) rA = _findings() assert not rA["votesGet"].ok From 06209519777293e956062ecac5b14a008572f115 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 02:41:43 -0700 Subject: [PATCH 09/16] server: the report exports declare their vote sign (summary.csv row, format.json) The CSV exports use agree = 1, disagree = -1, pass = 0, but no file said so. The convention module now builds one declaration from its export constants, agree=+1;disagree=-1;pass=0;format=polis-export/1 and the exports carry it without changing a CSV header or cell: - summary.csv gains one last key/value row, vote-convention,. Every row above it keeps its position and bytes. - /api/v3/reportExport/:report_id/format.json is a new sidecar beside the CSVs: the format id, the declaration, the vote values, and per file which columns carry a vote value, a count, or an importance flag. The module also reads a declaration back (the declaration string or a format.json document) and refuses one that is malformed or that names another convention, with two closed codes; the votes-bulk import uses that next. The byte pins gain the summary.csv and format.json goldens (fixture files under __tests__/fixtures/vote-declaration), and assert that summary.csv minus its last row is the pre-declaration bytes. The vote CSVs' pins are untouched. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../vote-declaration/declarations.json | 37 +++ .../fixtures/vote-declaration/format.json | 44 +++ .../fixtures/vote-declaration/summary.csv | 9 + server/__tests__/unit/exportRoutes.test.ts | 5 + .../unit/exportVoteSignBytes.test.ts | 107 +++++++ server/src/report.ts | 5 + server/src/routes/export.ts | 8 + server/src/votes/convention.ts | 274 ++++++++++++++++++ server/src/votes/exportFormat.ts | 78 +++++ 9 files changed, 567 insertions(+) create mode 100644 server/__tests__/fixtures/vote-declaration/declarations.json create mode 100644 server/__tests__/fixtures/vote-declaration/format.json create mode 100644 server/__tests__/fixtures/vote-declaration/summary.csv create mode 100644 server/src/votes/exportFormat.ts diff --git a/server/__tests__/fixtures/vote-declaration/declarations.json b/server/__tests__/fixtures/vote-declaration/declarations.json new file mode 100644 index 0000000000..4b5ad57ba8 --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/declarations.json @@ -0,0 +1,37 @@ +{ + "about": "Declaration values for the votes-bulk import and format.json (docs/export-format.md). Each case: the text after `# vote-convention:`, and the outcome the convention module must give. Generated fixture; no conversation data.", + "cases": [ + { "text": "agree=+1;disagree=-1;pass=0;format=polis-export/1", "outcome": "accepted", "format": "polis-export/1" }, + { "text": "agree=1;disagree=-1;pass=0", "outcome": "accepted", "format": null }, + { "text": " pass=0 ; disagree=-1 ; agree=+1 ", "outcome": "accepted", "format": null }, + { "text": "agree=+1;disagree=-1;pass=0;", "outcome": "accepted", "format": null }, + { "text": "agree=-1;disagree=+1;pass=0;format=polis-export/1", "outcome": "mismatch" }, + { "text": "agree=-1;disagree=1;pass=0", "outcome": "mismatch" }, + { "text": "agree=+1;disagree=-1;pass=0;format=polis-export/2", "outcome": "mismatch" }, + { "text": "agree=+1;disagree=-1;pass=0;format=other", "outcome": "mismatch" }, + { "text": "", "outcome": "malformed" }, + { "text": "agree=+1;disagree=-1", "outcome": "malformed" }, + { "text": "agree=+1;disagree=+1;pass=0", "outcome": "malformed" }, + { "text": "agree=+1;disagree=-1;pass=1", "outcome": "malformed" }, + { "text": "agree=2;disagree=-2;pass=0", "outcome": "malformed" }, + { "text": "agree=yes;disagree=no;pass=0", "outcome": "malformed" }, + { "text": "agree=+1;agree=+1;disagree=-1;pass=0", "outcome": "malformed" }, + { "text": "agree=+1;disagree=-1;pass=0;skip=0", "outcome": "malformed" }, + { "text": "agree +1, disagree -1, pass 0", "outcome": "malformed" } + ], + "documents": [ + { "doc": { "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1" }, "outcome": "accepted" }, + { "doc": { "format": "polis-export/1", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "accepted" }, + { "doc": { "format": "polis-export/1", "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "accepted" }, + { "doc": "{\"vote\": {\"agree\": 1, \"disagree\": -1, \"pass\": 0}}", "outcome": "accepted" }, + { "doc": { "vote": { "agree": -1, "disagree": 1, "pass": 0 } }, "outcome": "mismatch" }, + { "doc": { "format": "polis-export/2", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "mismatch" }, + { "doc": { "vote-convention": "agree=+1;disagree=-1;pass=0", "vote": { "agree": -1, "disagree": 1, "pass": 0 } }, "outcome": "malformed" }, + { "doc": { "format": "polis-export/1", "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/2" }, "outcome": "malformed" }, + { "doc": { "format": "polis-export/1" }, "outcome": "malformed" }, + { "doc": { "vote": [1, -1, 0] }, "outcome": "malformed" }, + { "doc": { "vote-convention": 1 }, "outcome": "malformed" }, + { "doc": "not json", "outcome": "malformed" }, + { "doc": [], "outcome": "malformed" } + ] +} diff --git a/server/__tests__/fixtures/vote-declaration/format.json b/server/__tests__/fixtures/vote-declaration/format.json new file mode 100644 index 0000000000..8f01dd5035 --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/format.json @@ -0,0 +1,44 @@ +{ + "format": "polis-export/1", + "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1", + "vote": { + "agree": 1, + "disagree": -1, + "pass": 0 + }, + "files": { + "summary.csv": { + "vote-convention": "declaration" + }, + "votes.csv": { + "vote": "vote" + }, + "participant-votes.csv": { + "": "vote", + "n-votes": "count", + "n-agree": "count", + "n-disagree": "count" + }, + "participant-importance.csv": { + "": "flag", + "n-votes": "count", + "n-important": "count" + }, + "comments.csv": { + "agrees": "count", + "disagrees": "count" + }, + "comment-groups.csv": { + "total-votes": "count", + "total-agrees": "count", + "total-disagrees": "count", + "total-passes": "count", + "group--votes": "count", + "group--agrees": "count", + "group--disagrees": "count", + "group--passes": "count" + }, + "comment-clusters.csv": {} + }, + "docs": "https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md" +} diff --git a/server/__tests__/fixtures/vote-declaration/summary.csv b/server/__tests__/fixtures/vote-declaration/summary.csv new file mode 100644 index 0000000000..e7b520a4bf --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/summary.csv @@ -0,0 +1,9 @@ +topic,"Test Topic" +url,https://example.com/test-zinvite +voters,2 +voters-in-conv,3 +commenters,10 +comments,20 +groups,1 +conversation-description,"Test Description" +vote-convention,agree=+1;disagree=-1;pass=0;format=polis-export/1 \ No newline at end of file diff --git a/server/__tests__/unit/exportRoutes.test.ts b/server/__tests__/unit/exportRoutes.test.ts index 604615945e..40cae3ae9e 100644 --- a/server/__tests__/unit/exportRoutes.test.ts +++ b/server/__tests__/unit/exportRoutes.test.ts @@ -16,6 +16,10 @@ import { getXids } from "../../src/routes/xids"; import { jest } from "@jest/globals"; import logger from "../../src/utils/logger"; import { failJson } from "../../src/utils/fail"; +import { + EXPORT_VOTE_CONVENTION, + VOTE_CONVENTION_KEY, +} from "../../src/votes/convention"; type Formatters = Record string>; @@ -202,6 +206,7 @@ describe("handle_GET_reportExport", () => { "comments,20", "groups,1", 'conversation-description,"Test Description"', + `${VOTE_CONVENTION_KEY},${EXPORT_VOTE_CONVENTION}`, ]); }); diff --git a/server/__tests__/unit/exportVoteSignBytes.test.ts b/server/__tests__/unit/exportVoteSignBytes.test.ts index afa3b66d70..4fae9fb4b6 100644 --- a/server/__tests__/unit/exportVoteSignBytes.test.ts +++ b/server/__tests__/unit/exportVoteSignBytes.test.ts @@ -4,13 +4,29 @@ * (report.ts with its hand-written `-row.vote` flips) and must not change: * agree exports as 1, disagree as -1, pass as 0, a NULL vote as 0, and an * out-of-range stored value (no CHECK constraint exists) as its negation. + * + * The declared sign (P-078 PR-E) is pinned here too: summary.csv gains exactly + * one last row, `vote-convention,...`, every byte before it unchanged, and the + * format.json sidecar is served with the bytes of the golden file in + * __tests__/fixtures/vote-declaration/. */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import { beforeEach, describe, expect, jest, test } from "@jest/globals"; import pg from "../../src/db/pg-query"; import { getPcaFromBundle } from "../../src/utils/pca"; +import { handle_GET_reportExport } from "../../src/routes/export"; +import { getZidForRid, getZinvite } from "../../src/utils/zinvite"; +import { failJson } from "../../src/utils/fail"; +import { + EXPORT_VOTE_CONVENTION, + parseVoteConventionDocument, + requireExportConvention, +} from "../../src/votes/convention"; import { formatDatetime, + sendConversationSummary, sendCommentSummary, sendParticipantImportance, sendParticipantVotesSummary, @@ -179,3 +195,94 @@ describe("vote CSV exports keep their bytes", () => { ); }); }); + +const golden = (name: string) => + readFileSync( + join(__dirname, "..", "fixtures", "vote-declaration", name), + "utf8" + ); + +// summary.csv as it was before the declaration row, for the conversation below. +const SUMMARY_BEFORE_DECLARATION = [ + 'topic,"Test Topic"', + "url,https://example.com/test-zinvite", + "voters,2", + "voters-in-conv,3", + "commenters,10", + "comments,20", + "groups,1", + 'conversation-description,"Test Description"', +].join("\n"); + +function summaryConversation() { + (getZinvite as jest.Mock).mockResolvedValue("test-zinvite" as never); + (pg.queryP_readOnly as jest.Mock) + .mockResolvedValueOnce([ + { topic: "Test Topic", description: "Test Description" }, + ] as never) + .mockResolvedValueOnce([{ count: 10 }] as never); + (getPcaFromBundle as jest.Mock).mockResolvedValue({ + asPOJO: { + "in-conv": [1, 2, 3], + "user-vote-counts": { 1: 5, 2: 3 }, + "group-clusters": { 1: { name: "Group 1" } }, + "n-cmts": 20, + }, + } as never); +} + +describe("the export files declare their vote sign", () => { + test("summary.csv: one declaration row appended, every earlier byte unchanged", async () => { + summaryConversation(); + const { res, text } = capture(); + await sendConversationSummary(7, "https://example.com", res as any); + expect(text()).toBe(golden("summary.csv")); + const lastBreak = text().lastIndexOf("\n"); + expect(text().slice(0, lastBreak)).toBe(SUMMARY_BEFORE_DECLARATION); + expect(text().slice(lastBreak + 1)).toBe( + `vote-convention,${EXPORT_VOTE_CONVENTION}` + ); + }); + + test("format.json is served beside the CSVs with the golden bytes", async () => { + (getZidForRid as jest.Mock).mockResolvedValue(7 as never); + const { res, text } = capture(); + await handle_GET_reportExport( + { + p: { rid: "r-public", report_type: "format.json" }, + headers: { host: "example.com", "x-forwarded-proto": "https" }, + }, + res as any + ); + expect(failJson).not.toHaveBeenCalled(); + expect(res.setHeader).toHaveBeenCalledWith( + "content-type", + "application/json" + ); + expect(text()).toBe(golden("format.json")); + // The sidecar is a declaration the import reader itself accepts. + expect( + requireExportConvention(parseVoteConventionDocument(text())) + ).toEqual({ agreeValue: 1, format: "polis-export/1" }); + // It and the summary row carry the same declaration. + expect(JSON.parse(text())["vote-convention"]).toBe(EXPORT_VOTE_CONVENTION); + }); + + test("format.json for an unknown report is a 404 like every other file", async () => { + (getZidForRid as jest.Mock).mockResolvedValue(null as never); + const { res, text } = capture(); + await handle_GET_reportExport( + { + p: { rid: "r-missing", report_type: "format.json" }, + headers: { host: "example.com", "x-forwarded-proto": "https" }, + }, + res as any + ); + expect(failJson).toHaveBeenCalledWith( + res, + 404, + "polis_error_data_unknown_report" + ); + expect(text()).toBe(""); + }); +}); diff --git a/server/src/report.ts b/server/src/report.ts index aef128b505..13fd794977 100644 --- a/server/src/report.ts +++ b/server/src/report.ts @@ -8,9 +8,11 @@ import { getPcaFromBundle } from "./utils/pca"; import { failJson } from "./utils/fail"; import logger from "./utils/logger"; import { + EXPORT_VOTE_CONVENTION, RawVote, storageToExport, storageToSemantic, + VOTE_CONVENTION_KEY, } from "./votes/convention"; import { getCommentsWithClusters } from "./utils/commentClusters"; import { presentPca } from "./utils/pcaPresentation"; @@ -289,6 +291,9 @@ export async function loadConversationSummary(zid: number, siteUrl: string) { : Object.keys(groupClusters).length, ], ["conversation-description", formatEscapedText(convo.description)], + // The declared vote sign of every export file (P-078 PR-E). Last, so every + // row above keeps its position and bytes. + [VOTE_CONVENTION_KEY, EXPORT_VOTE_CONVENTION], ].map((row) => row.join(",")); } diff --git a/server/src/routes/export.ts b/server/src/routes/export.ts index 209c36d8b3..5906faa8ac 100644 --- a/server/src/routes/export.ts +++ b/server/src/routes/export.ts @@ -14,6 +14,7 @@ import { sendCommentSummary, } from "../report"; import logger from "../utils/logger"; +import { exportFormatJson } from "../votes/exportFormat"; export async function handle_GET_reportExport( req: { @@ -37,6 +38,13 @@ export async function handle_GET_reportExport( break; } + // The export set's sidecar: the declared vote sign and the + // sign-bearing columns of every file above (docs/export-format.md). + case "format.json": + res.setHeader("content-type", "application/json"); + res.send(exportFormatJson()); + break; + case "comments.csv": await sendCommentSummary(zid, res); break; diff --git a/server/src/votes/convention.ts b/server/src/votes/convention.ts index a46f162ae8..9244ee3f2a 100644 --- a/server/src/votes/convention.ts +++ b/server/src/votes/convention.ts @@ -387,3 +387,277 @@ export function addToTally( else tally[vote] += n; return tally; } + +// --------------------------------------------------------------------------- +// The declared sign of the export file formats (P-078 PR-E) +// --------------------------------------------------------------------------- +// +// The CSV exports and the votes-bulk import carry the export convention above. +// They now say so: summary.csv has a `vote-convention` row, the export set has +// a format.json sidecar (exportFormat.ts), and a votes-bulk CSV may declare its +// sign on its first line or in a `format` object sent beside it. All three +// carry the same value, built here from the export constants: +// +// agree=+1;disagree=-1;pass=0;format=polis-export/1 +// +// docs/export-format.md is the prose statement of the same thing. + +/** The version of the export file formats the declaration names. */ +export const EXPORT_FORMAT_ID = "polis-export/1"; + +/** The summary.csv key, the format.json key and the import comment's name. */ +export const VOTE_CONVENTION_KEY = "vote-convention"; + +/** A votes-bulk CSV whose first line starts with this declares its sign. */ +export const IMPORT_DECLARATION_PREFIX = `# ${VOTE_CONVENTION_KEY}:`; + +function signedText(n: number): string { + return n > 0 ? `+${n}` : String(n); +} + +/** The declaration of the export convention, as written in every file that carries it. */ +export const EXPORT_VOTE_CONVENTION = [ + `agree=${signedText(EXPORT_AGREE)}`, + `disagree=${signedText(EXPORT_DISAGREE)}`, + `pass=${signedText(EXPORT_PASS)}`, + `format=${EXPORT_FORMAT_ID}`, +].join(";"); + +/** The vote values of the export convention, as format.json states them. */ +export const EXPORT_VOTE_VALUES: Readonly> = Object.freeze( + { + agree: EXPORT_AGREE, + disagree: EXPORT_DISAGREE, + pass: EXPORT_PASS, + } +); + +/** + * The closed set of reasons a declared sign is refused. A declaration that + * cannot be read is malformed; one that reads as a valid convention other than + * the export convention (agree = -1, say, or another format id) is a mismatch. + * Neither is ever converted: an import that names a sign the reader does not + * hold is refused, never guessed at. + */ +export const VOTE_DECLARATION_ERRORS = Object.freeze({ + malformed: "polis_err_vote_convention_declaration_malformed", + mismatch: "polis_err_vote_convention_declaration_mismatch", +} as const); + +export type VoteDeclarationErrorCode = + (typeof VOTE_DECLARATION_ERRORS)[keyof typeof VOTE_DECLARATION_ERRORS]; + +export class VoteDeclarationError extends Error { + readonly code: VoteDeclarationErrorCode; + readonly detail: string; + constructor(code: VoteDeclarationErrorCode, detail: string) { + super(code); + this.name = "VoteDeclarationError"; + this.code = code; + this.detail = detail; + } +} + +export interface DeclaredVoteConvention { + /** The value agree takes in the declared file. */ + readonly agreeValue: AgreeValue; + /** The format id the declaration names, or null when it names none. */ + readonly format: string | null; +} + +function malformed(detail: string): VoteDeclarationError { + return new VoteDeclarationError(VOTE_DECLARATION_ERRORS.malformed, detail); +} + +function declaredInteger(key: string, text: unknown): number { + const s = typeof text === "number" ? String(text) : text; + if (typeof s !== "string" || !/^[+-]?\d+$/.test(s.trim())) { + throw malformed(`${key} is not an integer`); + } + return parseInt(s.trim(), 10); +} + +/** + * The three declared values must form a convention: pass is 0, agree is +1 or + * -1, disagree is its negation. Anything else is malformed, not a mismatch. + */ +function declaredConvention( + values: Record, + format: unknown +): DeclaredVoteConvention { + for (const key of VOTES) { + if (!(key in values)) throw malformed(`${key} is missing`); + } + const agree = declaredInteger("agree", values.agree); + const disagree = declaredInteger("disagree", values.disagree); + const pass = declaredInteger("pass", values.pass); + if (pass !== 0 || (agree !== 1 && agree !== -1) || disagree !== -agree) { + throw malformed( + `agree=${agree};disagree=${disagree};pass=${pass} is not a vote convention` + ); + } + if (format !== undefined && format !== null && typeof format !== "string") { + throw malformed("format is not a string"); + } + return Object.freeze({ + agreeValue: agree as AgreeValue, + format: typeof format === "string" ? format.trim() : null, + }); +} + +/** + * Reads a declaration value: `agree=+1;disagree=-1;pass=0[;format=]`, keys + * in any order, whitespace around parts ignored. Unknown or repeated keys are + * malformed. + */ +export function parseVoteConventionDeclaration( + text: string +): DeclaredVoteConvention { + if (typeof text !== "string" || text.trim() === "") { + throw malformed("empty declaration"); + } + const values: Record = {}; + for (const part of text.split(";")) { + const trimmed = part.trim(); + if (trimmed === "") continue; + const eq = trimmed.indexOf("="); + if (eq <= 0) throw malformed(`"${trimmed}" is not key=value`); + const key = trimmed.slice(0, eq).trim(); + const value = trimmed.slice(eq + 1).trim(); + if (!isVote(key) && key !== "format") { + throw malformed(`unknown key "${key}"`); + } + if (key in values) throw malformed(`"${key}" is repeated`); + values[key] = value; + } + return declaredConvention(values, values.format); +} + +/** + * Reads a format.json document (the export sidecar, or the `format` object a + * votes-bulk request sends beside its CSV). It must carry the declaration + * string under `vote-convention`, the values under `vote`, or both; when both + * are present they must agree. A top-level `format` names the format id. + */ +export function parseVoteConventionDocument( + doc: unknown +): DeclaredVoteConvention { + let value = doc; + if (typeof value === "string") { + try { + value = JSON.parse(value); + } catch { + throw malformed("format is not JSON"); + } + } + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw malformed("format is not an object"); + } + const record = value as Record; + const text = record[VOTE_CONVENTION_KEY]; + const values = record.vote; + if (text === undefined && values === undefined) { + throw malformed( + `format carries neither "${VOTE_CONVENTION_KEY}" nor "vote"` + ); + } + if (text !== undefined && typeof text !== "string") { + throw malformed(`"${VOTE_CONVENTION_KEY}" is not a string`); + } + if ( + values !== undefined && + (values === null || typeof values !== "object" || Array.isArray(values)) + ) { + throw malformed(`"vote" is not an object`); + } + const fromText = + text === undefined ? null : parseVoteConventionDeclaration(text as string); + const fromValues = + values === undefined + ? null + : declaredConvention(values as Record, null); + if (fromText && fromValues && fromText.agreeValue !== fromValues.agreeValue) { + throw malformed(`"${VOTE_CONVENTION_KEY}" and "vote" disagree`); + } + if ( + fromText?.format != null && + record.format != null && + fromText.format !== record.format + ) { + throw malformed( + `"${VOTE_CONVENTION_KEY}" and "format" name different formats` + ); + } + const declared = (fromText ?? fromValues) as DeclaredVoteConvention; + const format = record.format ?? declared.format; + if (format !== null && typeof format !== "string") { + throw malformed("format is not a string"); + } + return Object.freeze({ + agreeValue: declared.agreeValue, + format: typeof format === "string" ? format.trim() : null, + }); +} + +/** + * The import rule. A declared sign must be the export convention (and, when it + * names a format, this one); otherwise it is refused with the mismatch code. + * The returned convention is the one the reader then converts from, through + * exportToStorage. + */ +export function requireExportConvention( + declared: DeclaredVoteConvention +): DeclaredVoteConvention { + if (declared.agreeValue !== EXPORT_AGREE_VALUE) { + throw new VoteDeclarationError( + VOTE_DECLARATION_ERRORS.mismatch, + `declared agree=${signedText( + declared.agreeValue + )}; the import reads ${EXPORT_VOTE_CONVENTION}` + ); + } + if (declared.format !== null && declared.format !== EXPORT_FORMAT_ID) { + throw new VoteDeclarationError( + VOTE_DECLARATION_ERRORS.mismatch, + `declared format=${declared.format}; the import reads ${EXPORT_FORMAT_ID}` + ); + } + return declared; +} + +/** + * If `line` (a CSV's first line, with or without its line ending or a byte + * order mark) is a `# vote-convention:` declaration, returns it parsed; + * otherwise null, and the line is the CSV header as it always was. + */ +export function readImportDeclarationLine( + line: string +): DeclaredVoteConvention | null { + const text = line.replace(/^\uFEFF/, "").replace(/\r?\n$/, ""); + if (!text.startsWith(IMPORT_DECLARATION_PREFIX)) return null; + return parseVoteConventionDeclaration( + text.slice(IMPORT_DECLARATION_PREFIX.length) + ); +} + +/** + * The votes-bulk rule in one call, used by the route (to refuse early) and the + * worker (which reads the stored file). `firstLine` is the CSV's first line; + * `formatDoc` the optional `format` sent beside it. Absent both: null, and the + * file is read as it always has been (export convention). Present: each must + * be the export convention, or this throws VoteDeclarationError. + */ +export function checkImportDeclaration( + firstLine: string | null, + formatDoc?: unknown +): DeclaredVoteConvention | null { + const inFile = + firstLine === null ? null : readImportDeclarationLine(firstLine); + const beside = + formatDoc === undefined || formatDoc === null || formatDoc === "" + ? null + : parseVoteConventionDocument(formatDoc); + if (inFile) requireExportConvention(inFile); + if (beside) requireExportConvention(beside); + return inFile ?? beside; +} diff --git a/server/src/votes/exportFormat.ts b/server/src/votes/exportFormat.ts new file mode 100644 index 0000000000..77fe60c645 --- /dev/null +++ b/server/src/votes/exportFormat.ts @@ -0,0 +1,78 @@ +/** + * format.json: the machine-readable declaration of the report export files. + * + * Served at /api/v3/reportExport/:report_id/format.json beside the CSVs it + * describes. It states the vote sign every vote-valued column uses (the export + * convention of convention.ts) and which columns carry a vote value, a count of + * named votes, or neither. The CSV bytes themselves are not changed; summary.csv + * carries the same declaration as its `vote-convention` row. + * + * docs/export-format.md is the prose version of this document; the unit test + * exportFormatDeclaration.test.ts keeps the two file lists the same. + */ +import { + EXPORT_FORMAT_ID, + EXPORT_VOTE_CONVENTION, + EXPORT_VOTE_VALUES, + VOTE_CONVENTION_KEY, +} from "./convention"; + +/** How a column relates to the vote sign. */ +export type ColumnKind = + /** each cell is a vote in the declared convention (an empty cell: no vote). */ + | "vote" + /** each cell counts votes of the named kind; no sign is involved. */ + | "count" + /** An importance flag (marked important or not), not a vote value. */ + | "flag" + /** the cell is the declaration itself. */ + | "declaration"; + +export const EXPORT_FORMAT_DOCS_URL = + "https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md"; + +/** The report export files and their sign-bearing columns. */ +export const EXPORT_FILES: Readonly< + Record> +> = Object.freeze({ + "summary.csv": { [VOTE_CONVENTION_KEY]: "declaration" }, + "votes.csv": { vote: "vote" }, + "participant-votes.csv": { + "": "vote", + "n-votes": "count", + "n-agree": "count", + "n-disagree": "count", + }, + "participant-importance.csv": { + "": "flag", + "n-votes": "count", + "n-important": "count", + }, + "comments.csv": { agrees: "count", disagrees: "count" }, + "comment-groups.csv": { + "total-votes": "count", + "total-agrees": "count", + "total-disagrees": "count", + "total-passes": "count", + "group--votes": "count", + "group--agrees": "count", + "group--disagrees": "count", + "group--passes": "count", + }, + "comment-clusters.csv": {}, +}); + +export function exportFormatDocument() { + return { + format: EXPORT_FORMAT_ID, + [VOTE_CONVENTION_KEY]: EXPORT_VOTE_CONVENTION, + vote: { ...EXPORT_VOTE_VALUES }, + files: EXPORT_FILES, + docs: EXPORT_FORMAT_DOCS_URL, + }; +} + +/** The exact bytes served as format.json (two-space JSON, trailing newline). */ +export function exportFormatJson(): string { + return JSON.stringify(exportFormatDocument(), null, 2) + "\n"; +} From 45389ba9ab83076e17445fb636064d793f3a2faf Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 02:41:50 -0700 Subject: [PATCH 10/16] server: the votes-bulk import accepts a declared vote sign A votes-bulk CSV may now declare its sign, on a first line before the header, # vote-convention: agree=+1;disagree=-1;pass=0;format=polis-export/1 or as a format field in the request body with format.json's shape. - Absent: the file is read in the export convention, exactly as before (the route stores the same bytes; the worker reads the same rows). - Present and the export convention: accepted. The worker drops the declaration line before the CSV header is parsed and converts every value through exportToStorage, as for an undeclared file. - Present and another sign or format: refused with polis_err_vote_convention_declaration_mismatch; unreadable: polis_err_vote_convention_declaration_malformed. The route answers 400 before anything is stored or enqueued; the worker checks the stored file's first line again and fails the job with the same code before opening a transaction. Any other first line, including another # line, still reaches the CSV parser as the header. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../import-declared-crlf-no-format.csv | 3 + .../vote-declaration/import-declared.csv | 5 + .../vote-declaration/import-malformed.csv | 5 + .../import-mismatch-format.csv | 5 + .../vote-declaration/import-mismatch-sign.csv | 5 + .../unit/importProcessorBoundaries.test.ts | 90 +++++++++- .../unit/votesBulkDeclaration.test.ts | 161 ++++++++++++++++++ server/src/routes/votes.ts | 21 +++ server/src/workers/import-processor.ts | 62 ++++++- 9 files changed, 352 insertions(+), 5 deletions(-) create mode 100644 server/__tests__/fixtures/vote-declaration/import-declared-crlf-no-format.csv create mode 100644 server/__tests__/fixtures/vote-declaration/import-declared.csv create mode 100644 server/__tests__/fixtures/vote-declaration/import-malformed.csv create mode 100644 server/__tests__/fixtures/vote-declaration/import-mismatch-format.csv create mode 100644 server/__tests__/fixtures/vote-declaration/import-mismatch-sign.csv create mode 100644 server/__tests__/unit/votesBulkDeclaration.test.ts diff --git a/server/__tests__/fixtures/vote-declaration/import-declared-crlf-no-format.csv b/server/__tests__/fixtures/vote-declaration/import-declared-crlf-no-format.csv new file mode 100644 index 0000000000..79ec93ff3c --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/import-declared-crlf-no-format.csv @@ -0,0 +1,3 @@ +# vote-convention: disagree=-1; agree=+1; pass=0 +vote_id,user_id,vote_value,timestamp,comment_id +v1,public-one,1,2024-01-01T00:00:00Z,comment-one diff --git a/server/__tests__/fixtures/vote-declaration/import-declared.csv b/server/__tests__/fixtures/vote-declaration/import-declared.csv new file mode 100644 index 0000000000..02ef0adc81 --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/import-declared.csv @@ -0,0 +1,5 @@ +# vote-convention: agree=+1;disagree=-1;pass=0;format=polis-export/1 +vote_id,user_id,vote_value,timestamp,comment_id +v1,public-one,1,2024-01-01T00:00:00Z,comment-one +v2,public-two,-1,2024-01-02T00:00:00Z,comment-one +v3,public-three,0,2024-01-03T00:00:00Z,comment-one diff --git a/server/__tests__/fixtures/vote-declaration/import-malformed.csv b/server/__tests__/fixtures/vote-declaration/import-malformed.csv new file mode 100644 index 0000000000..4f1956c24c --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/import-malformed.csv @@ -0,0 +1,5 @@ +# vote-convention: agree=+1;disagree=+1;pass=0 +vote_id,user_id,vote_value,timestamp,comment_id +v1,public-one,1,2024-01-01T00:00:00Z,comment-one +v2,public-two,-1,2024-01-02T00:00:00Z,comment-one +v3,public-three,0,2024-01-03T00:00:00Z,comment-one diff --git a/server/__tests__/fixtures/vote-declaration/import-mismatch-format.csv b/server/__tests__/fixtures/vote-declaration/import-mismatch-format.csv new file mode 100644 index 0000000000..88ac29ddf4 --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/import-mismatch-format.csv @@ -0,0 +1,5 @@ +# vote-convention: agree=+1;disagree=-1;pass=0;format=polis-export/2 +vote_id,user_id,vote_value,timestamp,comment_id +v1,public-one,1,2024-01-01T00:00:00Z,comment-one +v2,public-two,-1,2024-01-02T00:00:00Z,comment-one +v3,public-three,0,2024-01-03T00:00:00Z,comment-one diff --git a/server/__tests__/fixtures/vote-declaration/import-mismatch-sign.csv b/server/__tests__/fixtures/vote-declaration/import-mismatch-sign.csv new file mode 100644 index 0000000000..f915ede3d9 --- /dev/null +++ b/server/__tests__/fixtures/vote-declaration/import-mismatch-sign.csv @@ -0,0 +1,5 @@ +# vote-convention: agree=-1;disagree=+1;pass=0;format=polis-export/1 +vote_id,user_id,vote_value,timestamp,comment_id +v1,public-one,1,2024-01-01T00:00:00Z,comment-one +v2,public-two,-1,2024-01-02T00:00:00Z,comment-one +v3,public-three,0,2024-01-03T00:00:00Z,comment-one diff --git a/server/__tests__/unit/importProcessorBoundaries.test.ts b/server/__tests__/unit/importProcessorBoundaries.test.ts index 72496e4a1b..565f15ac03 100644 --- a/server/__tests__/unit/importProcessorBoundaries.test.ts +++ b/server/__tests__/unit/importProcessorBoundaries.test.ts @@ -32,7 +32,17 @@ jest.mock("../../src/email/senders", () => ({ sendTextEmail: jest.fn() })); import pg from "../../src/db/pg-query"; import logger from "../../src/utils/logger"; import { sendTextEmail } from "../../src/email/senders"; -import { processImportJob, s3Client } from "../../src/workers/import-processor"; +import { + processImportJob, + s3Client, + splitImportDeclaration, +} from "../../src/workers/import-processor"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + semanticToStorage, + VOTE_DECLARATION_ERRORS, +} from "../../src/votes/convention"; const query = jest.mocked(pg.queryP); const connect = jest.mocked(pg.connect); @@ -283,3 +293,81 @@ describe("CSV import transaction and lifecycle boundaries", () => { ); }); }); + +// The declared sign (P-078 PR-E). Fixture files hold the declaration lines; see +// docs/export-format.md for the rule. +const declarationFixture = (name: string) => + readFileSync( + join(__dirname, "..", "fixtures", "vote-declaration", name), + "utf8" + ); +const withoutFirstLine = (text: string) => text.slice(text.indexOf("\n") + 1); +const storedValues = () => voteInserts()[0][1][3]; + +describe("a declared vote sign on the import file", () => { + test("absent: the file is read in the export convention, as before", async () => { + csv = withoutFirstLine(declarationFixture("import-declared.csv")); + await processImportJob(payload); + expect(storedValues()).toEqual( + (["agree", "disagree", "pass"] as const).map((v) => semanticToStorage(v)) + ); + expect(jobUpdates("completed")).toHaveLength(1); + }); + + test("present and matching: the declaration line is dropped and every row stores what the undeclared file stores", async () => { + csv = withoutFirstLine(declarationFixture("import-declared.csv")); + await processImportJob(payload); + const undeclared = storedValues(); + jest.clearAllMocks(); + csv = declarationFixture("import-declared.csv"); + await processImportJob(payload); + expect(voteInserts()).toHaveLength(1); + expect(storedValues()).toEqual(undeclared); + expect(voteInserts()[0][1][1]).toEqual([0, 1, 2]); // three participants, header read + expect(jobUpdates("completed")).toHaveLength(1); + expect(jobUpdates("failed")).toHaveLength(0); + }); + + test("present with CRLF line endings and no format id: accepted", async () => { + csv = declarationFixture("import-declared-crlf-no-format.csv"); + await processImportJob(payload); + expect(storedValues()).toEqual([semanticToStorage("agree")]); + expect(jobUpdates("completed")).toHaveLength(1); + }); + + test.each([ + ["import-mismatch-sign.csv", VOTE_DECLARATION_ERRORS.mismatch], + ["import-mismatch-format.csv", VOTE_DECLARATION_ERRORS.mismatch], + ["import-malformed.csv", VOTE_DECLARATION_ERRORS.malformed], + ])("%s: refused with %s before any row is written", async (file, code) => { + csv = declarationFixture(file); + await expect(processImportJob(payload)).rejects.toThrow(code); + expect(connect).not.toHaveBeenCalled(); + expect(jobUpdates("failed")[0][1]).toEqual([11, code]); + expect(jobUpdates("completed")).toHaveLength(0); + expect(mail.mock.calls[0][3]).toContain(code); + }); + + test("the declaration is found however the object body is chunked, and no byte after it is lost", async () => { + const text = declarationFixture("import-declared.csv"); + const chunks = Array.from(text, (c) => Buffer.from(c)); + const { body, declared } = await splitImportDeclaration( + Readable.from(chunks) + ); + expect(declared).not.toBeNull(); + let rest = ""; + for await (const chunk of body) rest += Buffer.from(chunk).toString(); + expect(rest).toBe(withoutFirstLine(text)); + }); + + test("an undeclared body is replayed byte for byte", async () => { + const text = withoutFirstLine(declarationFixture("import-declared.csv")); + const { body, declared } = await splitImportDeclaration( + Readable.from([text.slice(0, 7), text.slice(7)]) + ); + expect(declared).toBeNull(); + let rest = ""; + for await (const chunk of body) rest += Buffer.from(chunk).toString(); + expect(rest).toBe(text); + }); +}); diff --git a/server/__tests__/unit/votesBulkDeclaration.test.ts b/server/__tests__/unit/votesBulkDeclaration.test.ts new file mode 100644 index 0000000000..21934a293d --- /dev/null +++ b/server/__tests__/unit/votesBulkDeclaration.test.ts @@ -0,0 +1,161 @@ +/** + * POST /api/v3/votes-bulk refuses a malformed or mismatched declared vote sign + * before anything is stored or enqueued (P-078 PR-E; docs/export-format.md). + * Absent a declaration the request goes through exactly as before. + */ +import { beforeEach, describe, expect, jest, test } from "@jest/globals"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const s3Send = jest.fn(); +const sqsSend = jest.fn(); +jest.mock("@aws-sdk/client-s3", () => ({ + S3Client: jest.fn().mockImplementation(() => ({ send: s3Send })), + PutObjectCommand: class { + constructor(public input: unknown) {} + }, +})); +jest.mock("@aws-sdk/client-sqs", () => ({ + SendMessageCommand: class { + constructor(public input: unknown) {} + }, +})); +jest.mock("../../src/utils/sqs", () => ({ + sqsClient: { send: (...a: unknown[]) => sqsSend(...a) }, +})); +jest.mock("../../src/db/pg-query", () => ({ + __esModule: true, + default: { queryP: jest.fn(), queryP_readOnly: jest.fn() }, +})); +jest.mock("../../src/utils/logger", () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + log: jest.fn(), + }, +})); +jest.mock("../../src/utils/fail", () => ({ failJson: jest.fn() })); +jest.mock("../../src/server-helpers", () => ({})); +jest.mock("../../src/nextComment", () => ({})); +jest.mock("../../src/user", () => ({})); + +import pg from "../../src/db/pg-query"; +import { failJson } from "../../src/utils/fail"; +import { handle_POST_votes_bulk } from "../../src/routes/votes"; +import { + VOTE_DECLARATION_ERRORS, + EXPORT_VOTE_CONVENTION, +} from "../../src/votes/convention"; +import { exportFormatDocument } from "../../src/votes/exportFormat"; + +const fixture = (name: string) => + readFileSync( + join(__dirname, "..", "fixtures", "vote-declaration", name), + "utf8" + ); +const declarations = JSON.parse(fixture("declarations.json")) as { + documents: { doc: unknown; outcome: string }[]; +}; +const undeclared = (text: string) => text.slice(text.indexOf("\n") + 1); + +function post(body: Record) { + const res = { json: jest.fn() }; + const req = { + p: { zid: 7, uid: 3, delphiEnabled: true }, + body, + }; + return { res, done: handle_POST_votes_bulk(req as any, res as any) }; +} + +beforeEach(() => { + jest.clearAllMocks(); + (pg.queryP_readOnly as jest.Mock).mockResolvedValue([ + { "?column?": 1 }, + ] as never); + (pg.queryP as jest.Mock).mockImplementation((async (sql: string) => + String(sql).includes("RETURNING id") + ? [{ id: 11 }] + : [{ email: "" }]) as never); + s3Send.mockResolvedValue({} as never); + sqsSend.mockResolvedValue({} as never); +}); + +function stored(): string | undefined { + const command = s3Send.mock.calls[0]?.[0] as { input: { Body: string } }; + return command?.input.Body; +} + +describe("votes-bulk declared sign", () => { + test("absent: accepted and stored byte for byte, as before", async () => { + const csv = undeclared(fixture("import-declared.csv")); + const { res, done } = post({ csv }); + await done; + expect(failJson).not.toHaveBeenCalled(); + expect(stored()).toBe(csv); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ status: "processing" }) + ); + }); + + test("present in the file and matching: accepted, stored as sent for the worker to read", async () => { + const csv = fixture("import-declared.csv"); + const { res, done } = post({ csv }); + await done; + expect(failJson).not.toHaveBeenCalled(); + expect(stored()).toBe(csv); + expect(res.json).toHaveBeenCalledTimes(1); + }); + + test("present beside the file (format.json's shape) and matching: accepted", async () => { + const csv = undeclared(fixture("import-declared.csv")); + for (const format of [ + exportFormatDocument(), + JSON.stringify(exportFormatDocument()), + { "vote-convention": EXPORT_VOTE_CONVENTION }, + ]) { + jest.clearAllMocks(); + const { res, done } = post({ csv, format }); + await done; + expect(failJson).not.toHaveBeenCalled(); + expect(res.json).toHaveBeenCalledTimes(1); + } + }); + + test.each([ + ["import-mismatch-sign.csv", VOTE_DECLARATION_ERRORS.mismatch], + ["import-mismatch-format.csv", VOTE_DECLARATION_ERRORS.mismatch], + ["import-malformed.csv", VOTE_DECLARATION_ERRORS.malformed], + ])( + "%s in the file: 400 %s, nothing stored or enqueued", + async (file, code) => { + const { res, done } = post({ csv: fixture(file) }); + await done; + expect(failJson).toHaveBeenCalledWith(res, 400, code, expect.anything()); + expect(s3Send).not.toHaveBeenCalled(); + expect(sqsSend).not.toHaveBeenCalled(); + expect(pg.queryP).not.toHaveBeenCalled(); + expect(res.json).not.toHaveBeenCalled(); + } + ); + + test.each( + declarations.documents + .filter((d) => d.outcome !== "accepted") + .map((d) => [JSON.stringify(d.doc), d.doc, d.outcome] as const) + )("format beside the file %s: refused", async (_, format, outcome) => { + const csv = undeclared(fixture("import-declared.csv")); + const { res, done } = post({ csv, format }); + await done; + expect(failJson).toHaveBeenCalledWith( + res, + 400, + outcome === "mismatch" + ? VOTE_DECLARATION_ERRORS.mismatch + : VOTE_DECLARATION_ERRORS.malformed, + expect.anything() + ); + expect(s3Send).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/routes/votes.ts b/server/src/routes/votes.ts index be7cd05a20..408b0c47cf 100644 --- a/server/src/routes/votes.ts +++ b/server/src/routes/votes.ts @@ -24,7 +24,9 @@ import Config from "../config"; import { SendMessageCommand } from "@aws-sdk/client-sqs"; import { sqsClient } from "../utils/sqs"; import { + checkImportDeclaration, storageRowToWire, + VoteDeclarationError, wireToSemantic, wireToStorage, } from "../votes/convention"; @@ -385,6 +387,25 @@ async function handle_POST_votes_bulk( return; } + // The optional declared sign: a `# vote-convention:` first line in the CSV, + // or a `format` object (format.json's shape) sent beside it. Absent both, the + // file is read in the export convention as it always has been. A malformed or + // mismatched declaration is refused here with its closed code; the worker + // checks the stored file's first line again before it reads a row. + try { + checkImportDeclaration( + typeof csv === "string" ? csv.split("\n", 1)[0] : null, + req.body.format + ); + } catch (err) { + if (err instanceof VoteDeclarationError) { + failJson(res, 400, err.code, err); + return; + } + failJson(res, 500, "polis_err_post_votes_bulk", err); + return; + } + try { // Validation Check: Ensure this conversation supports external mapping. // We check for the existence of AT LEAST ONE comment with a non-null original_id. diff --git a/server/src/workers/import-processor.ts b/server/src/workers/import-processor.ts index ae1a6daf8f..a878e36479 100644 --- a/server/src/workers/import-processor.ts +++ b/server/src/workers/import-processor.ts @@ -6,7 +6,11 @@ import pg from "../db/pg-query"; import logger from "../utils/logger"; import Config from "../config"; import { sendTextEmail } from "../email/senders"; -import { exportToStorage } from "../votes/convention"; +import { + checkImportDeclaration, + DeclaredVoteConvention, + exportToStorage, +} from "../votes/convention"; const customEndpoint = Config.AWS_S3_ENDPOINT; const config: S3ClientConfig = { @@ -61,7 +65,17 @@ export async function processImportJob(payload: { }); const response = await s3Client.send(command); if (!response.Body) throw new Error("Empty body from S3"); - const stream = response.Body as Readable; + // A `# vote-convention:` first line declares the file's sign; it is checked + // (a mismatch fails the job with the closed code) and removed before the CSV + // header is read. Without one the file is read as it always was. + const { body: stream, declared } = await splitImportDeclaration( + response.Body as Readable + ); + if (declared) { + logger.info( + `[Worker] Job ${jobId} declares agree=${declared.agreeValue}, format=${declared.format}` + ); + } let batch: any[] = []; await new Promise((resolve, reject) => { @@ -177,6 +191,46 @@ export async function processImportJob(payload: { } } +/** + * Reads the stored votes-bulk CSV up to its first line ending. If that line is + * a `# vote-convention:` declaration it is checked through the convention + * module (VoteDeclarationError on a malformed or mismatched one) and dropped; + * otherwise every byte is passed on unchanged. `body` replays the rest. + */ +export async function splitImportDeclaration(source: Readable): Promise<{ + body: Readable; + declared: DeclaredVoteConvention | null; +}> { + const iterator = source[Symbol.asyncIterator](); + let head = Buffer.alloc(0); + let newline = -1; + for (;;) { + const next = await iterator.next(); + if (next.done) break; + head = Buffer.concat([head, Buffer.from(next.value)]); + newline = head.indexOf(0x0a); + if (newline !== -1) break; + } + const firstLine = ( + newline === -1 ? head : head.subarray(0, newline + 1) + ).toString("utf8"); + const declared = checkImportDeclaration(firstLine); + const rest = declared + ? newline === -1 + ? Buffer.alloc(0) + : head.subarray(newline + 1) + : head; + async function* replay() { + if (rest.length > 0) yield rest; + for (;;) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + } + return { body: Readable.from(replay()), declared }; +} + async function markJobAsFailedInDb(jobId: number, errorMessage: string) { const query = ` UPDATE byod_import_jobs @@ -217,8 +271,8 @@ function mapRowData( if (!isNaN(parsed)) ts = parsed; } // vote_value is in the export convention (agree = +1, as the admin import - // screen documents). Values other than -1/0/1 are stored as given, as before; - // refusing them is a separate change (P-078 PR-E). + // screen documents and as a declared file must state; docs/export-format.md). + // A value outside the convention is stored as given, as before. const voteValue = exportToStorage(parseInt(row.vote_value, 10), { onInvalid: "keep", }); From d1682d960f416d861b98611af8491612feccff8d Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 02:41:59 -0700 Subject: [PATCH 11/16] docs: the export and import file formats state their vote sign docs/export-format.md is the in-repo statement of the sign of every export file and column (until now only the off-repo export page and the admin import screen said it): the export convention, the summary.csv row and format.json sidecar that declare it, each file's vote-valued and count columns, and the votes-bulk import rule with its two closed error codes. Linked from docs/configuration.md, delphi/docs/QUICK_START.md, the report page's raw data export section (which also links format.json and lists its curl line) and the admin vote import screen (which mentions the optional first-line declaration). A unit test reads the convention module's declaration cases from a fixture file and checks that the page states the declaration, the closed codes, and every file and column format.json lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../conversation-admin/BYODConfig.js | 8 + .../src/components/RawDataExport.jsx | 19 ++ delphi/docs/QUICK_START.md | 1 + docs/configuration.md | 2 + docs/export-format.md | 168 +++++++++++++++++ server/__tests__/unit/voteDeclaration.test.ts | 175 ++++++++++++++++++ 6 files changed, 373 insertions(+) create mode 100644 docs/export-format.md create mode 100644 server/__tests__/unit/voteDeclaration.test.ts diff --git a/client-admin/src/components/conversation-admin/BYODConfig.js b/client-admin/src/components/conversation-admin/BYODConfig.js index 27f4f815c5..6c5edbfd91 100644 --- a/client-admin/src/components/conversation-admin/BYODConfig.js +++ b/client-admin/src/components/conversation-admin/BYODConfig.js @@ -122,6 +122,14 @@ const BYODConfig = () => { vote_value MUST follow the pattern: 1 = agree, -1 = disagree, 0 = neutral/pass
+ Optionally, the file may declare this sign on a{' '} + # vote-convention: first line before the header; a file that declares + any other sign is refused. See{' '} + + docs/export-format.md + + . +
diff --git a/client-report/src/components/RawDataExport.jsx b/client-report/src/components/RawDataExport.jsx index 8e3c5d9310..f37810bba1 100644 --- a/client-report/src/components/RawDataExport.jsx +++ b/client-report/src/components/RawDataExport.jsx @@ -36,6 +36,11 @@ const RawDataExport = ({ conversation, report_id }) => {

{`The following data exports are anonymized. Participants are identifed by an integer representing the order in which they first voted. For a full description of files and columns, please see: `} https://compdemocracy.org/export/ + {` and, for the vote sign of each file: `} + + {" "} + docs/export-format.md{" "} +

{`----------Summary: `} @@ -93,6 +98,17 @@ const RawDataExport = ({ conversation, report_id }) => { {` (as comments x participants matrix)`}

)} +

+ {`------Format (sign): `} + + {`${conversation.conversation_id}-format.json`} + + {` (the agree / disagree / pass values every file uses; also the vote-convention row of summary.csv)`} +

{`---Comment groups: `} {

{`$ curl ${window.location.protocol}//${window.location.hostname}/api/v3/reportExport/${report_id}/comment-groups.csv`}

+

+ {`$ curl ${window.location.protocol}//${window.location.hostname}/api/v3/reportExport/${report_id}/format.json`} +

{doShowDataLicenseTerms && ( diff --git a/delphi/docs/QUICK_START.md b/delphi/docs/QUICK_START.md index a9d01ac6c9..bf2fffccd6 100644 --- a/delphi/docs/QUICK_START.md +++ b/delphi/docs/QUICK_START.md @@ -89,6 +89,7 @@ To work with your own data: 1. Prepare your data in CSV format with the following structure: - Votes: columns `voter-id`, `comment-id`, and `vote` (values: 1=agree, -1=disagree, 0=pass) + (the Polis export convention; see [docs/export-format.md](../../docs/export-format.md) for every export file's declared sign) - Comments: columns `comment-id` and `comment-body` 2. Use the Conversation class: diff --git a/docs/configuration.md b/docs/configuration.md index ad5fdf22f4..0268a50130 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,8 @@ Most application settings are environment variables. Copy the repository-root `example.env` to `.env` and modify it for your launch path; example values are not universal runtime defaults. CDK context and explicit CLI/config arguments are separate configuration inputs. +The data export and vote import file formats, including the vote sign every file declares, are described in [export and import file formats](export-format.md). + The [environment read reference](configuration-env-reference.md) records all 233 named inputs and 680 read sites in the scoped API, Delphi, coordinator and CDK source, including file:line, fallback and secret status. The [deployment reference](deployment-configuration.md) connects those settings to all 153 P065 service entries and explains dynamic/SDK input limits.
diff --git a/docs/export-format.md b/docs/export-format.md new file mode 100644 index 0000000000..c10e9485f4 --- /dev/null +++ b/docs/export-format.md @@ -0,0 +1,168 @@ +# Export and import file formats: the vote sign + +Polis exports a conversation as a set of CSV files, and admins can import votes +from a CSV. This page states the sign every vote value in those files uses, how +the files declare it, and the rule the importer applies. + +Format id: **`polis-export/1`**. + +## The export convention + +Every vote value in an export file, and in the votes-bulk import, uses: + +| vote | value | +|----------|-------| +| agree | `1` | +| disagree | `-1` | +| pass | `0` | + +This is fixed. It does not depend on how the server stores votes internally +(the database has always stored the opposite sign, agree as `-1`; the export +code converts). If the stored sign ever changes, the export files keep this +convention and these bytes. + +Written as a declaration, the convention is: + +``` +agree=+1;disagree=-1;pass=0;format=polis-export/1 +``` + +## Where the declaration lives + +CSV has no header comments, so the CSV files keep their column headers and +bytes exactly as before. The declaration is carried beside them: + +- **`summary.csv`** has a last key/value row: + + ``` + vote-convention,agree=+1;disagree=-1;pass=0;format=polis-export/1 + ``` + + Every row above it is unchanged and in the same position. Consumers that + read `summary.csv` by key are unaffected; consumers that read it by line + number see one more line at the end. + +- **`format.json`**, a sidecar served with the other files at + `/api/v3/reportExport//format.json`. It states the format id, + the declaration, the vote values, and for each file which columns carry a + vote value (`vote`), a count of named votes (`count`), an importance flag + (`flag`) or the declaration itself (`declaration`): + + ```json + { + "format": "polis-export/1", + "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1", + "vote": { "agree": 1, "disagree": -1, "pass": 0 }, + "files": { "votes.csv": { "vote": "vote" }, "...": {} }, + "docs": "https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md" + } + ``` + +The report page's "Raw Data Export" section links every file, `format.json` +included. Each file is served from +`/api/v3/reportExport//`; there is no zip bundle, so a +downloader who wants the declaration with the data fetches `format.json` (or +`summary.csv`) along with the CSVs. + +The older `/api/v3/dataExport` zip (the Clojure math worker's export) has no +producer since that worker was retired; it is not a source of these files. + +## Every file and column + +### `summary.csv` + +Key/value rows, no header: `topic`, `url`, `voters`, `voters-in-conv`, +`commenters`, `comments`, `groups`, `conversation-description`, then +`vote-convention` (the declaration above). No other row carries a vote value. + +### `votes.csv` + +One row per vote event: `timestamp`, `datetime`, `comment-id`, `voter-id`, +`vote`, and `important` when importance is enabled. + +- `vote`: a vote value in the export convention. A vote stored as NULL is + written `0`, as it always has been. + +### `participant-votes.csv` + +One row per participant: `participant`, `group-id`, `n-comments`, `n-votes`, +`n-agree`, `n-disagree`, then one column per comment. + +- `` columns: the participant's latest vote on that comment in + the export convention; empty when they did not vote on it. +- `n-votes`, `n-agree`, `n-disagree`: counts of votes; no sign involved. + +### `participant-importance.csv` + +Same layout as `participant-votes.csv` with `n-important` in place of +`n-agree` / `n-disagree`. + +- `` columns: `1` when the participant marked the comment + important, `0` when they voted without marking it, empty when they did not + vote. These are importance flags, not vote values. +- `n-votes`, `n-important`: counts. + +### `comments.csv` + +One row per comment: `timestamp`, `datetime`, `comment-id`, `author-id`, +`agrees`, `disagrees`, `moderated`, `comment-body` (and `importance` when +enabled). + +- `agrees`, `disagrees`: counts of agree and disagree votes; no sign involved. + +### `comment-groups.csv` + +One row per comment: `comment-id`, `comment`, `total-votes`, `total-agrees`, +`total-disagrees`, `total-passes`, then for each opinion group +`group--votes`, `group--agrees`, `group--disagrees`, +`group--passes`. All are counts; no sign involved. + +### `comment-clusters.csv` + +Cluster assignments per comment. No vote values. + +### `format.json` + +The sidecar described above. + +## The import rule (`POST /api/v3/votes-bulk`) + +The votes-bulk CSV (`vote_id,user_id,vote_value,timestamp,comment_id`) uses the +export convention for `vote_value`: agree `1`, disagree `-1`, pass `0`. The +importer converts each value to the stored sign through the server's vote +convention module. + +A file may declare its sign. The declaration is optional: + +- **in the file**, as its first line, before the CSV header: + + ``` + # vote-convention: agree=+1;disagree=-1;pass=0;format=polis-export/1 + ``` + + Keys may come in any order; `format` may be omitted. Any other first line + (including another `#` line) is read as the CSV header, as before. + +- **beside the file**, as a `format` field in the request body with the shape + of `format.json` (an object, or that object as a JSON string). It must carry + `vote-convention`, `vote`, or both. + +The rule: + +1. **No declaration**: the file is read in the export convention, exactly as + before this declaration existed. +2. **A declaration that states the export convention** (and, if it names a + format, `polis-export/1`): accepted; the file is read in the export + convention. A first-line declaration is removed before the header is read. +3. **A declaration that states any other convention or format** (for example + `agree=-1;disagree=+1;pass=0`): refused with + `polis_err_vote_convention_declaration_mismatch`. The importer never + converts from a declared sign other than the export convention. +4. **A declaration that cannot be read** (a missing key, an unknown or repeated + key, a value that is not an integer, values that do not form a convention, + a `format` that is not JSON): refused with + `polis_err_vote_convention_declaration_malformed`. + +These two codes are the closed set. The route refuses a bad declaration with +HTTP 400 and the code; the import worker checks the stored file's first line +again and fails the job with the same code. diff --git a/server/__tests__/unit/voteDeclaration.test.ts b/server/__tests__/unit/voteDeclaration.test.ts new file mode 100644 index 0000000000..6725bc53e1 --- /dev/null +++ b/server/__tests__/unit/voteDeclaration.test.ts @@ -0,0 +1,175 @@ +/** + * The declared vote sign of the export formats and the votes-bulk import + * (P-078 PR-E). The declaration cases live in a JSON fixture so this file + * carries no sign literal of its own. + */ +import { describe, expect, test } from "@jest/globals"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + checkImportDeclaration, + EXPORT_AGREE_VALUE, + EXPORT_FORMAT_ID, + EXPORT_VOTE_CONVENTION, + IMPORT_DECLARATION_PREFIX, + parseVoteConventionDeclaration, + parseVoteConventionDocument, + readImportDeclarationLine, + requireExportConvention, + VOTE_DECLARATION_ERRORS, + VoteDeclarationError, +} from "../../src/votes/convention"; +import { + EXPORT_FILES, + exportFormatDocument, +} from "../../src/votes/exportFormat"; + +type Outcome = "accepted" | "mismatch" | "malformed"; +const fixture = JSON.parse( + readFileSync( + join(__dirname, "..", "fixtures", "vote-declaration", "declarations.json"), + "utf8" + ) +) as { + cases: { text: string; outcome: Outcome; format?: string | null }[]; + documents: { doc: unknown; outcome: Outcome }[]; +}; + +function outcomeOf(fn: () => unknown): Outcome { + try { + fn(); + return "accepted"; + } catch (err) { + expect(err).toBeInstanceOf(VoteDeclarationError); + const code = (err as VoteDeclarationError).code; + expect(Object.values(VOTE_DECLARATION_ERRORS)).toContain(code); + expect((err as Error).message).toBe(code); + return code === VOTE_DECLARATION_ERRORS.mismatch ? "mismatch" : "malformed"; + } +} + +describe("the declaration value", () => { + test("is built from the export constants", () => { + expect(EXPORT_VOTE_CONVENTION).toBe( + `agree=+${EXPORT_AGREE_VALUE};disagree=${-EXPORT_AGREE_VALUE};pass=0;format=${EXPORT_FORMAT_ID}` + ); + expect( + requireExportConvention( + parseVoteConventionDeclaration(EXPORT_VOTE_CONVENTION) + ) + ).toEqual({ agreeValue: EXPORT_AGREE_VALUE, format: EXPORT_FORMAT_ID }); + }); + + test.each(fixture.cases)("$text → $outcome", ({ text, outcome, format }) => { + expect( + outcomeOf(() => + requireExportConvention(parseVoteConventionDeclaration(text)) + ) + ).toBe(outcome); + if (outcome === "accepted") { + expect(parseVoteConventionDeclaration(text)).toEqual({ + agreeValue: EXPORT_AGREE_VALUE, + format: format ?? null, + }); + } + }); + + test.each(fixture.documents)( + "format document → $outcome", + ({ doc, outcome }) => { + expect( + outcomeOf(() => + requireExportConvention(parseVoteConventionDocument(doc)) + ) + ).toBe(outcome); + } + ); + + test("the served format.json is itself an accepted import declaration", () => { + expect(checkImportDeclaration(null, exportFormatDocument())).toEqual({ + agreeValue: EXPORT_AGREE_VALUE, + format: EXPORT_FORMAT_ID, + }); + }); +}); + +describe("the import's first line", () => { + const declared = `${IMPORT_DECLARATION_PREFIX} ${EXPORT_VOTE_CONVENTION}`; + + test("a header line is not a declaration, and nothing is required", () => { + const header = "vote_id,user_id,vote_value,timestamp,comment_id\n"; + expect(readImportDeclarationLine(header)).toBeNull(); + expect(checkImportDeclaration(header)).toBeNull(); + expect(checkImportDeclaration(header, undefined)).toBeNull(); + expect(checkImportDeclaration(null, "")).toBeNull(); + // Another comment line is not a declaration either: it reaches the CSV + // parser as the header, exactly as it did before. + expect(readImportDeclarationLine("# exported by a script\n")).toBeNull(); + }); + + test.each([ + ["LF", `${declared}\n`], + ["CRLF", `${declared}\r\n`], + ["no line ending", declared], + ["byte order mark", `\uFEFF${declared}\n`], + ])("a declaration with %s is read", (_, line) => { + expect(checkImportDeclaration(line)).toEqual({ + agreeValue: EXPORT_AGREE_VALUE, + format: EXPORT_FORMAT_ID, + }); + }); + + test("a mismatched line is refused even when the format beside it matches", () => { + const mismatched = readFileSync( + join( + __dirname, + "..", + "fixtures", + "vote-declaration", + "import-mismatch-sign.csv" + ), + "utf8" + ).split("\n", 1)[0]; + expect( + outcomeOf(() => + checkImportDeclaration(mismatched, exportFormatDocument()) + ) + ).toBe("mismatch"); + }); + + test("a matched line is refused when the format beside it is mismatched", () => { + const mismatchedDoc = fixture.documents.find( + (d) => d.outcome === "mismatch" + )!.doc; + expect( + outcomeOf(() => checkImportDeclaration(declared, mismatchedDoc)) + ).toBe("mismatch"); + }); +}); + +describe("docs/export-format.md", () => { + const doc = readFileSync( + join(__dirname, "..", "..", "..", "docs", "export-format.md"), + "utf8" + ); + + test("states the declaration, the format id and the closed error codes", () => { + expect(doc).toContain(EXPORT_VOTE_CONVENTION); + expect(doc).toContain( + `${IMPORT_DECLARATION_PREFIX} ${EXPORT_VOTE_CONVENTION}` + ); + for (const code of Object.values(VOTE_DECLARATION_ERRORS)) { + expect(doc).toContain(code); + } + }); + + test("covers every file and every sign-bearing column format.json lists", () => { + for (const [file, columns] of Object.entries(EXPORT_FILES)) { + expect(doc).toContain(`\`${file}\``); + for (const column of Object.keys(columns)) { + expect(doc).toContain(`\`${column}\``); + } + } + }); +}); From 4257ac5bb763e6c95425d1231076d0f8699b7e25 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 02:48:32 -0700 Subject: [PATCH 12/16] server: keep the empty-math summary golden; declare the importance columns The empty-math integration test compares loadConversationSummary with rows recorded on edge. It now asserts that the last row is the vote-convention declaration and compares the rows before it with the golden, which is not re-recorded on this branch. format.json and docs/export-format.md now also list the two importance columns, neither of them a vote value: votes.csv important (a flag) and comments.csv importance (a count). Fixes the test file name in exportFormat.ts's header. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- docs/export-format.md | 8 ++++++-- .../fixtures/vote-declaration/format.json | 6 ++++-- .../integration/empty-math-comments.test.ts | 14 +++++++++++++- server/src/votes/exportFormat.ts | 6 +++--- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/export-format.md b/docs/export-format.md index c10e9485f4..c2333f758c 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -82,6 +82,8 @@ One row per vote event: `timestamp`, `datetime`, `comment-id`, `voter-id`, - `vote`: a vote value in the export convention. A vote stored as NULL is written `0`, as it always has been. +- `important` (only when importance is enabled): `1` when the vote was + marked important, `0` otherwise. An importance flag, not a vote value. ### `participant-votes.csv` @@ -105,10 +107,12 @@ Same layout as `participant-votes.csv` with `n-important` in place of ### `comments.csv` One row per comment: `timestamp`, `datetime`, `comment-id`, `author-id`, -`agrees`, `disagrees`, `moderated`, `comment-body` (and `importance` when -enabled). +`agrees`, `disagrees`, `moderated`, then `importance` when enabled, then +`comment-body`. - `agrees`, `disagrees`: counts of agree and disagree votes; no sign involved. +- `importance` (only when importance is enabled, before `comment-body`): the + number of votes on the comment marked important; no sign involved. ### `comment-groups.csv` diff --git a/server/__tests__/fixtures/vote-declaration/format.json b/server/__tests__/fixtures/vote-declaration/format.json index 8f01dd5035..1da8b0b393 100644 --- a/server/__tests__/fixtures/vote-declaration/format.json +++ b/server/__tests__/fixtures/vote-declaration/format.json @@ -11,7 +11,8 @@ "vote-convention": "declaration" }, "votes.csv": { - "vote": "vote" + "vote": "vote", + "important": "flag" }, "participant-votes.csv": { "": "vote", @@ -26,7 +27,8 @@ }, "comments.csv": { "agrees": "count", - "disagrees": "count" + "disagrees": "count", + "importance": "count" }, "comment-groups.csv": { "total-votes": "count", diff --git a/server/__tests__/integration/empty-math-comments.test.ts b/server/__tests__/integration/empty-math-comments.test.ts index 0d16db81ff..596ee80c5e 100644 --- a/server/__tests__/integration/empty-math-comments.test.ts +++ b/server/__tests__/integration/empty-math-comments.test.ts @@ -4,6 +4,10 @@ import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; import type { Agent } from "supertest"; import Config from "../../src/config"; import { loadConversationSummary } from "../../src/report"; +import { + EXPORT_VOTE_CONVENTION, + VOTE_CONVENTION_KEY, +} from "../../src/votes/convention"; import { setupAuthAndConvo, newAgent } from "../setup/api-test-helpers"; import { pool, closePool } from "../setup/db-test-helpers"; @@ -516,6 +520,14 @@ describe("zero-vote conversations serve exactly the bytes edge served", () => { `/api/v3/nextComment?conversation_id=${conversationId}&lang=en` ); const summary = await loadConversationSummary(zid, SITE_URL); + // summary.csv's last row is the declared vote sign (P-078 PR-E), added + // after the golden was recorded on edge. It is asserted here and set aside, + // so `summaryCsv` stays the edge-recorded rows the golden compares; the + // golden is not re-recorded on this branch. + expect(summary[summary.length - 1]).toBe( + `${VOTE_CONVENTION_KEY},${EXPORT_VOTE_CONVENTION}` + ); + const summaryBeforeDeclaration = summary.slice(0, -1); const servedTids: number[] = (comments.body || []).map( (c: { tid: number }) => c.tid @@ -548,7 +560,7 @@ describe("zero-vote conversations serve exactly the bytes edge served", () => { keys: next.body ? Object.keys(next.body).sort() : null, isServedComment: servedTids.includes(next.body?.tid), }, - summaryCsv: summary.map((row) => t(row)), + summaryCsv: summaryBeforeDeclaration.map((row) => t(row)), }; } diff --git a/server/src/votes/exportFormat.ts b/server/src/votes/exportFormat.ts index 77fe60c645..b1ce340617 100644 --- a/server/src/votes/exportFormat.ts +++ b/server/src/votes/exportFormat.ts @@ -8,7 +8,7 @@ * carries the same declaration as its `vote-convention` row. * * docs/export-format.md is the prose version of this document; the unit test - * exportFormatDeclaration.test.ts keeps the two file lists the same. + * __tests__/unit/voteDeclaration.test.ts keeps the two file lists the same. */ import { EXPORT_FORMAT_ID, @@ -36,7 +36,7 @@ export const EXPORT_FILES: Readonly< Record> > = Object.freeze({ "summary.csv": { [VOTE_CONVENTION_KEY]: "declaration" }, - "votes.csv": { vote: "vote" }, + "votes.csv": { vote: "vote", important: "flag" }, "participant-votes.csv": { "": "vote", "n-votes": "count", @@ -48,7 +48,7 @@ export const EXPORT_FILES: Readonly< "n-votes": "count", "n-important": "count", }, - "comments.csv": { agrees: "count", disagrees: "count" }, + "comments.csv": { agrees: "count", disagrees: "count", importance: "count" }, "comment-groups.csv": { "total-votes": "count", "total-agrees": "count", From 56ab1abe76b5c66601b62506e50555b05a45775f Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 03:51:11 -0700 Subject: [PATCH 13/16] server: the conversation data export streams a zip from the server when no S3 bucket is configured GET /api/v3/dataExport queued a generate_export_data task for the Clojure export worker, which has not consumed worker_tasks since 2022, and its results route signs a URL into the AWS polis-datadump bucket. With AWS_S3_BUCKET_NAME unset, or OFFLINE set, the route now builds the zip on request instead: format.json, summary.csv, comments.csv, votes.csv and participant-votes.csv from the report export functions, streamed entry by entry (deflate in 256 KiB sync-flushed runs, data descriptors, no zip64). A conversation over DATA_EXPORT_MAX_CELLS (max of vote rows and voters x comments; default 20,000,000) is refused with 413 polis_err_data_export_too_large before any byte. The results route answers 404 polis_err_data_export_results_not_stored in that mode. The ownership gate is unchanged. The OFFLINE config key is #2937's definition, verbatim, so the two merge cleanly. With a bucket set and OFFLINE unset nothing changes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- server/__tests__/unit/conversationZip.test.ts | 592 ++++++++++++++++++ server/src/config.ts | 25 + server/src/export/conversationZip.ts | 180 ++++++ server/src/export/zipStream.ts | 190 ++++++ server/src/routes/dataExport.ts | 72 ++- 5 files changed, 1057 insertions(+), 2 deletions(-) create mode 100644 server/__tests__/unit/conversationZip.test.ts create mode 100644 server/src/export/conversationZip.ts create mode 100644 server/src/export/zipStream.ts diff --git a/server/__tests__/unit/conversationZip.test.ts b/server/__tests__/unit/conversationZip.test.ts new file mode 100644 index 0000000000..87716db822 --- /dev/null +++ b/server/__tests__/unit/conversationZip.test.ts @@ -0,0 +1,592 @@ +/** + * The per-conversation export zip (src/export/zipStream.ts, + * src/export/conversationZip.ts) and the GET /api/v3/dataExport branch that + * streams it when no S3 bucket is configured or OFFLINE is set. + * + * Every CSV here is a generated fixture: deterministic rows built in this file, + * fed through producers that write the way the report export functions do + * (`send` once, or `write` many times across ticks and then `end`). The archive + * is read back with an independent reader below, so the writer is not checked + * against itself. + */ +import { + afterAll, + beforeEach, + describe, + expect, + jest, + test, +} from "@jest/globals"; +import { PassThrough } from "stream"; +import zlib from "zlib"; + +jest.mock("../../src/db/pg-query", () => ({ + __esModule: true, + default: { queryP_readOnly: jest.fn(), queryP: jest.fn() }, +})); +jest.mock("../../src/report", () => ({ + sendConversationSummary: jest.fn(), + sendCommentSummary: jest.fn(), + sendVotesSummary: jest.fn(), + sendParticipantVotesSummary: jest.fn(), +})); +jest.mock("../../src/utils/common", () => ({ + isModerator: jest.fn(), + doAddDataExportTask: jest.fn(), +})); +jest.mock("../../src/user", () => ({ getUserInfoForUid2: jest.fn() })); +jest.mock("../../src/utils/zinvite", () => ({ getZinvite: jest.fn() })); +jest.mock("../../src/utils/logger"); + +import Config from "../../src/config"; +import pg from "../../src/db/pg-query"; +import * as report from "../../src/report"; +import { doAddDataExportTask, isModerator } from "../../src/utils/common"; +import { getUserInfoForUid2 } from "../../src/user"; +import { getZinvite } from "../../src/utils/zinvite"; +import { exportFormatJson } from "../../src/votes/exportFormat"; +import { + CONVERSATION_ZIP_FILES, + ConversationZipFile, + ExportProducer, + dataExportStreamsLocally, + writeConversationZip, +} from "../../src/export/conversationZip"; +import { ZipStreamWriter } from "../../src/export/zipStream"; +import { + handle_GET_dataExport, + handle_GET_dataExport_results, +} from "../../src/routes/dataExport"; + +// ---------------------------------------------------------------- zip reader + +interface ReadEntry { + name: string; + data: Buffer; + flags: number; + method: number; +} + +/** Read a zip from its central directory, checking every size and CRC. */ +function readZip(buf: Buffer): ReadEntry[] { + const eocd = buf.length - 22; + expect(buf.readUInt32LE(eocd)).toBe(0x06054b50); + const count = buf.readUInt16LE(eocd + 10); + expect(buf.readUInt16LE(eocd + 8)).toBe(count); + const dirSize = buf.readUInt32LE(eocd + 12); + const dirOffset = buf.readUInt32LE(eocd + 16); + expect(dirOffset + dirSize).toBe(eocd); + + const entries: ReadEntry[] = []; + let p = dirOffset; + for (let i = 0; i < count; i++) { + expect(buf.readUInt32LE(p)).toBe(0x02014b50); + const flags = buf.readUInt16LE(p + 8); + const method = buf.readUInt16LE(p + 10); + const crc = buf.readUInt32LE(p + 16); + const csize = buf.readUInt32LE(p + 20); + const usize = buf.readUInt32LE(p + 24); + const nameLen = buf.readUInt16LE(p + 28); + const extraLen = buf.readUInt16LE(p + 30); + const commentLen = buf.readUInt16LE(p + 32); + const localOffset = buf.readUInt32LE(p + 42); + const name = buf.toString("utf8", p + 46, p + 46 + nameLen); + p += 46 + nameLen + extraLen + commentLen; + + expect(buf.readUInt32LE(localOffset)).toBe(0x04034b50); + const localNameLen = buf.readUInt16LE(localOffset + 26); + const localExtraLen = buf.readUInt16LE(localOffset + 28); + expect( + buf.toString("utf8", localOffset + 30, localOffset + 30 + localNameLen) + ).toBe(name); + const dataStart = localOffset + 30 + localNameLen + localExtraLen; + const compressed = buf.subarray(dataStart, dataStart + csize); + const data = method === 8 ? zlib.inflateRawSync(compressed) : compressed; + expect(data.length).toBe(usize); + expect(zlib.crc32(data) >>> 0).toBe(crc); + // The data descriptor after the data repeats the CRC and sizes. + const desc = dataStart + csize; + expect(buf.readUInt32LE(desc)).toBe(0x08074b50); + expect(buf.readUInt32LE(desc + 4)).toBe(crc); + expect(buf.readUInt32LE(desc + 8)).toBe(csize); + expect(buf.readUInt32LE(desc + 12)).toBe(usize); + entries.push({ name, data, flags, method }); + } + return entries; +} + +function collect(stream: PassThrough): Promise { + const chunks: Buffer[] = []; + stream.on("data", (c: Buffer) => chunks.push(c)); + return new Promise((resolve, reject) => { + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); +} + +// ---------------------------------------------------------- generated fixtures + +/** A small deterministic generator, so the fixtures are the same every run. */ +function lcg(seed: number) { + let s = seed >>> 0; + return () => { + s = (Math.imul(s, 1664525) + 1013904223) >>> 0; + return s / 0x100000000; + }; +} + +function generatedVotesCsv(rows: number, seed: number): string[] { + const rand = lcg(seed); + const lines = ["timestamp,datetime,comment-id,voter-id,vote\n"]; + for (let i = 0; i < rows; i++) { + const t = 1700000000 + Math.floor(rand() * 1e6); + const vote = [-1, 0, 1][Math.floor(rand() * 3)]; + lines.push( + `${t},${new Date(t * 1000).toString()},${Math.floor( + rand() * 50 + )},${i},${vote}\n` + ); + } + return lines; +} + +/** Writes like sendVotesSummary: header, rows over several ticks, then end(). */ +function streamingProducer(lines: string[]): ExportProducer { + return (_zid, _siteUrl, res) => { + res.setHeader("Content-Type", "text/csv"); + res.write(lines[0]); + let i = 1; + const tick = () => { + const stop = Math.min(i + 97, lines.length); + for (; i < stop; i++) res.write(lines[i]); + if (i < lines.length) setImmediate(tick); + else res.end(); + }; + setImmediate(tick); + }; +} + +/** Writes like sendCommentSummary: one send() after an await. */ +function sendingProducer(text: string): ExportProducer { + return async (_zid, _siteUrl, res) => { + await Promise.resolve(); + res.setHeader("content-type", "text/csv"); + res.send(text); + }; +} + +function generatedFixtures(): { + producers: Record; + expected: Record; +} { + const votes = generatedVotesCsv(4000, 7); + const matrix = [ + "participant,group-id,n-comments,n-votes,n-agree,n-disagree,1,2\n", + ]; + for (let pid = 0; pid < 300; pid++) { + matrix.push(`${pid},${pid % 3},0,2,1,1,1,-1\n`); + } + const summary = + 'topic,"Generated fixture: café ☕"\nurl,http://box/abc\nvoters,300\n' + + "vote-convention,agree=+1;disagree=-1;pass=0;format=polis-export/1"; + const comments = + "timestamp,datetime,comment-id,author-id,agrees,disagrees,moderated,comment-body\n" + + '1700000000,x,1,0,3,1,0,"a body, with ""quotes"" and\nnewline"\n'; + return { + producers: { + "format.json": sendingProducer(exportFormatJson()), + "summary.csv": sendingProducer(summary), + "comments.csv": sendingProducer(comments), + "votes.csv": streamingProducer(votes), + "participant-votes.csv": streamingProducer(matrix), + }, + expected: { + "format.json": exportFormatJson(), + "summary.csv": summary, + "comments.csv": comments, + "votes.csv": votes.join(""), + "participant-votes.csv": matrix.join(""), + }, + }; +} + +// ----------------------------------------------------------------- the writer + +describe("ZipStreamWriter", () => { + test("writes deflated entries with data descriptors that an independent reader accepts", async () => { + const out = new PassThrough(); + const bytes = collect(out); + const zip = new ZipStreamWriter( + out, + new Date(Date.UTC(2026, 9, 3, 12, 0, 0)) + ); + await zip.entry("a.txt", async (sink) => sink.write("hello\n")); + await zip.entry("empty.txt", async () => undefined); + await zip.entry("ünï.bin", async (sink) => + sink.write(Buffer.from([0, 1, 2, 255])) + ); + await zip.finish(); + const entries = readZip(await bytes); + expect(entries.map((e) => e.name)).toEqual([ + "a.txt", + "empty.txt", + "ünï.bin", + ]); + expect(entries[0].data.toString()).toBe("hello\n"); + expect(entries[1].data.length).toBe(0); + expect([...entries[2].data]).toEqual([0, 1, 2, 255]); + for (const e of entries) { + expect(e.method).toBe(8); + expect(e.flags & 0x0808).toBe(0x0808); + } + }); + + test("streams a multi-megabyte entry without holding it: bytes leave before the entry ends", async () => { + const out = new PassThrough(); + const bytes = collect(out); + let emittedBeforeEnd = 0; + out.on("data", (c: Buffer) => (emittedBeforeEnd += c.length)); + const zip = new ZipStreamWriter(out); + const rand = lcg(11); + const parts: string[] = []; + let sawOutputMidEntry = false; + await zip.entry("big.csv", async (sink) => { + for (let i = 0; i < 8000; i++) { + // incompressible enough that deflate has to emit as it goes + const line = + Array.from({ length: 40 }, () => Math.floor(rand() * 1e9)).join(",") + + "\n"; + parts.push(line); + sink.write(line); + if (i % 200 === 0) await new Promise((r) => setImmediate(r)); + if (i > 4000 && emittedBeforeEnd > 100_000) sawOutputMidEntry = true; + } + }); + await zip.finish(); + const [entry] = readZip(await bytes); + expect(entry.data.toString()).toBe(parts.join("")); + expect(entry.data.length).toBeGreaterThan(3_000_000); + expect(sawOutputMidEntry).toBe(true); + }); + + test("refuses entries out of sequence", async () => { + const zip = new ZipStreamWriter(new PassThrough()); + let release: () => void = () => undefined; + const first = zip.entry("a", () => new Promise((r) => (release = r))); + await expect(zip.entry("b", async () => undefined)).rejects.toThrow( + "polis_err_zip_entry_sequence" + ); + release(); + await first; + }); +}); + +// --------------------------------------------------------------- the assembly + +describe("writeConversationZip", () => { + test("the zip holds the five files, in order, each byte-identical to what its producer wrote", async () => { + const { producers, expected } = generatedFixtures(); + const out = new PassThrough(); + const bytes = collect(out); + await writeConversationZip(out, 42, "http://box", producers); + const entries = readZip(await bytes); + expect(entries.map((e) => e.name)).toEqual([...CONVERSATION_ZIP_FILES]); + expect(CONVERSATION_ZIP_FILES).toEqual([ + "format.json", + "summary.csv", + "comments.csv", + "votes.csv", + "participant-votes.csv", + ]); + for (const e of entries) { + expect( + e.data.equals( + Buffer.from(expected[e.name as ConversationZipFile], "utf8") + ) + ).toBe(true); + } + }); + + test("passes the conversation and site URL to every producer", async () => { + const { producers } = generatedFixtures(); + const seen: [number, string][] = []; + for (const name of CONVERSATION_ZIP_FILES) { + const inner = producers[name]; + producers[name] = (zid, siteUrl, res) => { + seen.push([zid, siteUrl]); + return inner(zid, siteUrl, res); + }; + } + const out = new PassThrough(); + const bytes = collect(out); + await writeConversationZip(out, 42, "https://box.local", producers); + await bytes; + expect(seen).toEqual( + CONVERSATION_ZIP_FILES.map(() => [42, "https://box.local"]) + ); + }); + + test("a producer that answers with an error status fails the export with its reason", async () => { + const { producers } = generatedFixtures(); + producers["votes.csv"] = (_zid, _siteUrl, res) => { + res.write("timestamp\n"); + res.status(500).json({ error: "polis_err_data_export" }); + }; + const out = new PassThrough(); + out.resume(); + await expect( + writeConversationZip(out, 1, "http://box", producers) + ).rejects.toThrow("polis_err_data_export (500)"); + }); + + test("a producer that rejects fails the export", async () => { + const { producers } = generatedFixtures(); + producers["participant-votes.csv"] = async () => { + throw new Error("polis_err_generated_failure"); + }; + const out = new PassThrough(); + out.resume(); + await expect( + writeConversationZip(out, 1, "http://box", producers) + ).rejects.toThrow("polis_err_generated_failure"); + }); +}); + +// ---------------------------------------------------------------- the route + +type MockRes = PassThrough & { + setHeader: jest.Mock; + status: jest.Mock; + json: jest.Mock; + redirect: jest.Mock; + headersSent: boolean; + statusCode?: number; + body?: any; +}; + +function mockRes(): MockRes { + const res = new PassThrough() as MockRes; + res.setHeader = jest.fn(); + res.json = jest.fn((body: any) => { + res.body = body; + res.end(); + return res; + }); + res.status = jest.fn((code: number) => { + res.statusCode = code; + return res; + }); + res.redirect = jest.fn(); + return res; +} + +const saved = { + bucket: Config.AWS_S3_BUCKET_NAME, + offline: Config.offline, + limit: Config.dataExportMaxCells, +}; + +function useGeneratedReport() { + const { expected } = generatedFixtures(); + (report.sendConversationSummary as jest.Mock).mockImplementation( + async (_zid: any, _siteUrl: any, res: any) => + res.send(expected["summary.csv"]) + ); + (report.sendCommentSummary as jest.Mock).mockImplementation( + async (_zid: any, res: any) => res.send(expected["comments.csv"]) + ); + (report.sendVotesSummary as jest.Mock).mockImplementation( + async (_zid: any, res: any) => { + res.write(expected["votes.csv"]); + res.end(); + } + ); + (report.sendParticipantVotesSummary as jest.Mock).mockImplementation( + async (_zid: any, res: any) => { + res.write(expected["participant-votes.csv"]); + res.end(); + } + ); + return expected; +} + +function counts(votes: number, voters: number, comments: number) { + (pg.queryP_readOnly as jest.Mock).mockResolvedValue([ + { + votes: String(votes), + voters: String(voters), + comments: String(comments), + }, + ] as never); +} + +describe("GET /api/v3/dataExport", () => { + beforeEach(() => { + jest.clearAllMocks(); + Config.AWS_S3_BUCKET_NAME = saved.bucket; + Config.offline = saved.offline; + Config.dataExportMaxCells = saved.limit; + (isModerator as jest.Mock).mockResolvedValue(true as never); + (getZinvite as jest.Mock).mockResolvedValue("abc123" as never); + (getUserInfoForUid2 as jest.Mock).mockResolvedValue({ + email: "owner@example.test", + } as never); + (doAddDataExportTask as jest.Mock).mockResolvedValue(undefined as never); + }); + + afterAll(() => { + Config.AWS_S3_BUCKET_NAME = saved.bucket; + Config.offline = saved.offline; + Config.dataExportMaxCells = saved.limit; + }); + + const req = { + p: { uid: 7, zid: 42, unixTimestamp: 1700000000, format: "csv" }, + headers: { host: "box.local", "x-forwarded-proto": "https" }, + }; + + test("serves locally when the bucket is unset or OFFLINE is set, and only then", () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.offline = false; + expect(dataExportStreamsLocally()).toBe(true); + Config.AWS_S3_BUCKET_NAME = "a-bucket"; + expect(dataExportStreamsLocally()).toBe(false); + Config.offline = true; + expect(dataExportStreamsLocally()).toBe(true); + }); + + test("with a bucket and OFFLINE unset, enqueues the worker task exactly as before", async () => { + Config.AWS_S3_BUCKET_NAME = "a-bucket"; + Config.offline = false; + const res = mockRes(); + await handle_GET_dataExport(req as any, res); + expect(doAddDataExportTask).toHaveBeenCalledTimes(1); + expect(res.json).toHaveBeenCalledWith({}); + expect(res.setHeader).not.toHaveBeenCalled(); + expect(pg.queryP_readOnly).not.toHaveBeenCalled(); + }); + + test.each([ + ["no bucket", undefined, false], + ["OFFLINE with a bucket", "a-bucket", true], + ])( + "%s: streams the zip of the report export files", + async (_label, bucket, offline) => { + Config.AWS_S3_BUCKET_NAME = bucket as any; + Config.offline = offline as boolean; + counts(4000, 300, 2); + const expected = useGeneratedReport(); + const res = mockRes(); + const bytes = collect(res); + await handle_GET_dataExport(req as any, res); + const entries = readZip(await bytes); + + expect(doAddDataExportTask).not.toHaveBeenCalled(); + expect(getUserInfoForUid2).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(res.setHeader).toHaveBeenCalledWith( + "Content-Type", + "application/zip" + ); + const disposition = (res.setHeader as jest.Mock).mock.calls.find( + ([k]) => k === "Content-Disposition" + )?.[1] as string; + expect(disposition).toMatch( + /^attachment; filename="polis-export-abc123-\d+\.zip"$/ + ); + expect(entries.map((e) => e.name)).toEqual([...CONVERSATION_ZIP_FILES]); + for (const e of entries) { + expect(e.data.toString("utf8")).toBe( + expected[e.name as ConversationZipFile] + ); + } + expect(report.sendConversationSummary).toHaveBeenCalledWith( + 42, + "https://box.local", + expect.anything() + ); + } + ); + + test("over the size limit: 413 with the closed reason, before any byte of zip", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.dataExportMaxCells = 1000; + // 10 votes but 50 voters x 40 comments = 2000 matrix cells + counts(10, 50, 40); + const res = mockRes(); + await handle_GET_dataExport(req as any, res); + expect(res.status).toHaveBeenCalledWith(413); + expect(res.body).toMatchObject({ + error: "polis_err_data_export_too_large", + cells: 2000, + limit: 1000, + }); + expect(res.setHeader).not.toHaveBeenCalled(); + expect(report.sendVotesSummary).not.toHaveBeenCalled(); + }); + + test("at the size limit exactly: streams", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.dataExportMaxCells = 2000; + counts(2000, 10, 10); + useGeneratedReport(); + const res = mockRes(); + const bytes = collect(res); + await handle_GET_dataExport(req as any, res); + expect(readZip(await bytes)).toHaveLength(5); + expect(res.status).not.toHaveBeenCalled(); + }); + + test("the ownership gate is unchanged when streaming", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + (isModerator as jest.Mock).mockResolvedValue(false as never); + const res = mockRes(); + await handle_GET_dataExport(req as any, res); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.body.error).toBe("polis_err_data_export_auth"); + expect(pg.queryP_readOnly).not.toHaveBeenCalled(); + }); + + test("a failure after the zip has started cuts the connection instead of finishing a short archive", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + counts(1, 1, 1); + useGeneratedReport(); + (report.sendVotesSummary as jest.Mock).mockImplementation( + async (_zid: any, res: any) => { + res.write("timestamp\n"); + res.status(500).json({ error: "polis_err_data_export" }); + } + ); + const res = mockRes(); + res.resume(); + let destroyed = false; + res.on("close", () => (destroyed = true)); + // PassThrough reports headersSent only through this flag + res.headersSent = true; + await handle_GET_dataExport(req as any, res); + expect(destroyed || res.destroyed).toBe(true); + expect(res.status).not.toHaveBeenCalled(); + }); +}); + +describe("GET /api/v3/dataExport/results", () => { + const req = { + p: { + uid: 7, + zid: 42, + conversation_id: "abc123", + filename: "polis-export-abc123-1758000000000.zip", + }, + }; + + test("when the export is streamed, there is no stored dump to sign: 404", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + (isModerator as jest.Mock).mockResolvedValue(true as never); + const res = mockRes(); + await handle_GET_dataExport_results(req as any, res); + expect(res.status).toHaveBeenCalledWith(404); + expect(res.body.error).toBe("polis_err_data_export_results_not_stored"); + expect(res.redirect).not.toHaveBeenCalled(); + Config.AWS_S3_BUCKET_NAME = saved.bucket; + }); +}); diff --git a/server/src/config.ts b/server/src/config.ts index e0d04959b4..deac509abf 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -126,8 +126,27 @@ export default { 10 ), mathEnv: process.env.MATH_ENV as string, + // The largest conversation GET /api/v3/dataExport streams as a zip when it + // serves the export itself (no S3 bucket, or OFFLINE). Counted as the larger + // of the vote rows (votes.csv) and voters x comments (participant-votes.csv + // cells). Over it the route answers 413 polis_err_data_export_too_large. + // See src/export/conversationZip.ts and docs/configuration.md. + dataExportMaxCells: positiveIntOr( + process.env.DATA_EXPORT_MAX_CELLS, + 20_000_000 + ), nodeEnv: process.env.NODE_ENV as string, isTesting: isTrue(process.env.TESTING), + // OFFLINE=1 (or true): the server runs on a box with no network. It skips + // the hosted services it would otherwise call: the Auth0 Management lookup + // in isProConvo, the Akismet key check, dd-trace, Google Translate and (when + // SES_ENDPOINT is unset) SES. Unset, nothing changes. See + // src/utils/offline.ts and docs/configuration.md. + // Only `1` and `true` (any case), the values the Delphi image and its + // job poller accept, so one OFFLINE value means the same on every service. + offline: ["1", "true"].includes( + (process.env.OFFLINE || "").trim().toLowerCase() + ), openaiApiKey: process.env.OPENAI_API_KEY || null, // Protected operations pages (/api/v3/ops/*, /ops in the admin console). // Off unless OPS_ENABLED=true; see docs/configuration.md. @@ -200,6 +219,12 @@ export default { }; // Use this function when a value should default to true if not set. +/** A positive integer from the environment, or the fallback when unset or not one. */ +function positiveIntOr(val: string | undefined, fallback: number): number { + const n = Number((val || "").trim()); + return Number.isSafeInteger(n) && n > 0 ? n : fallback; +} + function isTrueOrBlank(val: string | boolean | undefined): boolean { return val === undefined || val === "" || isTrue(val); } diff --git a/server/src/export/conversationZip.ts b/server/src/export/conversationZip.ts new file mode 100644 index 0000000000..6bbf8d05c3 --- /dev/null +++ b/server/src/export/conversationZip.ts @@ -0,0 +1,180 @@ +/** + * The per-conversation export zip that GET /api/v3/dataExport streams when the + * server serves the export itself. + * + * The route's original producer was the Clojure math worker, which read the + * `generate_export_data` task the route enqueues in `worker_tasks`, wrote a zip + * to the `polis-datadump` S3 bucket and mailed a link. Production stopped + * consuming those tasks in 2022, and the worker is retired. On a box with no S3 + * bucket (or with OFFLINE set) there is nowhere for a dump to go either, so the + * route builds the zip on request from the same functions that serve the report + * export (/api/v3/reportExport//): each file in the zip is the + * byte sequence that route serves for the same conversation. + * + * The zip is written entry by entry to the response as the rows stream out of + * Postgres; it is never assembled in memory. A conversation over + * `Config.dataExportMaxCells` is refused before the first byte (413). + */ +import type { Writable } from "stream"; +import Config from "../config"; +import pg from "../db/pg-query"; +import { + sendCommentSummary, + sendConversationSummary, + sendParticipantVotesSummary, + sendVotesSummary, +} from "../report"; +import { exportFormatJson } from "../votes/exportFormat"; +import { ZipEntrySink, ZipStreamWriter } from "./zipStream"; + +/** The files in the zip, in order (docs/export-format.md). */ +export const CONVERSATION_ZIP_FILES = [ + "format.json", + "summary.csv", + "comments.csv", + "votes.csv", + "participant-votes.csv", +] as const; + +export type ConversationZipFile = (typeof CONVERSATION_ZIP_FILES)[number]; + +export const DATA_EXPORT_TOO_LARGE = "polis_err_data_export_too_large"; + +/** + * The response surface the report export functions write to (report.ts): + * `send` writes everything at once, `write`/`end` stream, and an error goes + * out through failJson as `status(code).json(body)`. + */ +export interface ExportResponse { + setHeader: (key: string, value: string) => void; + send: (data: string) => void; + write: (data: string) => void; + end: () => void; + status: (code: number) => { json: (body: unknown) => void }; +} + +export type ExportProducer = ( + zid: number, + siteUrl: string, + res: ExportResponse +) => unknown; + +/** The report export's producer for each file, as handle_GET_reportExport calls it. */ +export const REPORT_EXPORT_PRODUCERS: Record< + ConversationZipFile, + ExportProducer +> = { + "format.json": (_zid, _siteUrl, res) => { + res.setHeader("content-type", "application/json"); + res.send(exportFormatJson()); + }, + "summary.csv": (zid, siteUrl, res) => + sendConversationSummary(zid, siteUrl, res), + "comments.csv": (zid, _siteUrl, res) => sendCommentSummary(zid, res), + "votes.csv": (zid, _siteUrl, res) => sendVotesSummary(zid, res), + "participant-votes.csv": (zid, _siteUrl, res) => + sendParticipantVotesSummary(zid, res), +}; + +/** + * True when the route serves the export itself: no S3 bucket is configured, + * or OFFLINE is set. Read at request time. + */ +export function dataExportStreamsLocally(): boolean { + return Boolean(Config.offline) || !Config.AWS_S3_BUCKET_NAME; +} + +/** + * Run one report export producer into a zip entry. Resolves when the producer + * ends its response; rejects if it answers with an error status or throws. + */ +export function runProducerIntoEntry( + produce: ExportProducer, + zid: number, + siteUrl: string, + sink: ZipEntrySink +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const settle = (err?: unknown) => { + if (settled) return; + settled = true; + if (err) reject(err); + else resolve(); + }; + const write = (data: string) => { + if (settled) return; + try { + sink.write(data); + } catch (err) { + // The producers write from inside Postgres stream callbacks, where a + // throw would escape; fail the entry instead. + settle(err); + } + }; + const res: ExportResponse = { + setHeader: () => undefined, + write, + send: (data) => { + write(data); + settle(); + }, + end: () => settle(), + status: (code) => ({ + json: (body) => { + const reason = + (body as { error?: string } | undefined)?.error || + "polis_err_data_export"; + settle(new Error(`${reason} (${code})`)); + }, + }), + }; + Promise.resolve() + .then(() => produce(zid, siteUrl, res)) + .catch((err) => settle(err)); + }); +} + +/** Write every file of the conversation's export into `out` as a zip. */ +export async function writeConversationZip( + out: Writable, + zid: number, + siteUrl: string, + producers: Record< + ConversationZipFile, + ExportProducer + > = REPORT_EXPORT_PRODUCERS, + modified: Date = new Date() +): Promise { + const zip = new ZipStreamWriter(out, modified); + for (const name of CONVERSATION_ZIP_FILES) { + await zip.entry(name, (sink) => + runProducerIntoEntry(producers[name], zid, siteUrl, sink) + ); + } + await zip.finish(); +} + +export interface ConversationExportSize { + votes: number; + voters: number; + comments: number; + /** max(votes, voters x comments): the vote rows or matrix cells to write. */ + cells: number; +} + +/** Count what the export would write, before writing any of it. */ +export async function conversationExportSize( + zid: number +): Promise { + const rows = (await pg.queryP_readOnly( + `SELECT (SELECT COUNT(*) FROM votes WHERE zid = $1) AS votes, + (SELECT COUNT(DISTINCT pid) FROM votes WHERE zid = $1) AS voters, + (SELECT COUNT(*) FROM comments WHERE zid = $1) AS comments`, + [zid] + )) as { votes: string; voters: string; comments: string }[]; + const votes = Number(rows[0]?.votes || 0); + const voters = Number(rows[0]?.voters || 0); + const comments = Number(rows[0]?.comments || 0); + return { votes, voters, comments, cells: Math.max(votes, voters * comments) }; +} diff --git a/server/src/export/zipStream.ts b/server/src/export/zipStream.ts new file mode 100644 index 0000000000..022148ffeb --- /dev/null +++ b/server/src/export/zipStream.ts @@ -0,0 +1,190 @@ +/** + * A minimal streaming zip writer (PKWARE APPNOTE 6.3, no zip64). + * + * Entries are written one after another straight to `out`: a local header, + * the deflated bytes as they are produced, then a data descriptor carrying the + * CRC-32 and sizes (general purpose flag bit 3), so nothing about an entry has + * to be known before its first byte goes out. Input is compressed + * synchronously in runs of BLOCK_BYTES, each ending in a sync flush, so the + * writer holds at most one run of input; the central directory (about 60 bytes + * per entry) is kept until `finish()`. Like the report export's CSV routes, it + * does not wait for `out` to drain. + * + * Without zip64 an archive, and every entry in it, must stay under 4 GiB; the + * writer fails rather than emit a corrupt archive past that. Callers bound the + * input first (see conversationZip.ts). + */ +import zlib from "zlib"; +import type { Writable } from "stream"; + +const ZIP32_MAX = 0xffffffff; +// bit 3: sizes and CRC follow the data; bit 11: the name is UTF-8. +const FLAGS = 0x0808; +const METHOD_DEFLATE = 8; +const VERSION_NEEDED = 20; +// Made by: Unix (3), spec 2.0, so the external attributes carry a file mode. +const VERSION_MADE_BY = (3 << 8) | 20; +const FILE_MODE = 0o100644; +// Input is compressed in runs of about this many bytes; at most one run is held. +const BLOCK_BYTES = 256 * 1024; +const SYNC_FLUSH = zlib.constants.Z_SYNC_FLUSH; +// A final, empty fixed-Huffman block: ends the deflate stream. +const FINAL_BLOCK = zlib.deflateRawSync(Buffer.alloc(0)); + +export const ZIP_TOO_LARGE = "polis_err_zip_too_large"; + +/** Where an entry's producer writes its uncompressed bytes. */ +export interface ZipEntrySink { + write(data: string | Buffer): void; +} + +function dosDateTime(date: Date): { time: number; date: number } { + const year = Math.max(date.getUTCFullYear(), 1980); + return { + time: + (date.getUTCHours() << 11) | + (date.getUTCMinutes() << 5) | + (date.getUTCSeconds() >> 1), + date: + ((year - 1980) << 9) | + ((date.getUTCMonth() + 1) << 5) | + date.getUTCDate(), + }; +} + +export class ZipStreamWriter { + private offset = 0; + private readonly central: Buffer[] = []; + private busy = false; + private finished = false; + private readonly stamp: { time: number; date: number }; + + constructor(private readonly out: Writable, modified: Date = new Date()) { + this.stamp = dosDateTime(modified); + } + + private emit(buf: Buffer) { + if (this.offset + buf.length > ZIP32_MAX) { + throw new Error(ZIP_TOO_LARGE); + } + this.offset += buf.length; + this.out.write(buf); + } + + /** + * Add one entry. `produce` writes the entry's bytes to the sink and resolves + * when it has written them all; entries are written strictly in sequence. + */ + async entry( + name: string, + produce: (sink: ZipEntrySink) => Promise + ): Promise { + if (this.busy || this.finished) { + throw new Error("polis_err_zip_entry_sequence"); + } + if (this.central.length >= 0xffff) { + throw new Error(ZIP_TOO_LARGE); + } + this.busy = true; + const nameBuf = Buffer.from(name, "utf8"); + const headerOffset = this.offset; + + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(VERSION_NEEDED, 4); + local.writeUInt16LE(FLAGS, 6); + local.writeUInt16LE(METHOD_DEFLATE, 8); + local.writeUInt16LE(this.stamp.time, 10); + local.writeUInt16LE(this.stamp.date, 12); + // CRC-32 and both sizes stay zero here; the data descriptor carries them. + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); + this.emit(local); + this.emit(nameBuf); + + let crc = 0; + let size = 0; + let compressedSize = 0; + let pending: Buffer[] = []; + let pendingBytes = 0; + const out = (buf: Buffer) => { + compressedSize += buf.length; + this.emit(buf); + }; + // Compress what is pending as one deflate block run ending in a sync flush + // (never a final block), synchronously, and send it on. Concatenated, these + // runs plus one final empty block are a single valid deflate stream. + const flush = () => { + if (pendingBytes === 0) return; + const block = Buffer.concat(pending, pendingBytes); + pending = []; + pendingBytes = 0; + out(zlib.deflateRawSync(block, { finishFlush: SYNC_FLUSH })); + }; + + const sink: ZipEntrySink = { + write: (data) => { + const buf = typeof data === "string" ? Buffer.from(data, "utf8") : data; + if (buf.length === 0) return; + if (size + buf.length > ZIP32_MAX) { + throw new Error(ZIP_TOO_LARGE); + } + crc = zlib.crc32(buf, crc); + size += buf.length; + pending.push(buf); + pendingBytes += buf.length; + if (pendingBytes >= BLOCK_BYTES) flush(); + }, + }; + + await produce(sink); + flush(); + out(FINAL_BLOCK); + + const descriptor = Buffer.alloc(16); + descriptor.writeUInt32LE(0x08074b50, 0); + descriptor.writeUInt32LE(crc >>> 0, 4); + descriptor.writeUInt32LE(compressedSize, 8); + descriptor.writeUInt32LE(size, 12); + this.emit(descriptor); + + const record = Buffer.alloc(46); + record.writeUInt32LE(0x02014b50, 0); + record.writeUInt16LE(VERSION_MADE_BY, 4); + record.writeUInt16LE(VERSION_NEEDED, 6); + record.writeUInt16LE(FLAGS, 8); + record.writeUInt16LE(METHOD_DEFLATE, 10); + record.writeUInt16LE(this.stamp.time, 12); + record.writeUInt16LE(this.stamp.date, 14); + record.writeUInt32LE(crc >>> 0, 16); + record.writeUInt32LE(compressedSize, 20); + record.writeUInt32LE(size, 24); + record.writeUInt16LE(nameBuf.length, 28); + // extra length, comment length, disk number, internal attributes: zero + record.writeUInt32LE((FILE_MODE << 16) >>> 0, 38); + record.writeUInt32LE(headerOffset, 42); + this.central.push(Buffer.concat([record, nameBuf])); + this.busy = false; + } + + /** Write the central directory and end-of-central-directory record, then end `out`. */ + async finish(): Promise { + if (this.busy || this.finished) { + throw new Error("polis_err_zip_entry_sequence"); + } + this.finished = true; + const directoryOffset = this.offset; + for (const record of this.central) { + this.emit(record); + } + const directorySize = this.offset - directoryOffset; + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(this.central.length, 8); + end.writeUInt16LE(this.central.length, 10); + end.writeUInt32LE(directorySize, 12); + end.writeUInt32LE(directoryOffset, 16); + this.emit(end); + await new Promise((resolve) => this.out.end(resolve)); + } +} diff --git a/server/src/routes/dataExport.ts b/server/src/routes/dataExport.ts index 4a2643961d..eb6f83eb5c 100644 --- a/server/src/routes/dataExport.ts +++ b/server/src/routes/dataExport.ts @@ -5,13 +5,69 @@ import Config from "../config"; import { failJson } from "../utils/fail"; import AWS from "aws-sdk"; import { UserInfo } from "../d"; +import logger from "../utils/logger"; +import { + conversationExportSize, + DATA_EXPORT_TOO_LARGE, + dataExportStreamsLocally, + writeConversationZip, +} from "../export/conversationZip"; AWS.config.update({ region: Config.awsRegion }); const s3Client = new AWS.S3({ apiVersion: "2006-03-01" }); +/** + * The export served by this server when there is no S3 bucket for a dump to + * go to (or OFFLINE is set): the conversation's report export files, streamed + * as one zip in this response. See src/export/conversationZip.ts. + */ +async function streamConversationExport( + req: { headers?: Record }, + res: any, + zid: number +) { + let size: Awaited>; + let zinvite: string | undefined; + try { + size = await conversationExportSize(zid); + zinvite = (await getZinvite(zid)) as string | undefined; + } catch (err) { + return failJson(res, 500, "polis_err_data_export_size", err); + } + if (size.cells > Config.dataExportMaxCells) { + return failJson(res, 413, DATA_EXPORT_TOO_LARGE, undefined, { + cells: size.cells, + limit: Config.dataExportMaxCells, + }); + } + + // Same site URL the report export puts in summary.csv. + const headers = req.headers || {}; + const siteUrl = `${headers["x-forwarded-proto"]}://${headers.host}`; + // The name the S3 worker gave its dumps: polis-export--.zip + const name = `polis-export-${zinvite || zid}-${Date.now()}.zip`; + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", `attachment; filename="${name}"`); + res.setHeader("Cache-Control", "no-store"); + try { + await writeConversationZip(res, zid, siteUrl); + } catch (err) { + if (!res.headersSent) { + return failJson(res, 500, "polis_err_data_export_stream", err); + } + // Part of a zip is already out. Cut the connection so the client sees a + // failed download, not a short archive that looks complete. + logger.error("polis_err_data_export_stream", err); + res.destroy?.(); + } +} + async function handle_GET_dataExport( - req: { p: { uid?: number; zid: number; unixTimestamp: number; format: any } }, - res: { json: (arg0: {}) => void } + req: { + p: { uid?: number; zid: number; unixTimestamp: number; format: any }; + headers?: Record; + }, + res: any ) { const { uid, zid } = req.p; @@ -35,6 +91,12 @@ async function handle_GET_dataExport( return failJson(res, 403, "polis_err_data_export_auth"); } + // No S3 bucket (or OFFLINE): nothing would ever produce or hold the dump, + // so build the zip now and send it in this response. + if (dataExportStreamsLocally()) { + return await streamConversationExport(req, res, zid); + } + let user: UserInfo; try { user = await getUserInfoForUid2(uid); @@ -108,6 +170,12 @@ async function handle_GET_dataExport_results( return failJson(res, 403, "polis_err_data_export_results_auth"); } + // The zip is streamed by GET /api/v3/dataExport itself here; no dump is + // stored, and a signed URL would point at a bucket that is not there. + if (dataExportStreamsLocally()) { + return failJson(res, 404, "polis_err_data_export_results_not_stored"); + } + if (!filename || typeof filename !== "string") { return failJson( res, From 66f2ae876546e4821cce8827b1796c2e87f73e74 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 03:51:11 -0700 Subject: [PATCH 14/16] server: integration case: the streamed export zip is byte-identical to the report export Generated fixture conversation written straight to Postgres; both handlers mounted on a bare Express app; every zip entry compared with the bytes /api/v3/reportExport serves. Also the OFFLINE-with-bucket mode, the 413 refusal and the unchanged ownership refusal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .../integration/data-export-stream.test.ts | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 server/__tests__/integration/data-export-stream.test.ts diff --git a/server/__tests__/integration/data-export-stream.test.ts b/server/__tests__/integration/data-export-stream.test.ts new file mode 100644 index 0000000000..47b9d5ad45 --- /dev/null +++ b/server/__tests__/integration/data-export-stream.test.ts @@ -0,0 +1,263 @@ +/** + * GET /api/v3/dataExport, served by this server (no S3 bucket configured, or + * OFFLINE set): the zip it streams holds the report export's files, and each + * one is byte-identical to what /api/v3/reportExport// serves + * for the same conversation. + * + * Needs only Postgres. The conversation is a generated fixture written straight + * into the database (owner, participants, comments with commas, quotes, + * newlines and non-ASCII text, a vote history including changed and NULL + * votes). Both handlers are mounted on a bare Express app so the comparison is + * of the bytes each one puts on the wire; the routing and auth middleware in + * app.ts are covered by data-export-authz.test.ts and the unit tests. + */ +import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; +import express from "express"; +import request from "supertest"; +import zlib from "zlib"; +import Config from "../../src/config"; +import { handle_GET_dataExport } from "../../src/routes/dataExport"; +import { handle_GET_reportExport } from "../../src/routes/export"; +import { CONVERSATION_ZIP_FILES } from "../../src/export/conversationZip"; +import { pool } from "../setup/db-test-helpers"; + +/** Entries of a zip, read from its central directory. */ +function unzip(buf: Buffer): Map { + const eocd = buf.length - 22; + expect(buf.readUInt32LE(eocd)).toBe(0x06054b50); + const count = buf.readUInt16LE(eocd + 10); + let p = buf.readUInt32LE(eocd + 16); + const out = new Map(); + for (let i = 0; i < count; i++) { + expect(buf.readUInt32LE(p)).toBe(0x02014b50); + const crc = buf.readUInt32LE(p + 16); + const csize = buf.readUInt32LE(p + 20); + const nameLen = buf.readUInt16LE(p + 28); + const skip = nameLen + buf.readUInt16LE(p + 30) + buf.readUInt16LE(p + 32); + const local = buf.readUInt32LE(p + 42); + const name = buf.toString("utf8", p + 46, p + 46 + nameLen); + const start = + local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28); + const data = zlib.inflateRawSync(buf.subarray(start, start + csize)); + expect(zlib.crc32(data) >>> 0).toBe(crc); + out.set(name, data); + p += 46 + skip; + } + return out; +} + +function binary(res: any, cb: (err: Error | null, body: Buffer) => void) { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => cb(null, Buffer.concat(chunks))); +} + +describe("dataExport streamed from the server", () => { + const tag = `p053x${Date.now().toString(36)}`; + const saved = { + bucket: Config.AWS_S3_BUCKET_NAME, + offline: Config.offline, + limit: Config.dataExportMaxCells, + }; + let ownerUid: number; + let zid: number; + let rid: number; + let voteRows = 0; + let app: any; + + beforeAll(async () => { + const owner = await pool.query( + "INSERT INTO users (hname, email, is_owner) VALUES ($1, $2, true) RETURNING uid", + ["Generated fixture owner", `${tag}@example.test`] + ); + ownerUid = owner.rows[0].uid; + const convo = await pool.query( + "INSERT INTO conversations (topic, description, owner) VALUES ($1, $2, $3) RETURNING zid", + [`Generated fixture, "quoted", café`, "line one\nline two", ownerUid] + ); + zid = convo.rows[0].zid; + await pool.query("INSERT INTO zinvites (zid, zinvite) VALUES ($1, $2)", [ + zid, + tag, + ]); + const report = await pool.query( + "INSERT INTO reports (report_id, zid) VALUES ($1, $2) RETURNING rid", + [`r${tag}`, zid] + ); + rid = report.rows[0].rid; + + const pids: number[] = []; + const uids: number[] = [ownerUid]; + for (let i = 0; i < 9; i++) { + const u = await pool.query( + "INSERT INTO users (hname) VALUES ($1) RETURNING uid", + [`Generated fixture participant ${i}`] + ); + uids.push(u.rows[0].uid); + } + for (const uid of uids) { + const p = await pool.query( + "INSERT INTO participants (pid, zid, uid) VALUES (NULL, $1, $2) RETURNING pid", + [zid, uid] + ); + pids.push(p.rows[0].pid); + } + + const bodies = [ + "plain statement", + 'has a comma, and "quotes"', + "has a\nnewline", + "non-ASCII: naïve façade ☕ 日本語", + "", + "trailing space ", + ]; + const tids: number[] = []; + for (let i = 0; i < bodies.length; i++) { + const author = i % 3; + const c = await pool.query( + "INSERT INTO comments (tid, zid, pid, uid, txt, created, velocity, mod) VALUES (NULL, $1, $2, $3, $4, $5, $6, $7) RETURNING tid", + [ + zid, + pids[author], + uids[author], + bodies[i] || "(empty)", + 1700000000000 + i * 1000, + 1 - i / 10, + [0, 1, -1][i % 3], + ] + ); + tids.push(c.rows[0].tid); + } + + // A deterministic vote history: most participants vote on most comments, + // a few change their vote later, one vote is stored NULL. + let t = 1700000100000; + for (let pi = 0; pi < pids.length; pi++) { + for (let ci = 0; ci < tids.length; ci++) { + if ((pi * 7 + ci * 3) % 5 === 0) continue; + const stored = [-1, 1, 0][(pi + ci) % 3]; + await pool.query( + "INSERT INTO votes (zid, pid, tid, vote, created, high_priority) VALUES ($1, $2, $3, $4, $5, $6)", + [zid, pids[pi], tids[ci], stored, (t += 1000), (pi + ci) % 4 === 0] + ); + voteRows++; + if ((pi + ci) % 6 === 0) { + await pool.query( + "INSERT INTO votes (zid, pid, tid, vote, created) VALUES ($1, $2, $3, $4, $5)", + [zid, pids[pi], tids[ci], -stored || 1, (t += 1000)] + ); + voteRows++; + } + } + } + await pool.query( + "INSERT INTO votes (zid, pid, tid, vote, created) VALUES ($1, $2, $3, NULL, $4)", + [zid, pids[1], tids[0], (t += 1000)] + ); + voteRows++; + + app = express(); + app.get("/report/:file", (req: any, res: any) => { + req.p = { rid, report_id: `r${tag}`, report_type: req.params.file }; + handle_GET_reportExport(req, res); + }); + app.get("/dataExport", (req: any, res: any) => { + req.p = { + uid: Number(req.query.uid), + zid, + unixTimestamp: 1, + format: "csv", + }; + handle_GET_dataExport(req, res); + }); + }); + + afterAll(async () => { + Config.AWS_S3_BUCKET_NAME = saved.bucket; + Config.offline = saved.offline; + Config.dataExportMaxCells = saved.limit; + }); + + const headers = { host: "box.local", "x-forwarded-proto": "http" }; + + async function streamedZip(): Promise<{ + status: number; + body: Buffer; + type: string; + }> { + const res = await request(app) + .get(`/dataExport?uid=${ownerUid}`) + .set(headers) + .buffer(true) + .parse(binary); + return { + status: res.status, + body: res.body as Buffer, + type: res.headers["content-type"], + }; + } + + test("with no bucket configured, every file in the zip is byte-identical to the report export's", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.offline = false; + Config.dataExportMaxCells = saved.limit; + + const zip = await streamedZip(); + expect(zip.status).toBe(200); + expect(zip.type).toContain("application/zip"); + const entries = unzip(zip.body); + expect([...entries.keys()]).toEqual([...CONVERSATION_ZIP_FILES]); + + for (const file of CONVERSATION_ZIP_FILES) { + const res = await request(app) + .get(`/report/${file}`) + .set(headers) + .buffer(true) + .parse(binary); + expect(res.status).toBe(200); + const served = res.body as Buffer; + expect(served.length).toBeGreaterThan(0); + expect(entries.get(file)!.equals(served)).toBe(true); + } + + // The fixture reached the files: every vote row, and the awkward bodies. + const votesCsv = entries.get("votes.csv")!.toString("utf8"); + expect(votesCsv.trimEnd().split("\n").length - 1).toBe(voteRows); + expect(entries.get("comments.csv")!.toString("utf8")).toContain("日本語"); + }); + + test("with OFFLINE set and a bucket configured, it streams the same zip", async () => { + Config.AWS_S3_BUCKET_NAME = "a-bucket"; + Config.offline = true; + const zip = await streamedZip(); + expect(zip.status).toBe(200); + expect([...unzip(zip.body).keys()]).toEqual([...CONVERSATION_ZIP_FILES]); + }); + + test("over the size limit it refuses with 413 and the closed reason", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.offline = false; + Config.dataExportMaxCells = voteRows - 1; + const res = await request(app) + .get(`/dataExport?uid=${ownerUid}`) + .set(headers); + expect(res.status).toBe(413); + expect(res.body.error).toBe("polis_err_data_export_too_large"); + expect(res.body.limit).toBe(voteRows - 1); + expect(res.body.cells).toBeGreaterThanOrEqual(voteRows); + expect(res.headers["content-type"]).not.toContain("application/zip"); + Config.dataExportMaxCells = saved.limit; + }); + + test("a caller who does not own the conversation is refused, as before", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + const stranger = await pool.query( + "INSERT INTO users (hname) VALUES ('Generated fixture stranger') RETURNING uid" + ); + const res = await request(app) + .get(`/dataExport?uid=${stranger.rows[0].uid}`) + .set(headers); + expect(res.status).toBe(403); + expect(res.body.error).toBe("polis_err_data_export_auth"); + }); +}); From 8454ba2052572f9f5995ec98197856cd087ff2ed Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 03:51:11 -0700 Subject: [PATCH 15/16] docs: the conversation zip's file list and the DATA_EXPORT_MAX_CELLS knob Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- docs/configuration.md | 8 ++++++++ docs/export-format.md | 35 +++++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0268a50130..45e7346616 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,6 +108,14 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b - **`ADMIN_EMAILS`** array of email addresses to receive team notifications. - **`POLIS_FROM_ADDRESS`** email address from which other emails are sent. +### Conversation data export + +`GET /api/v3/dataExport?conversation_id=` (conversation owners only) has two modes, chosen per request: + +- **`AWS_S3_BUCKET_NAME` set and `OFFLINE` unset** (production): unchanged. The route queues a `generate_export_data` task in `worker_tasks` and answers `{}`; `/api/v3/dataExport/results` signs a URL into the `polis-datadump` bucket. Nothing has consumed those tasks since the Clojure export worker stopped in 2022. +- **`AWS_S3_BUCKET_NAME` unset, or `OFFLINE` set to `1`/`true`**: the server streams the export itself as `polis-export--.zip`, holding the report export's `format.json`, `summary.csv`, `comments.csv`, `votes.csv` and `participant-votes.csv`, each byte-identical to `/api/v3/reportExport//` ([export-format.md](export-format.md#the-conversation-zip)). `/api/v3/dataExport/results` answers 404 `polis_err_data_export_results_not_stored`. +- **`DATA_EXPORT_MAX_CELLS`** (default `20000000`) the largest conversation the second mode streams, counted as the larger of its vote rows and its voters × comments. Over it the route answers 413 `polis_err_data_export_too_large` with `cells` and `limit`, before any byte of zip. A value that is not a positive integer means the default. The zip has no zip64 records, so it must stay under 4 GiB; at the default, `votes.csv` is about 2 GB at most. + ### Boolean Flags (All can be left blank, or `false`) diff --git a/docs/export-format.md b/docs/export-format.md index c2333f758c..d18a3b45d6 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -60,12 +60,35 @@ bytes exactly as before. The declaration is carried beside them: The report page's "Raw Data Export" section links every file, `format.json` included. Each file is served from -`/api/v3/reportExport//`; there is no zip bundle, so a -downloader who wants the declaration with the data fetches `format.json` (or -`summary.csv`) along with the CSVs. - -The older `/api/v3/dataExport` zip (the Clojure math worker's export) has no -producer since that worker was retired; it is not a source of these files. +`/api/v3/reportExport//`, so a downloader who wants the +declaration with the data fetches `format.json` (or `summary.csv`) along with +the CSVs, or takes the conversation zip below, which carries it. + +## The conversation zip + +`GET /api/v3/dataExport?conversation_id=` (conversation owners only) +streams one zip, `polis-export--.zip`, when the server +has no S3 bucket configured (`AWS_S3_BUCKET_NAME` unset) or runs with +`OFFLINE` set. It holds, in this order: + +| file | contents | +|-------------------------|--------------------------------------------| +| `format.json` | the sidecar: format id and declaration | +| `summary.csv` | ends with the `vote-convention` row | +| `comments.csv` | as below | +| `votes.csv` | as below | +| `participant-votes.csv` | as below | + +Each file is byte-identical to what +`/api/v3/reportExport//` serves for the same conversation at +the same moment; the zip adds no file and changes no byte. The zip is built +on request and streamed (deflate, no zip64). A conversation over the +`DATA_EXPORT_MAX_CELLS` limit is refused with HTTP 413 +`polis_err_data_export_too_large` ([configuration](configuration.md#conversation-data-export)). + +With a bucket configured and `OFFLINE` unset, the route keeps its older +behaviour: it queues a task for the Clojure math worker's export, which has had +no consumer since that worker stopped; that path is not a source of these files. ## Every file and column From f4e9269a656b891bc7b6dd65a0806c9bcf86cd1f Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Sat, 3 Oct 2026 04:07:13 -0700 Subject: [PATCH 16/16] server: the streamed export stops when the client goes away and paces its reads to the client Review of #2940 (F-1): a client that disconnected left the handler pending forever (a destroyed response never emits finish), and the producers read and compressed the rest of the conversation into a dead socket. - src/db/streamControl.ts: an AsyncLocalStorage context for the Postgres row streams. stream_queryP_readOnly, inside it, destroys the running stream on abort (cursor closed, client discarded), fails a stream that has not started, and pauses while the output needs draining. Outside a context nothing changes. - writeConversationZip aborts on the response's close and runs each producer inside the context with the response as the output. - ZipStreamWriter: writes to a destroyed or ended output throw polis_err_zip_output_closed; finish() resolves on finish and rejects on close. Each entry also carries the UTC extended timestamp (0x5455). - The handler logs a client close at info and returns. - The size count reads votes once (COUNT and COUNT DISTINCT together). - F-2: positiveIntOr no longer sits under isTrueOrBlank's comment. Tests: unit for the destroyed output, the abort reaching the producer and the handler returning; unit for the stream control (abort mid-stream, abort before start, pause and resume). Integration on a 600,000-vote generated fixture: a client that stops reading holds the votes.csv query paused, and on disconnect the handler returns and the backend is gone; a reading client gets every row. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- docs/configuration.md | 2 +- docs/export-format.md | 3 +- .../integration/data-export-stream.test.ts | 151 ++++++++++++++++++ server/__tests__/unit/conversationZip.test.ts | 126 ++++++++++++++- server/__tests__/unit/streamControl.test.ts | 133 +++++++++++++++ server/src/config.ts | 2 +- server/src/db/pg-query.ts | 22 +++ server/src/db/streamControl.ts | 28 ++++ server/src/export/conversationZip.ts | 56 +++++-- server/src/export/zipStream.ts | 53 +++++- server/src/routes/dataExport.ts | 5 + 11 files changed, 555 insertions(+), 26 deletions(-) create mode 100644 server/__tests__/unit/streamControl.test.ts create mode 100644 server/src/db/streamControl.ts diff --git a/docs/configuration.md b/docs/configuration.md index 45e7346616..25dbeced61 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,7 +113,7 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b `GET /api/v3/dataExport?conversation_id=` (conversation owners only) has two modes, chosen per request: - **`AWS_S3_BUCKET_NAME` set and `OFFLINE` unset** (production): unchanged. The route queues a `generate_export_data` task in `worker_tasks` and answers `{}`; `/api/v3/dataExport/results` signs a URL into the `polis-datadump` bucket. Nothing has consumed those tasks since the Clojure export worker stopped in 2022. -- **`AWS_S3_BUCKET_NAME` unset, or `OFFLINE` set to `1`/`true`**: the server streams the export itself as `polis-export--.zip`, holding the report export's `format.json`, `summary.csv`, `comments.csv`, `votes.csv` and `participant-votes.csv`, each byte-identical to `/api/v3/reportExport//` ([export-format.md](export-format.md#the-conversation-zip)). `/api/v3/dataExport/results` answers 404 `polis_err_data_export_results_not_stored`. +- **`AWS_S3_BUCKET_NAME` unset, or `OFFLINE` set to `1`/`true`**: the server streams the export itself as `polis-export--.zip`, holding the report export's `format.json`, `summary.csv`, `comments.csv`, `votes.csv` and `participant-votes.csv`, each byte-identical to `/api/v3/reportExport//` ([export-format.md](export-format.md#the-conversation-zip)). `/api/v3/dataExport/results` answers 404 `polis_err_data_export_results_not_stored`. The Postgres reads pause while the client is not keeping up, and stop (cursor closed) if the client disconnects. Note that this route treats an unset bucket as a mode switch, while the server's other S3 users fall back to `polis-delphi`: a deployment that drops `AWS_S3_BUCKET_NAME` switches this route to streaming. - **`DATA_EXPORT_MAX_CELLS`** (default `20000000`) the largest conversation the second mode streams, counted as the larger of its vote rows and its voters × comments. Over it the route answers 413 `polis_err_data_export_too_large` with `cells` and `limit`, before any byte of zip. A value that is not a positive integer means the default. The zip has no zip64 records, so it must stay under 4 GiB; at the default, `votes.csv` is about 2 GB at most. ### Boolean Flags diff --git a/docs/export-format.md b/docs/export-format.md index d18a3b45d6..8db691b66b 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -82,7 +82,8 @@ has no S3 bucket configured (`AWS_S3_BUCKET_NAME` unset) or runs with Each file is byte-identical to what `/api/v3/reportExport//` serves for the same conversation at the same moment; the zip adds no file and changes no byte. The zip is built -on request and streamed (deflate, no zip64). A conversation over the +on request and streamed (deflate, no zip64): the database reads pause while +the client is not keeping up and stop if it disconnects. A conversation over the `DATA_EXPORT_MAX_CELLS` limit is refused with HTTP 413 `polis_err_data_export_too_large` ([configuration](configuration.md#conversation-data-export)). diff --git a/server/__tests__/integration/data-export-stream.test.ts b/server/__tests__/integration/data-export-stream.test.ts index 47b9d5ad45..0da7183623 100644 --- a/server/__tests__/integration/data-export-stream.test.ts +++ b/server/__tests__/integration/data-export-stream.test.ts @@ -13,6 +13,7 @@ */ import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; import express from "express"; +import http from "http"; import request from "supertest"; import zlib from "zlib"; import Config from "../../src/config"; @@ -261,3 +262,153 @@ describe("dataExport streamed from the server", () => { expect(res.body.error).toBe("polis_err_data_export_auth"); }); }); + +/** + * A large generated-fixture conversation (votes written with generate_series), + * served over a real socket, to check what happens when the client is slow or + * goes away: the Postgres stream behind votes.csv pauses while the response + * needs draining, and a client that disconnects ends the handler and leaves no + * streaming query behind. + */ +describe("dataExport streamed: slow and vanished clients", () => { + const tag = `p053y${Date.now().toString(36)}`; + const VOTERS = 2000; + const COMMENTS = 300; + const VOTES_CSV_QUERY = "%FROM votes WHERE zid = $1 ORDER BY tid, pid%"; + const savedBucket = Config.AWS_S3_BUCKET_NAME; + let ownerUid: number; + let zid: number; + let server: http.Server; + let port: number; + let lastHandler: Promise | undefined; + + beforeAll(async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + const owner = await pool.query( + "INSERT INTO users (hname) VALUES ('Generated fixture owner') RETURNING uid" + ); + ownerUid = owner.rows[0].uid; + zid = ( + await pool.query( + "INSERT INTO conversations (topic, description, owner) VALUES ('Generated fixture, large', 'generated', $1) RETURNING zid", + [ownerUid] + ) + ).rows[0].zid; + await pool.query("INSERT INTO zinvites (zid, zinvite) VALUES ($1, $2)", [ + zid, + tag, + ]); + const pid = ( + await pool.query( + "INSERT INTO participants (pid, zid, uid) VALUES (NULL, $1, $2) RETURNING pid", + [zid, ownerUid] + ) + ).rows[0].pid; + await pool.query( + `INSERT INTO comments (tid, zid, pid, uid, txt) + SELECT g, $1, $2, $3, 'generated statement ' || g FROM generate_series(0, $4 - 1) g`, + [zid, pid, ownerUid, COMMENTS] + ); + await pool.query( + `INSERT INTO votes (zid, pid, tid, vote, created) + SELECT $1, p, t, ((p + t) % 3) - 1, 1700000000000 + p * 1000 + t + FROM generate_series(0, $2 - 1) p, generate_series(0, $3 - 1) t`, + [zid, VOTERS, COMMENTS] + ); + + const app = express(); + app.get("/dataExport", (req: any, res: any) => { + req.p = { uid: ownerUid, zid, unixTimestamp: 1, format: "csv" }; + lastHandler = handle_GET_dataExport(req, res); + }); + server = http.createServer(app); + await new Promise((r) => server.listen(0, r)); + port = (server.address() as { port: number }).port; + }, 120000); + + afterAll(async () => { + Config.AWS_S3_BUCKET_NAME = savedBucket; + await new Promise((r) => server.close(r)); + }); + + /** Backends whose current or last statement is the votes.csv stream. */ + async function streamingBackends(activeOnly: boolean): Promise { + const r = await pool.query( + `SELECT count(*)::int AS n FROM pg_stat_activity + WHERE query LIKE $1 AND pid <> pg_backend_pid() + AND ($2::boolean IS FALSE OR state <> 'idle')`, + [VOTES_CSV_QUERY, activeOnly] + ); + return r.rows[0].n; + } + + async function waitFor(check: () => Promise, ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + if (await check()) return true; + await new Promise((r) => setTimeout(r, 100)); + } + return false; + } + + test("a client that stops reading holds the votes.csv stream paused; when it disconnects, the handler returns and no query remains", async () => { + lastHandler = undefined; + let received = 0; + const req = http.get({ + host: "127.0.0.1", + port, + path: "/dataExport", + headers: { host: "box.local", "x-forwarded-proto": "http" }, + }); + const response = await new Promise((resolve) => + req.on("response", resolve) + ); + expect(response.statusCode).toBe(200); + response.on("data", (c: Buffer) => { + received += c.length; + // Stop reading once votes.csv has started arriving. + if (received > 64 * 1024) response.pause(); + }); + + // The votes.csv query is open and stays open while the client does not + // read: the stream is paused, not run to the end into Node's buffers. + expect( + await waitFor( + async () => response.isPaused() && (await streamingBackends(true)) > 0, + 20000 + ) + ).toBe(true); + const before = received; + await new Promise((r) => setTimeout(r, 1500)); + expect(await streamingBackends(true)).toBe(1); + expect(received).toBe(before); + + req.destroy(); + const outcome = await Promise.race([ + lastHandler!.then(() => "returned"), + new Promise((r) => setTimeout(() => r("hung"), 10000)), + ]); + expect(outcome).toBe("returned"); + // The cursor was closed and its client discarded: the backend is gone. + expect( + await waitFor(async () => (await streamingBackends(false)) === 0, 5000) + ).toBe(true); + }, 60000); + + test("a reading client still gets the whole zip", async () => { + lastHandler = undefined; + const res = await request(`http://127.0.0.1:${port}`) + .get("/dataExport") + .set({ host: "box.local", "x-forwarded-proto": "http" }) + .buffer(true) + .parse(binary); + expect(res.status).toBe(200); + const votes = unzip(res.body as Buffer) + .get("votes.csv")! + .toString("utf8"); + expect(votes.trimEnd().split("\n").length - 1).toBe(VOTERS * COMMENTS); + await lastHandler; + // Run to the end: the client went back to the pool idle, nothing running. + expect(await streamingBackends(true)).toBe(0); + }, 60000); +}); diff --git a/server/__tests__/unit/conversationZip.test.ts b/server/__tests__/unit/conversationZip.test.ts index 87716db822..e2cda478df 100644 --- a/server/__tests__/unit/conversationZip.test.ts +++ b/server/__tests__/unit/conversationZip.test.ts @@ -52,7 +52,8 @@ import { dataExportStreamsLocally, writeConversationZip, } from "../../src/export/conversationZip"; -import { ZipStreamWriter } from "../../src/export/zipStream"; +import { ZIP_OUTPUT_CLOSED, ZipStreamWriter } from "../../src/export/zipStream"; +import { streamControl } from "../../src/db/streamControl"; import { handle_GET_dataExport, handle_GET_dataExport_results, @@ -239,6 +240,14 @@ describe("ZipStreamWriter", () => { expect(e.method).toBe(8); expect(e.flags & 0x0808).toBe(0x0808); } + // Every local header carries the UTC extended timestamp (0x5455). + const buf = await bytes; + expect(buf.readUInt16LE(28)).toBe(9); + const extra = 30 + "a.txt".length; + expect(buf.readUInt16LE(extra)).toBe(0x5455); + expect(buf.readUInt32LE(extra + 5)).toBe( + Date.UTC(2026, 9, 3, 12, 0, 0) / 1000 + ); }); test("streams a multi-megabyte entry without holding it: bytes leave before the entry ends", async () => { @@ -590,3 +599,118 @@ describe("GET /api/v3/dataExport/results", () => { Config.AWS_S3_BUCKET_NAME = saved.bucket; }); }); + +// ------------------------------------------------- a client that goes away + +/** A producer that writes forever, a row per tick, until the export is aborted. */ +function endlessProducer(seen: { aborted?: boolean; output?: unknown }) { + return (_zid: number, _siteUrl: string, res: any) => { + const control = streamControl.getStore(); + seen.output = control?.output; + let i = 0; + const tick = () => { + if (control?.signal?.aborted) { + seen.aborted = true; + return; + } + res.write(`${i++},row\n`.repeat(2000)); + setImmediate(tick); + }; + setImmediate(tick); + }; +} + +describe("a destroyed response", () => { + test("ZipStreamWriter: a write after the output is destroyed throws, and finish() rejects instead of hanging", async () => { + const out = new PassThrough(); + out.resume(); + const zip = new ZipStreamWriter(out); + await zip.entry("a.txt", async (sink) => sink.write("hello")); + out.destroy(); + await expect( + zip.entry("b.txt", async (sink) => sink.write("x")) + ).rejects.toThrow(ZIP_OUTPUT_CLOSED); + + // Nobody reads out2 and its buffers are small, so end() cannot finish; + // the stream is destroyed while finish() waits. + const out2 = new PassThrough({ highWaterMark: 16 }); + const zip2 = new ZipStreamWriter(out2); + const noise = Buffer.from( + Array.from({ length: 20_000 }, (_, i) => (i * 7919) % 251) + ); + await zip2.entry("noise.bin", async (sink) => sink.write(noise)); + const finishing = zip2.finish(); + await new Promise((r) => setImmediate(r)); + out2.destroy(); + await expect(finishing).rejects.toThrow(ZIP_OUTPUT_CLOSED); + }); + + test("writeConversationZip: the export rejects and the producer sees the abort", async () => { + const { producers } = generatedFixtures(); + const seen: { aborted?: boolean; output?: unknown } = {}; + producers["votes.csv"] = endlessProducer(seen); + const out = new PassThrough(); + let received = 0; + out.on("data", (c: Buffer) => { + received += c.length; + if (received > 200_000) out.destroy(); + }); + await expect( + writeConversationZip(out, 1, "http://box", producers) + ).rejects.toThrow(ZIP_OUTPUT_CLOSED); + await new Promise((r) => setImmediate(r)); + await new Promise((r) => setImmediate(r)); + expect(seen.aborted).toBe(true); + // The producers run with the response as the output to pace against. + expect(seen.output).toBe(out); + }); + + test("an output already destroyed: rejects before running any producer", async () => { + const { producers } = generatedFixtures(); + const formatJson = jest.fn(producers["format.json"]); + producers["format.json"] = formatJson as ExportProducer; + const out = new PassThrough(); + out.destroy(); + await expect( + writeConversationZip(out, 1, "http://box", producers) + ).rejects.toThrow(ZIP_OUTPUT_CLOSED); + expect(formatJson).not.toHaveBeenCalled(); + }); + + test("GET /api/v3/dataExport returns when the client goes away mid-stream", async () => { + Config.AWS_S3_BUCKET_NAME = undefined; + Config.offline = false; + (isModerator as jest.Mock).mockResolvedValue(true as never); + (getZinvite as jest.Mock).mockResolvedValue("abc123" as never); + counts(10, 2, 2); + useGeneratedReport(); + const seen: { aborted?: boolean } = {}; + (report.sendVotesSummary as jest.Mock).mockImplementation( + async (_zid: any, res: any) => endlessProducer(seen)(0, "", res) + ); + const res = mockRes(); + let received = 0; + res.on("data", (c: Buffer) => { + received += c.length; + if (received > 200_000) res.destroy(); + }); + const handled = handle_GET_dataExport( + { + p: { uid: 7, zid: 42, unixTimestamp: 1, format: "csv" }, + headers: { host: "box.local", "x-forwarded-proto": "http" }, + } as any, + res + ); + await expect( + Promise.race([ + handled.then(() => "returned"), + new Promise((r) => setTimeout(() => r("hung"), 5000)), + ]) + ).resolves.toBe("returned"); + await new Promise((r) => setImmediate(r)); + await new Promise((r) => setImmediate(r)); + expect(seen.aborted).toBe(true); + expect(res.status).not.toHaveBeenCalled(); + Config.AWS_S3_BUCKET_NAME = saved.bucket; + }); +}); diff --git a/server/__tests__/unit/streamControl.test.ts b/server/__tests__/unit/streamControl.test.ts new file mode 100644 index 0000000000..0dc7421b48 --- /dev/null +++ b/server/__tests__/unit/streamControl.test.ts @@ -0,0 +1,133 @@ +/** + * pg-query's `stream_queryP_readOnly` under src/db/streamControl.ts: outside a + * control context it behaves as before; inside one, an abort destroys the + * running stream (closing its cursor and discarding the client) or fails a + * stream that has not started, and the stream pauses while the output needs + * draining. `pg` is replaced by a fake pool whose query stream is a plain + * object-mode Readable of generated rows. + */ +import { beforeEach, describe, expect, jest, test } from "@jest/globals"; +import { PassThrough, Readable } from "stream"; + +const fake: { stream?: Readable; done?: jest.Mock; rows: number } = { + rows: 0, +}; + +jest.mock("pg", () => { + class Pool { + on() { + return this; + } + connect(cb: (err: unknown, client: unknown, done: unknown) => void) { + fake.done = jest.fn(); + const client = { + query: () => { + let i = 0; + fake.stream = new Readable({ + objectMode: true, + highWaterMark: 4, + read() { + // One row per tick, like rows arriving off a socket. + setImmediate(() => { + if (i < fake.rows) this.push({ n: i++ }); + else this.push(null); + }); + }, + }); + return fake.stream; + }, + }; + setImmediate(() => cb(undefined, client, fake.done)); + } + } + return { __esModule: true, Pool, default: { Pool } }; +}); +jest.mock("pg-query-stream", () => ({ + __esModule: true, + default: class { + constructor() { + return {}; + } + }, +})); +jest.mock("pg-connection-string", () => ({ parse: () => ({}) })); +jest.mock("../../src/utils/logger"); + +import pg from "../../src/db/pg-query"; +import { QUERY_ABORTED, streamControl } from "../../src/db/streamControl"; + +function run(control?: { signal?: AbortSignal; output?: any }) { + const rows: any[] = []; + let resolveDone: (v: string) => void = () => undefined; + const settled = new Promise((r) => (resolveDone = r)); + const onRow = jest.fn((row: any) => rows.push(row)); + const start = () => + pg.stream_queryP_readOnly( + "SELECT generated", + [], + onRow, + () => resolveDone("end"), + (err: Error) => resolveDone(`error:${err.message}`) + ); + if (control) streamControl.run(control, start); + else start(); + return { rows, settled, onRow }; +} + +const tick = () => new Promise((r) => setImmediate(r)); + +describe("stream_queryP_readOnly with stream control", () => { + beforeEach(() => { + fake.stream = undefined; + fake.rows = 0; + }); + + test("without a control context: every row, then end, client released cleanly", async () => { + fake.rows = 50; + const { rows, settled } = run(); + expect(await settled).toBe("end"); + expect(rows).toHaveLength(50); + expect(fake.done).toHaveBeenCalledWith(); + }); + + test("an abort mid-stream destroys the stream and discards the client", async () => { + fake.rows = 100_000; + const aborter = new AbortController(); + const { rows, settled } = run({ signal: aborter.signal }); + while (rows.length < 100) await tick(); + aborter.abort(); + expect(await settled).toBe(`error:${QUERY_ABORTED}`); + expect(fake.stream!.destroyed).toBe(true); + expect(rows.length).toBeLessThan(100_000); + // done(error): the pool discards the client instead of reusing it. + expect(fake.done).toHaveBeenCalledWith(expect.any(Error)); + }); + + test("a signal aborted before the pool hands over a client: no query starts", async () => { + fake.rows = 10; + const aborter = new AbortController(); + aborter.abort(); + const { settled, onRow } = run({ signal: aborter.signal }); + expect(await settled).toBe(`error:${QUERY_ABORTED}`); + expect(fake.stream).toBeUndefined(); + expect(onRow).not.toHaveBeenCalled(); + expect(fake.done).toHaveBeenCalledWith(); + }); + + test("pauses while the output needs draining, resumes on drain", async () => { + fake.rows = 1000; + const output = new PassThrough({ highWaterMark: 16 }); + // Nobody reads `output`; fill it past its high-water mark. + output.write(Buffer.alloc(64)); + expect(output.writableNeedDrain).toBe(true); + const { rows, settled } = run({ output }); + while (rows.length < 1) await tick(); + for (let i = 0; i < 10; i++) await tick(); + // One row, then paused. + expect(rows).toHaveLength(1); + expect(fake.stream!.isPaused()).toBe(true); + output.resume(); // drains -> "drain" -> stream resumes + expect(await settled).toBe("end"); + expect(rows).toHaveLength(1000); + }); +}); diff --git a/server/src/config.ts b/server/src/config.ts index deac509abf..62f81c8534 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -218,13 +218,13 @@ export default { webserverUsername: process.env.WEBSERVER_USERNAME as string, }; -// Use this function when a value should default to true if not set. /** A positive integer from the environment, or the fallback when unset or not one. */ function positiveIntOr(val: string | undefined, fallback: number): number { const n = Number((val || "").trim()); return Number.isSafeInteger(n) && n > 0 ? n : fallback; } +// Use this function when a value should default to true if not set. function isTrueOrBlank(val: string | boolean | undefined): boolean { return val === undefined || val === "" || isTrue(val); } diff --git a/server/src/db/pg-query.ts b/server/src/db/pg-query.ts index d982ef48a5..5ed03c7738 100644 --- a/server/src/db/pg-query.ts +++ b/server/src/db/pg-query.ts @@ -6,6 +6,7 @@ import QueryStream from "pg-query-stream"; import Config from "../config"; import logger from "../utils/logger"; import { MPromise } from "../utils/metered"; +import { QUERY_ABORTED, streamControl } from "./streamControl"; // # DB Connections // @@ -218,25 +219,46 @@ function stream_queryP_readOnly( onError: (error: Error) => void ) { const query = new QueryStream(queryString, params); + // Captured now: the pool callback below may run outside the caller's context. + const control = streamControl.getStore(); + const signal = control?.signal; + const output = control?.output; readPool.connect((err, client, done) => { if (err) { onError(err); return; } + if (signal?.aborted) { + done(); + onError(new Error(QUERY_ABORTED)); + return; + } const stream = client.query(query); + // Destroying the stream closes its cursor; the error path below then + // discards the client, so nothing of the query outlives the abort. + const onAbort = () => stream.destroy(new Error(QUERY_ABORTED)); + signal?.addEventListener("abort", onAbort, { once: true }); + const detach = () => signal?.removeEventListener("abort", onAbort); + stream.on("data", (row: QueryResult) => { onRow(row); + if (output && output.writableNeedDrain && !stream.isPaused()) { + stream.pause(); + output.once("drain", () => stream.resume()); + } }); stream.on("end", () => { + detach(); done(); onEnd(); }); stream.on("error", (error: Error) => { + detach(); done(error); onError(error); }); diff --git a/server/src/db/streamControl.ts b/server/src/db/streamControl.ts new file mode 100644 index 0000000000..dc03fb5405 --- /dev/null +++ b/server/src/db/streamControl.ts @@ -0,0 +1,28 @@ +/** + * Optional control over the Postgres row streams (pg-query.ts + * `stream_queryP_readOnly`) started inside `streamControl.run(...)`. + * + * The report export functions start their streams themselves and do not + * return them, so a caller that needs to stop or slow those streams (the + * streamed conversation zip, src/export/conversationZip.ts) sets this context + * around them instead: + * + * - `signal`: when it aborts, a running stream is destroyed (its cursor closed + * and its pool client discarded) and a stream not yet started fails at once, + * both with QUERY_ABORTED through the caller's onError. + * - `output`: after each row, while `output.writableNeedDrain`, the stream is + * paused until `output` emits "drain". + * + * Outside such a context nothing changes. + */ +import { AsyncLocalStorage } from "async_hooks"; +import type { Writable } from "stream"; + +export interface StreamControl { + signal?: AbortSignal; + output?: Writable; +} + +export const QUERY_ABORTED = "polis_err_query_aborted"; + +export const streamControl = new AsyncLocalStorage(); diff --git a/server/src/export/conversationZip.ts b/server/src/export/conversationZip.ts index 6bbf8d05c3..dbe6343966 100644 --- a/server/src/export/conversationZip.ts +++ b/server/src/export/conversationZip.ts @@ -12,12 +12,19 @@ * byte sequence that route serves for the same conversation. * * The zip is written entry by entry to the response as the rows stream out of - * Postgres; it is never assembled in memory. A conversation over + * Postgres; the writer holds at most one 256 KiB run of input. The Postgres + * streams are paused while the response needs draining, so a slow client + * slows the reads rather than growing Node's socket buffer, and when the + * response closes early (the client went away) the current stream is + * destroyed, its cursor closed, and the export rejects (src/db/streamControl.ts). + * comments.csv and summary.csv are still built as one string each, exactly as + * the report export builds them. A conversation over * `Config.dataExportMaxCells` is refused before the first byte (413). */ import type { Writable } from "stream"; import Config from "../config"; import pg from "../db/pg-query"; +import { streamControl } from "../db/streamControl"; import { sendCommentSummary, sendConversationSummary, @@ -25,7 +32,7 @@ import { sendVotesSummary, } from "../report"; import { exportFormatJson } from "../votes/exportFormat"; -import { ZipEntrySink, ZipStreamWriter } from "./zipStream"; +import { ZIP_OUTPUT_CLOSED, ZipEntrySink, ZipStreamWriter } from "./zipStream"; /** The files in the zip, in order (docs/export-format.md). */ export const CONVERSATION_ZIP_FILES = [ @@ -92,16 +99,21 @@ export function runProducerIntoEntry( produce: ExportProducer, zid: number, siteUrl: string, - sink: ZipEntrySink + sink: ZipEntrySink, + control: { signal?: AbortSignal; output?: Writable } = {} ): Promise { return new Promise((resolve, reject) => { let settled = false; + const onAbort = () => settle(new Error(ZIP_OUTPUT_CLOSED)); const settle = (err?: unknown) => { if (settled) return; settled = true; + control.signal?.removeEventListener("abort", onAbort); if (err) reject(err); else resolve(); }; + if (control.signal?.aborted) return onAbort(); + control.signal?.addEventListener("abort", onAbort, { once: true }); const write = (data: string) => { if (settled) return; try { @@ -129,9 +141,13 @@ export function runProducerIntoEntry( }, }), }; - Promise.resolve() - .then(() => produce(zid, siteUrl, res)) - .catch((err) => settle(err)); + // The producer's Postgres streams see the signal and the output through + // this context: they stop on abort and pause while the output drains. + streamControl.run(control, () => { + Promise.resolve() + .then(() => produce(zid, siteUrl, res)) + .catch((err) => settle(err)); + }); }); } @@ -147,12 +163,23 @@ export async function writeConversationZip( modified: Date = new Date() ): Promise { const zip = new ZipStreamWriter(out, modified); - for (const name of CONVERSATION_ZIP_FILES) { - await zip.entry(name, (sink) => - runProducerIntoEntry(producers[name], zid, siteUrl, sink) - ); + // A response that closes before the zip is complete (the client went away) + // stops the current producer and its query; see streamControl.ts. + const aborter = new AbortController(); + const onClose = () => aborter.abort(); + out.once("close", onClose); + if (out.destroyed) aborter.abort(); + const control = { signal: aborter.signal, output: out }; + try { + for (const name of CONVERSATION_ZIP_FILES) { + await zip.entry(name, (sink) => + runProducerIntoEntry(producers[name], zid, siteUrl, sink, control) + ); + } + await zip.finish(); + } finally { + out.off("close", onClose); } - await zip.finish(); } export interface ConversationExportSize { @@ -168,9 +195,10 @@ export async function conversationExportSize( zid: number ): Promise { const rows = (await pg.queryP_readOnly( - `SELECT (SELECT COUNT(*) FROM votes WHERE zid = $1) AS votes, - (SELECT COUNT(DISTINCT pid) FROM votes WHERE zid = $1) AS voters, - (SELECT COUNT(*) FROM comments WHERE zid = $1) AS comments`, + `SELECT v.votes, v.voters, + (SELECT COUNT(*) FROM comments WHERE zid = $1) AS comments + FROM (SELECT COUNT(*) AS votes, COUNT(DISTINCT pid) AS voters + FROM votes WHERE zid = $1) v`, [zid] )) as { votes: string; voters: string; comments: string }[]; const votes = Number(rows[0]?.votes || 0); diff --git a/server/src/export/zipStream.ts b/server/src/export/zipStream.ts index 022148ffeb..7e85908026 100644 --- a/server/src/export/zipStream.ts +++ b/server/src/export/zipStream.ts @@ -7,8 +7,17 @@ * to be known before its first byte goes out. Input is compressed * synchronously in runs of BLOCK_BYTES, each ending in a sync flush, so the * writer holds at most one run of input; the central directory (about 60 bytes - * per entry) is kept until `finish()`. Like the report export's CSV routes, it - * does not wait for `out` to drain. + * per entry) is kept until `finish()`. The writer itself never waits for `out` + * to drain; the caller paces its producers (conversationZip.ts pauses the + * Postgres streams while `out` needs draining). + * + * Once `out` is destroyed or ended by someone else (a client that went away), + * every further write throws ZIP_OUTPUT_CLOSED, and `finish()` rejects rather + * than wait for a "finish" that will never come. + * + * Times: the DOS date/time fields carry UTC (the DOS fields have no zone), and + * every entry also carries the extended-timestamp extra field (0x5455) with the + * UTC Unix modification time, which unzip, Info-ZIP, bsdtar and 7-Zip prefer. * * Without zip64 an archive, and every entry in it, must stay under 4 GiB; the * writer fails rather than emit a corrupt archive past that. Callers bound the @@ -32,6 +41,7 @@ const SYNC_FLUSH = zlib.constants.Z_SYNC_FLUSH; const FINAL_BLOCK = zlib.deflateRawSync(Buffer.alloc(0)); export const ZIP_TOO_LARGE = "polis_err_zip_too_large"; +export const ZIP_OUTPUT_CLOSED = "polis_err_zip_output_closed"; /** Where an entry's producer writes its uncompressed bytes. */ export interface ZipEntrySink { @@ -58,12 +68,25 @@ export class ZipStreamWriter { private busy = false; private finished = false; private readonly stamp: { time: number; date: number }; + // Extended timestamp extra field: tag 0x5455, 5 data bytes, flags = mtime. + private readonly extra: Buffer; constructor(private readonly out: Writable, modified: Date = new Date()) { this.stamp = dosDateTime(modified); + this.extra = Buffer.alloc(9); + this.extra.writeUInt16LE(0x5455, 0); + this.extra.writeUInt16LE(5, 2); + this.extra.writeUInt8(1, 4); + this.extra.writeUInt32LE( + Math.max(0, Math.floor(modified.getTime() / 1000)) >>> 0, + 5 + ); } private emit(buf: Buffer) { + if (this.out.destroyed || this.out.writableEnded) { + throw new Error(ZIP_OUTPUT_CLOSED); + } if (this.offset + buf.length > ZIP32_MAX) { throw new Error(ZIP_TOO_LARGE); } @@ -98,9 +121,8 @@ export class ZipStreamWriter { local.writeUInt16LE(this.stamp.date, 12); // CRC-32 and both sizes stay zero here; the data descriptor carries them. local.writeUInt16LE(nameBuf.length, 26); - local.writeUInt16LE(0, 28); - this.emit(local); - this.emit(nameBuf); + local.writeUInt16LE(this.extra.length, 28); + this.emit(Buffer.concat([local, nameBuf, this.extra])); let crc = 0; let size = 0; @@ -160,10 +182,11 @@ export class ZipStreamWriter { record.writeUInt32LE(compressedSize, 20); record.writeUInt32LE(size, 24); record.writeUInt16LE(nameBuf.length, 28); - // extra length, comment length, disk number, internal attributes: zero + record.writeUInt16LE(this.extra.length, 30); + // comment length, disk number, internal attributes: zero record.writeUInt32LE((FILE_MODE << 16) >>> 0, 38); record.writeUInt32LE(headerOffset, 42); - this.central.push(Buffer.concat([record, nameBuf])); + this.central.push(Buffer.concat([record, nameBuf, this.extra])); this.busy = false; } @@ -185,6 +208,20 @@ export class ZipStreamWriter { end.writeUInt32LE(directorySize, 12); end.writeUInt32LE(directoryOffset, 16); this.emit(end); - await new Promise((resolve) => this.out.end(resolve)); + await new Promise((resolve, reject) => { + // A destroyed response never emits "finish" (and never calls an end + // callback); "close" without "finish" first means the zip did not arrive. + const onFinish = () => { + this.out.off("close", onClose); + resolve(); + }; + const onClose = () => { + this.out.off("finish", onFinish); + reject(new Error(ZIP_OUTPUT_CLOSED)); + }; + this.out.once("finish", onFinish); + this.out.once("close", onClose); + this.out.end(); + }); } } diff --git a/server/src/routes/dataExport.ts b/server/src/routes/dataExport.ts index eb6f83eb5c..eac319ab06 100644 --- a/server/src/routes/dataExport.ts +++ b/server/src/routes/dataExport.ts @@ -52,6 +52,11 @@ async function streamConversationExport( try { await writeConversationZip(res, zid, siteUrl); } catch (err) { + if (res.destroyed) { + // The client went away; the producers and their queries have stopped. + logger.info("polis_data_export_client_closed", { zid }); + return; + } if (!res.headersSent) { return failJson(res, 500, "polis_err_data_export_stream", err); }