Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7fa7242
server: add the vote convention module (src/votes/convention.ts)
colinmegill Oct 3, 2026
114a385
server: route the CSV exports through the vote convention module
colinmegill Oct 3, 2026
8e4b6f7
server: route the votes-bulk import through exportToStorage
colinmegill Oct 3, 2026
cfac05c
server: route the vote write path through the vote convention module
colinmegill Oct 3, 2026
7d37f25
server: route the vote read sites through the vote convention module
colinmegill Oct 3, 2026
f605f58
server: fail on a new vote sign literal outside the convention module
colinmegill Oct 3, 2026
108a5e2
server: the projection-gate witness runs the real vote convention module
colinmegill Oct 3, 2026
c72038b
delphi: the projection-gate source-bound test copies the vote convent…
colinmegill Oct 3, 2026
0620951
server: the report exports declare their vote sign (summary.csv row, …
colinmegill Oct 3, 2026
45389ba
server: the votes-bulk import accepts a declared vote sign
colinmegill Oct 3, 2026
d1682d9
docs: the export and import file formats state their vote sign
colinmegill Oct 3, 2026
4257ac5
server: keep the empty-math summary golden; declare the importance co…
colinmegill Oct 3, 2026
56ab1ab
server: the conversation data export streams a zip from the server wh…
colinmegill Oct 3, 2026
66f2ae8
server: integration case: the streamed export zip is byte-identical t…
colinmegill Oct 3, 2026
8454ba2
docs: the conversation zip's file list and the DATA_EXPORT_MAX_CELLS …
colinmegill Oct 3, 2026
f4e9269
server: the streamed export stops when the client goes away and paces…
colinmegill Oct 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions client-admin/src/components/conversation-admin/BYODConfig.js
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,14 @@ const BYODConfig = () => {
</pre>
vote_value MUST follow the pattern: 1 = agree, -1 = disagree, 0 = neutral/pass
<br />
Optionally, the file may declare this sign on a{' '}
<code># vote-convention:</code> first line before the header; a file that declares
any other sign is refused. See{' '}
<a href="https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md">
docs/export-format.md
</a>
.
<br />
</>
<Box sx={{ mt: 2, display: 'block' }}>
<input onChange={handleFileChange} type="file" id="csvFile" accept=".csv"></input>
Expand Down
19 changes: 19 additions & 0 deletions client-report/src/components/RawDataExport.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ const RawDataExport = ({ conversation, report_id }) => {
<p style={{ wordBreak: "break-all", fontFamily: "monospace", fontStyle: "italic" }}>
{`The following data exports are anonymized. Participants are identifed by an integer representing the order in which they first voted. For a full description of files and columns, please see: `}
<a href="https://compdemocracy.org/export/"> https://compdemocracy.org/export/ </a>
{` and, for the vote sign of each file: `}
<a href="https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md">
{" "}
docs/export-format.md{" "}
</a>
</p>
<p style={{ wordBreak: "break-all", fontFamily: "monospace" }}>
{`----------Summary: `}
Expand Down Expand Up @@ -93,6 +98,17 @@ const RawDataExport = ({ conversation, report_id }) => {
{` (as comments x participants matrix)`}
</p>
)}
<p style={{ wordBreak: "break-all", fontFamily: "monospace" }}>
{`------Format (sign): `}
<a
download={`${conversation.conversation_id}-format.json`}
href={`//${window.location.hostname}/api/v3/reportExport/${report_id}/format.json`}
type="application/json"
>
{`${conversation.conversation_id}-format.json`}
</a>
{` (the agree / disagree / pass values every file uses; also the vote-convention row of summary.csv)`}
</p>
<p style={{ wordBreak: "break-all", fontFamily: "monospace" }}>
{`---Comment groups: `}
<a
Expand Down Expand Up @@ -128,6 +144,9 @@ const RawDataExport = ({ conversation, report_id }) => {
<p style={{ wordBreak: "break-all", fontFamily: "monospace" }}>
{`$ curl ${window.location.protocol}//${window.location.hostname}/api/v3/reportExport/${report_id}/comment-groups.csv`}
</p>
<p style={{ wordBreak: "break-all", fontFamily: "monospace" }}>
{`$ curl ${window.location.protocol}//${window.location.hostname}/api/v3/reportExport/${report_id}/format.json`}
</p>
</div>

{doShowDataLicenseTerms && (
Expand Down
1 change: 1 addition & 0 deletions delphi/docs/QUICK_START.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ To work with your own data:

1. Prepare your data in CSV format with the following structure:
- Votes: columns `voter-id`, `comment-id`, and `vote` (values: 1=agree, -1=disagree, 0=pass)
(the Polis export convention; see [docs/export-format.md](../../docs/export-format.md) for every export file's declared sign)
- Comments: columns `comment-id` and `comment-body`

2. Use the Conversation class:
Expand Down
10 changes: 7 additions & 3 deletions delphi/tests/projgate/projection_gate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ def test_wire_gate_negative_control_extra_field(dsn: str) -> None:


def _copy_server_src(dst_root: str) -> str:
"""Copy the two real source files the witness reads into <dst_root>/src."""
"""Copy the real source files the witness reads into <dst_root>/src."""
import shutil

server_dir = pg._resolve_server_dir()
Expand All @@ -569,6 +569,9 @@ def _copy_server_src(dst_root: str) -> str:
src = os.path.join(dst_root, "src")
os.makedirs(os.path.join(src, "routes"), exist_ok=True)
os.makedirs(os.path.join(src, "db"), exist_ok=True)
os.makedirs(os.path.join(src, "votes"), exist_ok=True)
shutil.copy(os.path.join(server_dir, "src", "votes", "convention.ts"),
os.path.join(src, "votes", "convention.ts"))
shutil.copy(os.path.join(server_dir, "src", "routes", "votes.ts"),
os.path.join(src, "routes", "votes.ts"))
shutil.copy(os.path.join(server_dir, "src", "server-helpers.ts"),
Expand Down Expand Up @@ -606,9 +609,10 @@ def _findings(**flt):
return {r.site.name: r for r in _wire_or_skip(dsn, {"zid": PUBLIC_FIXTURE_ZID, "pid": 0}, **kw)}

# Mutation A: flip the served vote sign in the ACTUAL route.
assert "resolve(results.rows);" in votes_orig
served_rows = "resolve(results.rows.map((row) => storageRowToWire(row)));"
assert served_rows in votes_orig
_write(votes_ts, votes_orig.replace(
"resolve(results.rows);",
served_rows,
"resolve(results.rows.map((r) => ({ ...r, vote: -r.vote })));"))
rA = _findings()
assert not rA["votesGet"].ok
Expand Down
10 changes: 10 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

Most application settings are environment variables. Copy the repository-root `example.env` to `.env` and modify it for your launch path; example values are not universal runtime defaults. CDK context and explicit CLI/config arguments are separate configuration inputs.

The data export and vote import file formats, including the vote sign every file declares, are described in [export and import file formats](export-format.md).

The [environment read reference](configuration-env-reference.md) records all 233 named inputs and 680 read sites in the scoped API, Delphi, coordinator and CDK source, including file:line, fallback and secret status. The [deployment reference](deployment-configuration.md) connects those settings to all 153 P065 service entries and explains dynamic/SDK input limits.

</br>
Expand Down Expand Up @@ -106,6 +108,14 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b
- **`ADMIN_EMAILS`** array of email addresses to receive team notifications.
- **`POLIS_FROM_ADDRESS`** email address from which other emails are sent.

### Conversation data export

`GET /api/v3/dataExport?conversation_id=<id>` (conversation owners only) has two modes, chosen per request:

- **`AWS_S3_BUCKET_NAME` set and `OFFLINE` unset** (production): unchanged. The route queues a `generate_export_data` task in `worker_tasks` and answers `{}`; `/api/v3/dataExport/results` signs a URL into the `polis-datadump` bucket. Nothing has consumed those tasks since the Clojure export worker stopped in 2022.
- **`AWS_S3_BUCKET_NAME` unset, or `OFFLINE` set to `1`/`true`**: the server streams the export itself as `polis-export-<conversation_id>-<millis>.zip`, holding the report export's `format.json`, `summary.csv`, `comments.csv`, `votes.csv` and `participant-votes.csv`, each byte-identical to `/api/v3/reportExport/<report_id>/<file>` ([export-format.md](export-format.md#the-conversation-zip)). `/api/v3/dataExport/results` answers 404 `polis_err_data_export_results_not_stored`. The Postgres reads pause while the client is not keeping up, and stop (cursor closed) if the client disconnects. Note that this route treats an unset bucket as a mode switch, while the server's other S3 users fall back to `polis-delphi`: a deployment that drops `AWS_S3_BUCKET_NAME` switches this route to streaming.
- **`DATA_EXPORT_MAX_CELLS`** (default `20000000`) the largest conversation the second mode streams, counted as the larger of its vote rows and its voters × comments. Over it the route answers 413 `polis_err_data_export_too_large` with `cells` and `limit`, before any byte of zip. A value that is not a positive integer means the default. The zip has no zip64 records, so it must stay under 4 GiB; at the default, `votes.csv` is about 2 GB at most.

### Boolean Flags

(All can be left blank, or `false`)
Expand Down
196 changes: 196 additions & 0 deletions docs/export-format.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
# Export and import file formats: the vote sign

Polis exports a conversation as a set of CSV files, and admins can import votes
from a CSV. This page states the sign every vote value in those files uses, how
the files declare it, and the rule the importer applies.

Format id: **`polis-export/1`**.

## The export convention

Every vote value in an export file, and in the votes-bulk import, uses:

| vote | value |
|----------|-------|
| agree | `1` |
| disagree | `-1` |
| pass | `0` |

This is fixed. It does not depend on how the server stores votes internally
(the database has always stored the opposite sign, agree as `-1`; the export
code converts). If the stored sign ever changes, the export files keep this
convention and these bytes.

Written as a declaration, the convention is:

```
agree=+1;disagree=-1;pass=0;format=polis-export/1
```

## Where the declaration lives

CSV has no header comments, so the CSV files keep their column headers and
bytes exactly as before. The declaration is carried beside them:

- **`summary.csv`** has a last key/value row:

```
vote-convention,agree=+1;disagree=-1;pass=0;format=polis-export/1
```

Every row above it is unchanged and in the same position. Consumers that
read `summary.csv` by key are unaffected; consumers that read it by line
number see one more line at the end.

- **`format.json`**, a sidecar served with the other files at
`/api/v3/reportExport/<report_id>/format.json`. It states the format id,
the declaration, the vote values, and for each file which columns carry a
vote value (`vote`), a count of named votes (`count`), an importance flag
(`flag`) or the declaration itself (`declaration`):

```json
{
"format": "polis-export/1",
"vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1",
"vote": { "agree": 1, "disagree": -1, "pass": 0 },
"files": { "votes.csv": { "vote": "vote" }, "...": {} },
"docs": "https://github.com/compdemocracy/polis/blob/edge/docs/export-format.md"
}
```

The report page's "Raw Data Export" section links every file, `format.json`
included. Each file is served from
`/api/v3/reportExport/<report_id>/<file>`, so a downloader who wants the
declaration with the data fetches `format.json` (or `summary.csv`) along with
the CSVs, or takes the conversation zip below, which carries it.

## The conversation zip

`GET /api/v3/dataExport?conversation_id=<id>` (conversation owners only)
streams one zip, `polis-export-<conversation_id>-<millis>.zip`, when the server
has no S3 bucket configured (`AWS_S3_BUCKET_NAME` unset) or runs with
`OFFLINE` set. It holds, in this order:

| file | contents |
|-------------------------|--------------------------------------------|
| `format.json` | the sidecar: format id and declaration |
| `summary.csv` | ends with the `vote-convention` row |
| `comments.csv` | as below |
| `votes.csv` | as below |
| `participant-votes.csv` | as below |

Each file is byte-identical to what
`/api/v3/reportExport/<report_id>/<file>` serves for the same conversation at
the same moment; the zip adds no file and changes no byte. The zip is built
on request and streamed (deflate, no zip64): the database reads pause while
the client is not keeping up and stop if it disconnects. A conversation over the
`DATA_EXPORT_MAX_CELLS` limit is refused with HTTP 413
`polis_err_data_export_too_large` ([configuration](configuration.md#conversation-data-export)).

With a bucket configured and `OFFLINE` unset, the route keeps its older
behaviour: it queues a task for the Clojure math worker's export, which has had
no consumer since that worker stopped; that path is not a source of these files.

## Every file and column

### `summary.csv`

Key/value rows, no header: `topic`, `url`, `voters`, `voters-in-conv`,
`commenters`, `comments`, `groups`, `conversation-description`, then
`vote-convention` (the declaration above). No other row carries a vote value.

### `votes.csv`

One row per vote event: `timestamp`, `datetime`, `comment-id`, `voter-id`,
`vote`, and `important` when importance is enabled.

- `vote`: a vote value in the export convention. A vote stored as NULL is
written `0`, as it always has been.
- `important` (only when importance is enabled): `1` when the vote was
marked important, `0` otherwise. An importance flag, not a vote value.

### `participant-votes.csv`

One row per participant: `participant`, `group-id`, `n-comments`, `n-votes`,
`n-agree`, `n-disagree`, then one column per comment.

- `<comment-id>` columns: the participant's latest vote on that comment in
the export convention; empty when they did not vote on it.
- `n-votes`, `n-agree`, `n-disagree`: counts of votes; no sign involved.

### `participant-importance.csv`

Same layout as `participant-votes.csv` with `n-important` in place of
`n-agree` / `n-disagree`.

- `<comment-id>` columns: `1` when the participant marked the comment
important, `0` when they voted without marking it, empty when they did not
vote. These are importance flags, not vote values.
- `n-votes`, `n-important`: counts.

### `comments.csv`

One row per comment: `timestamp`, `datetime`, `comment-id`, `author-id`,
`agrees`, `disagrees`, `moderated`, then `importance` when enabled, then
`comment-body`.

- `agrees`, `disagrees`: counts of agree and disagree votes; no sign involved.
- `importance` (only when importance is enabled, before `comment-body`): the
number of votes on the comment marked important; no sign involved.

### `comment-groups.csv`

One row per comment: `comment-id`, `comment`, `total-votes`, `total-agrees`,
`total-disagrees`, `total-passes`, then for each opinion group
`group-<letter>-votes`, `group-<letter>-agrees`, `group-<letter>-disagrees`,
`group-<letter>-passes`. All are counts; no sign involved.

### `comment-clusters.csv`

Cluster assignments per comment. No vote values.

### `format.json`

The sidecar described above.

## The import rule (`POST /api/v3/votes-bulk`)

The votes-bulk CSV (`vote_id,user_id,vote_value,timestamp,comment_id`) uses the
export convention for `vote_value`: agree `1`, disagree `-1`, pass `0`. The
importer converts each value to the stored sign through the server's vote
convention module.

A file may declare its sign. The declaration is optional:

- **in the file**, as its first line, before the CSV header:

```
# vote-convention: agree=+1;disagree=-1;pass=0;format=polis-export/1
```

Keys may come in any order; `format` may be omitted. Any other first line
(including another `#` line) is read as the CSV header, as before.

- **beside the file**, as a `format` field in the request body with the shape
of `format.json` (an object, or that object as a JSON string). It must carry
`vote-convention`, `vote`, or both.

The rule:

1. **No declaration**: the file is read in the export convention, exactly as
before this declaration existed.
2. **A declaration that states the export convention** (and, if it names a
format, `polis-export/1`): accepted; the file is read in the export
convention. A first-line declaration is removed before the header is read.
3. **A declaration that states any other convention or format** (for example
`agree=-1;disagree=+1;pass=0`): refused with
`polis_err_vote_convention_declaration_mismatch`. The importer never
converts from a declared sign other than the export convention.
4. **A declaration that cannot be read** (a missing key, an unknown or repeated
key, a value that is not an integer, values that do not form a convention,
a `format` that is not JSON): refused with
`polis_err_vote_convention_declaration_malformed`.

These two codes are the closed set. The route refuses a bad declaration with
HTTP 400 and the code; the import worker checks the stored file's first line
again and fails the job with the same code.
37 changes: 37 additions & 0 deletions server/__tests__/fixtures/vote-declaration/declarations.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"about": "Declaration values for the votes-bulk import and format.json (docs/export-format.md). Each case: the text after `# vote-convention:`, and the outcome the convention module must give. Generated fixture; no conversation data.",
"cases": [
{ "text": "agree=+1;disagree=-1;pass=0;format=polis-export/1", "outcome": "accepted", "format": "polis-export/1" },
{ "text": "agree=1;disagree=-1;pass=0", "outcome": "accepted", "format": null },
{ "text": " pass=0 ; disagree=-1 ; agree=+1 ", "outcome": "accepted", "format": null },
{ "text": "agree=+1;disagree=-1;pass=0;", "outcome": "accepted", "format": null },
{ "text": "agree=-1;disagree=+1;pass=0;format=polis-export/1", "outcome": "mismatch" },
{ "text": "agree=-1;disagree=1;pass=0", "outcome": "mismatch" },
{ "text": "agree=+1;disagree=-1;pass=0;format=polis-export/2", "outcome": "mismatch" },
{ "text": "agree=+1;disagree=-1;pass=0;format=other", "outcome": "mismatch" },
{ "text": "", "outcome": "malformed" },
{ "text": "agree=+1;disagree=-1", "outcome": "malformed" },
{ "text": "agree=+1;disagree=+1;pass=0", "outcome": "malformed" },
{ "text": "agree=+1;disagree=-1;pass=1", "outcome": "malformed" },
{ "text": "agree=2;disagree=-2;pass=0", "outcome": "malformed" },
{ "text": "agree=yes;disagree=no;pass=0", "outcome": "malformed" },
{ "text": "agree=+1;agree=+1;disagree=-1;pass=0", "outcome": "malformed" },
{ "text": "agree=+1;disagree=-1;pass=0;skip=0", "outcome": "malformed" },
{ "text": "agree +1, disagree -1, pass 0", "outcome": "malformed" }
],
"documents": [
{ "doc": { "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1" }, "outcome": "accepted" },
{ "doc": { "format": "polis-export/1", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "accepted" },
{ "doc": { "format": "polis-export/1", "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/1", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "accepted" },
{ "doc": "{\"vote\": {\"agree\": 1, \"disagree\": -1, \"pass\": 0}}", "outcome": "accepted" },
{ "doc": { "vote": { "agree": -1, "disagree": 1, "pass": 0 } }, "outcome": "mismatch" },
{ "doc": { "format": "polis-export/2", "vote": { "agree": 1, "disagree": -1, "pass": 0 } }, "outcome": "mismatch" },
{ "doc": { "vote-convention": "agree=+1;disagree=-1;pass=0", "vote": { "agree": -1, "disagree": 1, "pass": 0 } }, "outcome": "malformed" },
{ "doc": { "format": "polis-export/1", "vote-convention": "agree=+1;disagree=-1;pass=0;format=polis-export/2" }, "outcome": "malformed" },
{ "doc": { "format": "polis-export/1" }, "outcome": "malformed" },
{ "doc": { "vote": [1, -1, 0] }, "outcome": "malformed" },
{ "doc": { "vote-convention": 1 }, "outcome": "malformed" },
{ "doc": "not json", "outcome": "malformed" },
{ "doc": [], "outcome": "malformed" }
]
}
Loading
Loading