From dc6b20066f0f0334fd15060da8a891bd55a959aa Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Tue, 10 Nov 2026 01:26:14 -0800 Subject: [PATCH] polis-api: the database tests also run on the shape of an installation whose database predates the migrations, and four reads no longer depend on the migrations' shape A second pass of the database tests runs with the older shape installed (json where the migrations build jsonb, another column order). Four reads that failed or handed back there are fixed: the next-comment priorities read and the math generation read take the column's type from the column; the votes and participants reads follow the stored column order, as Node's rows do. Bytes on a database built from the migrations are unchanged. No schema change, no migration, no Node change. Co-Authored-By: Claude --- ci/local/polis-api.sh | 19 + docs/local-checks.md | 44 ++ queue-rs/polis-api/README.md | 3 + .../examples/older_database_shape.rs | 46 ++ queue-rs/polis-api/src/main.rs | 3 + queue-rs/polis-api/src/math_bundle.rs | 22 +- .../polis-api/src/math_membership_tests.rs | 188 ++++++++ queue-rs/polis-api/src/next_priorities.rs | 16 +- queue-rs/polis-api/src/next_priority_tests.rs | 116 +++++ .../src/older_database_shape_tests.rs | 420 +++++++++++++++++ queue-rs/polis-api/src/participants_read.rs | 124 +++-- .../polis-api/src/participants_read_tests.rs | 97 +++- queue-rs/polis-api/src/votes_read.rs | 79 ++-- queue-rs/polis-api/src/votes_read_tests.rs | 95 +++- .../polis-api/tests/older_database_shape.sql | 430 ++++++++++++++++++ 15 files changed, 1628 insertions(+), 74 deletions(-) create mode 100644 queue-rs/polis-api/examples/older_database_shape.rs create mode 100644 queue-rs/polis-api/src/older_database_shape_tests.rs create mode 100644 queue-rs/polis-api/tests/older_database_shape.sql diff --git a/ci/local/polis-api.sh b/ci/local/polis-api.sh index ee5659a36..55970180b 100644 --- a/ci/local/polis-api.sh +++ b/ci/local/polis-api.sh @@ -41,12 +41,31 @@ check_project "$IMPORTANCE_PROJECT" # Keep aggregate test pools below PostgreSQL's default connection limit. # Pin this here so local and hosted checks use the same database budget. export RUST_TEST_THREADS=2 + # The first pass runs on test databases as PostgreSQL creates them. An + # interrupted earlier run may have left the second pass's shape installed. + cargo run --locked -p polis-api --example older_database_shape -- remove cargo test --locked -p polis-api --features db-tests # Compare timely replies with full selections from the same run. Execute # the optimized queue test in both local and hosted CI. cargo test --release --locked -p polis-api --features db-tests sustained_votes_keep_delivering -- --nocapture ) +( + cd "$CHECK_ROOT/queue-rs" + export POLIS_API_TEST_DATABASE_URL + export POLIS_AUTH_TEST_DATABASE_URL + export RUST_TEST_THREADS=2 + # The second pass: the same tests, unchanged, on test databases in the shape + # of an installation whose database predates the migrations (json math + # documents, another column order, a few other columns). While + # polis-api/tests/older_database_shape.sql is installed in the test + # database and in template1, the tables it lists take that shape as the + # tests create them. It is removed again whatever the tests answer. + trap 'cargo run --locked -p polis-api --example older_database_shape -- remove' EXIT + cargo run --locked -p polis-api --example older_database_shape -- install + POLIS_API_TEST_DATABASE_SHAPE=older cargo test --locked -p polis-api --features db-tests +) + ( cd "$CHECK_ROOT/queue-rs" cargo build --locked --release -p polis-api diff --git a/docs/local-checks.md b/docs/local-checks.md index 6c83e64fb..dd46b2e40 100644 --- a/docs/local-checks.md +++ b/docs/local-checks.md @@ -115,6 +115,50 @@ These are sampled peaks, not upper bounds. At a shell open-file limit of 256, default parallelism also fails tests with "Too many open files"; two threads pass. Two threads leave headroom without requiring a larger test database; test selection, assertions and concurrency within each test are unchanged. +The API entry point runs its full Rust database test suite twice. The first +pass uses test databases as PostgreSQL creates them, so every table a test +builds has the shape the migrations give it. The second pass runs the same +tests, unchanged, on the shape of an installation whose database predates the +migrations: such a database stores the math documents as `json` where the +migrations build `jsonb`, keeps some columns in another order, has a few +columns the migrations never create and lacks two they do. A statement can +depend on any of that without saying so (a function that exists for `jsonb` +only, a `SELECT *` read by position), and only a database in that shape shows +it. `queue-rs/polis-api/tests/older_database_shape.sql` records the shape of +the thirteen tables that differ and that the API reads, and reshapes a table +with one of those names when it is created; its header says exactly what is +and is not reproduced. Between the passes the entry point installs it in the +test database and in that server's `template1` (every database a test creates +is a copy of `template1`), sets `POLIS_API_TEST_DATABASE_SHAPE=older` for the +second pass, and removes it afterwards, also when the tests fail. Installing +needs a superuser, and `template1` is shared by the whole server, so +`POLIS_API_TEST_DATABASE_URL` must name a server that belongs to this run. One +test in each pass checks that its databases have the shape the pass names, so +a shape left installed by an interrupted run fails the first pass instead of +changing it silently; the first pass also begins by removing it. To run one +pass by hand: + +```sh +cd queue-rs +cargo run --locked -p polis-api --example older_database_shape -- install +POLIS_API_TEST_DATABASE_SHAPE=older RUST_TEST_THREADS=2 \ + cargo test --locked -p polis-api --features db-tests +cargo run --locked -p polis-api --example older_database_shape -- remove +``` + +A test that needs a table in one particular column order, in both passes, +creates it under another name and renames it; a test that loads a table by +position declares fewer columns than the migrations' table or names the +columns it inserts. The release-mode queue test runs in the first pass only. + +A migration that adds a column to one of the thirteen tables also adds it to +`older_database_shape.sql`, as the last row of that table and marked `later` +(the file's header says why). Until it does, +`every_migration_applies_and_the_listed_tables_take_the_older_shape` fails in +both passes and says so. Most tables the tests declare have fewer columns than +the migrations' tables: in the second pass those keep the order the test gave +them, and only their `jsonb` math documents become `json`. + CLI differential needs its `CLI_DIFF_*_IMAGE` variables and an independent `SETUP_ADMISSION_PROJECT`. Mixed identity uses all six `MIXED_*_PORT` / `POLIS_MIXED_WORKER_PORT` values. diff --git a/queue-rs/polis-api/README.md b/queue-rs/polis-api/README.md index 0354a10e0..0f6843363 100644 --- a/queue-rs/polis-api/README.md +++ b/queue-rs/polis-api/README.md @@ -225,6 +225,9 @@ cargo clippy --locked -p polis-api --all-targets -- -D warnings cargo test --locked -p polis-api POLIS_API_TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:25462/scratch \ cargo test --locked -p polis-api --features db-tests # a throwaway Postgres +# `make check-polis-api` runs these database tests a second time on the shape +# of an installation whose database predates the migrations +# (tests/older_database_shape.sql; see docs/local-checks.md). cd .. queue-rs/polis-api/conformance/nginx-routing.sh # needs Docker # Byte-for-byte replay (needs Docker and the local characterization images; diff --git a/queue-rs/polis-api/examples/older_database_shape.rs b/queue-rs/polis-api/examples/older_database_shape.rs new file mode 100644 index 000000000..e895b9875 --- /dev/null +++ b/queue-rs/polis-api/examples/older_database_shape.rs @@ -0,0 +1,46 @@ +//! Installs or removes `tests/older_database_shape.sql` on the test server +//! named by `POLIS_API_TEST_DATABASE_URL`: in that database, where tests +//! create their schemas, and in `template1`, which every database a test +//! creates is copied from. While it is installed, every table the crate's +//! database tests create under one of the names that file lists comes out in +//! the shape of an installation whose database predates the migrations. +//! +//! cargo run --locked -p polis-api --example older_database_shape -- install +//! cargo run --locked -p polis-api --example older_database_shape -- remove +//! +//! `template1` is shared by the whole server, so use this only on a server +//! that belongs to the test run. `remove` is safe to run when nothing is +//! installed. +use anyhow::{Context, Result, bail}; +use postgres::{Client, Config, NoTls}; + +const INSTALL: &str = include_str!("../tests/older_database_shape.sql"); +const REMOVE: &str = "DROP EVENT TRIGGER IF EXISTS older_database_shape; + DROP SCHEMA IF EXISTS older_database_shape CASCADE;"; + +fn main() -> Result<()> { + let action = std::env::args().nth(1).unwrap_or_default(); + let sql = match action.as_str() { + "install" => INSTALL, + "remove" => REMOVE, + _ => bail!("usage: older_database_shape install|remove"), + }; + let url = std::env::var("POLIS_API_TEST_DATABASE_URL") + .context("POLIS_API_TEST_DATABASE_URL names the test database")?; + let tests: Config = url + .parse() + .context("POLIS_API_TEST_DATABASE_URL is not a PostgreSQL URL")?; + let mut template = tests.clone(); + template.dbname("template1"); + for (name, config) in [("the test database", tests), ("template1", template)] { + // Quiet the notices of DROP ... IF EXISTS. + let mut client: Client = config + .connect(NoTls) + .with_context(|| format!("connecting to {name}"))?; + client + .batch_execute(&format!("SET client_min_messages = warning; {sql}")) + .with_context(|| format!("older database shape: {action} in {name}"))?; + } + println!("older database shape: {action} done in the test database and in template1"); + Ok(()) +} diff --git a/queue-rs/polis-api/src/main.rs b/queue-rs/polis-api/src/main.rs index c37dc97ec..7df2bd696 100644 --- a/queue-rs/polis-api/src/main.rs +++ b/queue-rs/polis-api/src/main.rs @@ -2070,3 +2070,6 @@ mod health_tests { #[cfg(all(test, feature = "db-tests"))] mod startup_wiring_tests; + +#[cfg(all(test, feature = "db-tests"))] +mod older_database_shape_tests; diff --git a/queue-rs/polis-api/src/math_bundle.rs b/queue-rs/polis-api/src/math_bundle.rs index 6b01d812b..f51271499 100644 --- a/queue-rs/polis-api/src/math_bundle.rs +++ b/queue-rs/polis-api/src/math_bundle.rs @@ -88,7 +88,15 @@ impl Cache { } } pub fn read(c: &mut postgres::Client, zid: i32, env: &str) -> anyhow::Result> { - let rows=c.query("SELECT m.math_tick,b.math_tick,p.math_tick,t.math_tick,CASE WHEN octet_length(m.data::text)<=65536 THEN m.data::text END,CASE WHEN octet_length(b.data::text)<=65536 THEN b.data::text END,p.data IS NOT NULL AND p.data<>'null'::jsonb FROM math_main m LEFT JOIN math_bidtopid b ON b.zid=m.zid AND b.math_env=m.math_env LEFT JOIN math_ptptstats p ON p.zid=m.zid AND p.math_env=m.math_env LEFT JOIN math_ticks t ON t.zid=m.zid AND t.math_env=m.math_env WHERE m.zid=$1 AND m.math_env=$2 LIMIT 2",&[&zid,&env])?; + // The three `data` columns are jsonb in a database built from the + // migrations and json in installations older than them. `::text` exists + // for both. Whether the participant statistics are a JSON null is asked + // of the column's own type: only that arm of the CASE is evaluated, and + // its cast is to the type the column already has, so no document is + // converted. On jsonb this is the comparison with 'null'::jsonb it + // replaces. A column of another type gives SQL NULL here for a stored + // document, and the read is refused. + let rows=c.query("SELECT m.math_tick,b.math_tick,p.math_tick,t.math_tick,CASE WHEN octet_length(m.data::text)<=65536 THEN m.data::text END,CASE WHEN octet_length(b.data::text)<=65536 THEN b.data::text END,p.data IS NOT NULL AND CASE pg_typeof(p.data) WHEN 'pg_catalog.jsonb'::regtype THEN jsonb_typeof(p.data::jsonb) WHEN 'pg_catalog.json'::regtype THEN json_typeof(p.data::json) END<>'null' FROM math_main m LEFT JOIN math_bidtopid b ON b.zid=m.zid AND b.math_env=m.math_env LEFT JOIN math_ptptstats p ON p.zid=m.zid AND p.math_env=m.math_env LEFT JOIN math_ticks t ON t.zid=m.zid AND t.math_env=m.math_env WHERE m.zid=$1 AND m.math_env=$2 LIMIT 2",&[&zid,&env])?; if rows.len() > 1 { return Ok(None); } @@ -104,10 +112,14 @@ pub fn read(c: &mut postgres::Client, zid: i32, env: &str) -> anyhow::Result>(5)?; let matching = (1..=3).all(|i| r.try_get::<_, Option>(i).ok() == Some(Some(tick))); - if !matching - || mapping.as_deref() == Some("null") - || !r.try_get::<_, Option>(6)?.unwrap_or(false) - { + // SQL NULL only for statistics stored in a column that is neither jsonb + // nor json: the expression is false, never NULL, without a document. + let Some(statistics) = r.try_get::<_, Option>(6)? else { + return Ok(None); + }; + // A json column keeps a document's text as written, so its null may be + // surrounded by white space; a jsonb column always gives `null`. + if !matching || mapping.as_deref().map(str::trim) == Some("null") || !statistics { return Ok(Some(Read::Incoherent(tick))); } let Some(mapping) = mapping else { diff --git a/queue-rs/polis-api/src/math_membership_tests.rs b/queue-rs/polis-api/src/math_membership_tests.rs index fce02be27..e449fa237 100644 --- a/queue-rs/polis-api/src/math_membership_tests.rs +++ b/queue-rs/polis-api/src/math_membership_tests.rs @@ -211,3 +211,191 @@ async fn wait_for_generation( ) }) } + +/// The three math documents are jsonb in a database built from the migrations +/// and json in an installation whose database predates them. The whole +/// generation read gives the same answer on both, and refuses statistics kept +/// in a column of any other type. +#[test] +fn json_and_jsonb_math_columns_give_the_same_generation() { + let url = std::env::var("POLIS_API_TEST_DATABASE_URL").unwrap(); + let mut c = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + let schema = format!("math_bundle_types_{}", std::process::id()); + c.batch_execute(&format!( + "CREATE SCHEMA {schema}; SET search_path={schema}; + CREATE TABLE math_main(zid integer,math_env text,data text,math_tick bigint); + CREATE TABLE math_bidtopid(zid integer,math_env text,data text,math_tick bigint); + CREATE TABLE math_ptptstats(zid integer,math_env text,data text,math_tick bigint); + CREATE TABLE math_ticks(zid integer,math_env text,math_tick bigint);" + )) + .unwrap(); + let main = r#"{"base-clusters":{"id":[42,99]},"group-clusters":[{"members":[42]}],"n":2}"#; + let spaced = "{ \"group-clusters\" : [ { \"members\" : [ 99 ] } ] ,\n\t\"base-clusters\" : { \"id\" : [ 42 , 99 ] } }"; + // (zid, main, mapping, statistics, ticks of mapping, statistics, math_ticks) + type Case = ( + i32, + &'static str, + Option<&'static str>, + Option<&'static str>, + [i64; 3], + ); + let cases: Vec = vec![ + ( + 1, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("{}"), + [7; 3], + ), + ( + 2, + spaced, + Some("{ \"bidToPid\" : [ [ 0 , 1 ] , [ 2 ] ] }"), + Some(" { \"a\" : 1 } "), + [7; 3], + ), + ( + 3, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("null"), + [7; 3], + ), + ( + 4, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some(" null\n"), + [7; 3], + ), + (5, main, Some(r#"{"bidToPid":[[0],[2]]}"#), None, [7; 3]), + (6, main, Some("null"), Some("{}"), [7; 3]), + (7, main, Some("\tnull "), Some("{}"), [7; 3]), + (8, main, None, Some("{}"), [7; 3]), + ( + 9, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("[1]"), + [7, 6, 7], + ), + ( + 10, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("0"), + [7; 3], + ), + ( + 11, + main, + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("\"null\""), + [7; 3], + ), + ( + 12, + "[1]", + Some(r#"{"bidToPid":[[0],[2]]}"#), + Some("false"), + [7; 3], + ), + ]; + let mut reads: Vec> = vec![]; + for data_type in ["jsonb", "json"] { + // The types are set here, whatever shape the test database gives a + // newly created table. + c.batch_execute(&format!( + "TRUNCATE math_main, math_bidtopid, math_ptptstats, math_ticks; + ALTER TABLE math_main ALTER COLUMN data TYPE {data_type} USING data::{data_type}; + ALTER TABLE math_bidtopid ALTER COLUMN data TYPE {data_type} USING data::{data_type}; + ALTER TABLE math_ptptstats ALTER COLUMN data TYPE {data_type} USING data::{data_type};" + )) + .unwrap(); + for (zid, main, mapping, statistics, ticks) in &cases { + c.execute( + &format!("INSERT INTO math_main VALUES($1,'a',$2::text::{data_type},7)"), + &[zid, main], + ) + .unwrap(); + if let Some(mapping) = mapping { + c.execute( + &format!("INSERT INTO math_bidtopid VALUES($1,'a',$2::text::{data_type},$3)"), + &[zid, mapping, &ticks[0]], + ) + .unwrap(); + } + if let Some(statistics) = statistics { + c.execute( + &format!("INSERT INTO math_ptptstats VALUES($1,'a',$2::text::{data_type},$3)"), + &[zid, statistics, &ticks[1]], + ) + .unwrap(); + } + c.execute( + "INSERT INTO math_ticks VALUES($1,'a',$2)", + &[zid, &ticks[2]], + ) + .unwrap(); + } + assert_eq!( + c.query_one( + "SELECT pg_typeof(m.data)::text, pg_typeof(b.data)::text, pg_typeof(p.data)::text + FROM math_main m, math_bidtopid b, math_ptptstats p LIMIT 1", + &[] + ) + .map(|row| [row.get::<_, String>(0), row.get(1), row.get(2)]) + .unwrap(), + [data_type; 3] + ); + reads.push( + (1..=13) + .map(|zid| format!("{:?}", math_bundle::read(&mut c, zid, "a").unwrap())) + .collect(), + ); + } + assert_eq!(reads[0], reads[1], "jsonb and json differ"); + let complete = |read: &str| read.starts_with("Some(Complete("); + for (index, expected) in [ + (1, "complete"), + (2, "complete"), + (3, "incoherent"), + (4, "incoherent"), + (5, "incoherent"), + (6, "incoherent"), + (7, "incoherent"), + (8, "incoherent"), + (9, "incoherent"), + (10, "complete"), + (11, "complete"), + (12, "refused"), + (13, "absent"), + ] { + let read = reads[1][index - 1].as_str(); + let found = if complete(read) { + "complete" + } else if read == "Some(Incoherent(7))" { + "incoherent" + } else if read == "Some(Absent)" { + "absent" + } else if read == "None" { + "refused" + } else { + read + }; + assert_eq!(found, expected, "conversation {index}: {read}"); + } + // The members of the first two generations, read from a json column. + for (zid, pid, bid) in [(1, 2, Some(99)), (2, 1, Some(42)), (2, 2, Some(99))] { + let Some(Read::Complete(bundle)) = math_bundle::read(&mut c, zid, "a").unwrap() else { + panic!("conversation {zid} is a complete generation"); + }; + assert_eq!(bundle.bid(pid), bid); + } + // Statistics in a column that is neither jsonb nor json are refused. + c.batch_execute("ALTER TABLE math_ptptstats ALTER COLUMN data TYPE text") + .unwrap(); + assert!(math_bundle::read(&mut c, 1, "a").unwrap().is_none()); + c.batch_execute(&format!("DROP SCHEMA {schema} CASCADE")) + .unwrap(); +} diff --git a/queue-rs/polis-api/src/next_priorities.rs b/queue-rs/polis-api/src/next_priorities.rs index fd390fbec..19dc9093f 100644 --- a/queue-rs/polis-api/src/next_priorities.rs +++ b/queue-rs/polis-api/src/next_priorities.rs @@ -201,8 +201,22 @@ fn refresh(c: &mut Client, zid: i32, env: &str, cached: Option) -> Result // Explicit parameter types batch Parse/Bind/Execute into one exchange. // CASE does not evaluate or transfer the JSONB projection on a cache hit. // A missing priority field is distinct from a reused generation (SQL NULL). + // + // `data` is jsonb in a database built from the migrations and json in + // installations older than them. Nothing here remembers which: the + // server resolves `->` and the inner CASE against the column it is + // reading, each time the statement runs. Only the arm of the column's + // own type is evaluated, and its cast is to the type the column already + // has, so it does nothing; the other arm's cast, which would convert the + // whole document, never runs. A column of any other type has no `->` and + // fails the statement. let rows = c.query_typed( - "select math_tick, case when math_tick = $3 then null else jsonb_typeof(data) end, \ + "select math_tick, case when math_tick = $3 then null else \ + case pg_typeof(data) \ + when 'pg_catalog.jsonb'::regtype then jsonb_typeof(data::jsonb) \ + when 'pg_catalog.json'::regtype then json_typeof(data::json) \ + end \ + end, \ case when math_tick = $3 then null else (data->'comment-priorities')::text end \ from math_main where zid=$1 and math_env=$2 and math_tick>=0", &[(&zid, Type::INT4), (&env, Type::TEXT), (&tick, Type::INT8)], diff --git a/queue-rs/polis-api/src/next_priority_tests.rs b/queue-rs/polis-api/src/next_priority_tests.rs index 8b7c1b5eb..942918b1c 100644 --- a/queue-rs/polis-api/src/next_priority_tests.rs +++ b/queue-rs/polis-api/src/next_priority_tests.rs @@ -107,3 +107,119 @@ fn overlapping_priority_reads_issue_one_statement() { .get(0); assert_eq!(count, 1, "overlapping callers must share one read"); } + +/// `math_main.data` is jsonb in a database built from the migrations and json +/// in an installation whose database predates them. The priorities read gives +/// the same answer and the same refusal on both, and refuses a column of any +/// other type. A json column keeps a document's text as written (spacing, +/// repeated keys, number spelling); jsonb normalises it. +#[test] +fn json_and_jsonb_math_columns_give_the_same_priorities() { + let mut f = Fixture::new(); + let documents = [ + r#"{"comment-priorities":{"0":2,"1":0.5,"9":4},"unneeded":"wide payload"}"#, + "{ \"n\" : 5,\n\t\"comment-priorities\" : { \"9\" : 4e0 , \"1\":5E-1,\"0\":2.0, \"2\":0, \"3\":null, \"5\":false } ,\r\n \"tids\":[1,2] }", + r#"{"comment-priorities":{"0":1},"comment-priorities":{"0":2,"0":3,"1":7}}"#, + r#"{"comment-priorities":{"1":6,"10":1.50}}"#, + r#"{"comment-priorities":{}}"#, + r#"{"comment-priorities":null}"#, + r#"{"comment-priorities":[1,2]}"#, + r#"{"n":1}"#, + "{}", + "[1]", + "3", + "\"x\"", + "null", + ]; + let mut reads: Vec, String>>> = vec![]; + for data_type in ["jsonb", "json"] { + // The type is set here, whatever shape the test database gives a + // newly created table. + f.client + .batch_execute(&format!( + "DELETE FROM math_main; + ALTER TABLE math_main ALTER COLUMN data TYPE {data_type} USING data::text::{data_type}" + )) + .unwrap(); + for (zid, document) in documents.iter().enumerate() { + f.client + .execute( + &format!("INSERT INTO math_main VALUES($1,'test',$2,$3::text::{data_type})"), + &[&(zid as i32), &(zid as i64 + 5), document], + ) + .unwrap(); + } + let stored: String = f + .client + .query_one("SELECT pg_typeof(data)::text FROM math_main LIMIT 1", &[]) + .unwrap() + .get(0); + assert_eq!(stored, data_type); + reads.push( + (0..=documents.len()) + .map(|zid| { + Priorities::new(4) + .read(&mut f.client, zid as i32, "test") + .map(|data| data.map(|data| js::stringify(&data))) + .map_err(|error| format!("{error:#}")) + }) + .collect(), + ); + } + assert_eq!(reads[0], reads[1], "jsonb and json differ"); + let priorities = |text: &str| Ok(Some(format!("{{\"comment-priorities\":{text}}}"))); + let refused = || Err("unsupported math shape".to_owned()); + let expected = [ + priorities(r#"{"0":2,"1":0.5,"9":4}"#), + priorities(r#"{"0":2,"1":0.5,"2":0,"3":null,"5":false,"9":4}"#), + // The last of a repeated key is the one read, on both types. + priorities(r#"{"0":3,"1":7}"#), + priorities(r#"{"1":6,"10":1.5}"#), + priorities("{}"), + priorities("null"), + priorities("[1,2]"), + Ok(Some("{}".to_owned())), + Ok(Some("{}".to_owned())), + refused(), + refused(), + refused(), + refused(), + // No committed row for the conversation. + Ok(None), + ]; + for (zid, expected) in expected.iter().enumerate() { + assert_eq!(&reads[1][zid], expected, "document {zid}"); + } + // Nothing remembers the column's type: on this one connection the same + // statement gives the same answer after the type changes under it. + f.client + .batch_execute("ALTER TABLE math_main ALTER COLUMN data TYPE jsonb") + .unwrap(); + let cache = Priorities::new(4); + let first = cache.read(&mut f.client, 0, "test").unwrap().unwrap(); + f.client + .batch_execute("ALTER TABLE math_main ALTER COLUMN data TYPE json") + .unwrap(); + assert_eq!( + Priorities::new(4) + .read(&mut f.client, 0, "test") + .unwrap() + .unwrap(), + first + ); + // A generation already held is revalidated without its document on json + // too: the tick is all that is read. + cache.expire(0, "test"); + f.client + .batch_execute("UPDATE math_main SET data='{\"comment-priorities\":{\"0\":9}}' WHERE zid=0") + .unwrap(); + assert_eq!( + cache.read(&mut f.client, 0, "test").unwrap().unwrap(), + first + ); + // A column of any other type is refused by the statement itself. + f.client + .batch_execute("ALTER TABLE math_main ALTER COLUMN data TYPE text") + .unwrap(); + assert!(Priorities::new(4).read(&mut f.client, 0, "test").is_err()); +} diff --git a/queue-rs/polis-api/src/older_database_shape_tests.rs b/queue-rs/polis-api/src/older_database_shape_tests.rs new file mode 100644 index 000000000..0c72a2f8d --- /dev/null +++ b/queue-rs/polis-api/src/older_database_shape_tests.rs @@ -0,0 +1,420 @@ +//! The crate's database tests run twice (ci/local/polis-api.sh): once on test +//! databases as PostgreSQL creates them, where every table has the shape the +//! migrations give it, and once with `tests/older_database_shape.sql` +//! installed on the test server, where the tables it lists take the shape of +//! an installation whose database predates the migrations. These are the +//! tests of that file, and the test that says which of the two a run is. +use postgres::{Client, Config, NoTls}; + +const SHAPE: &str = include_str!("../tests/older_database_shape.sql"); +const WITHOUT: &str = "DROP EVENT TRIGGER IF EXISTS older_database_shape; + DROP SCHEMA IF EXISTS older_database_shape CASCADE;"; +/// Unset or `migrations`: the test server is as PostgreSQL made it. `older`: +/// the shape file is installed in the test database and in `template1`. +const SETTING: &str = "POLIS_API_TEST_DATABASE_SHAPE"; +/// What to do when a migration changes the columns of a table the shape file +/// lists. Said by every assertion such a migration trips. +const A_MIGRATION_CHANGED_A_LISTED_TABLE: &str = "the columns the migrations build for a table that tests/older_database_shape.sql lists are not the columns that file records. If a migration added a column, add it to the file as the last row of that table and mark it 'later' (never 'first', which is for 000000_initial.sql alone), and, for conversations or participants_extended, add it to the list of last columns in this test"; +/// What to do when a run finds the other shape than the one it names. +const WRONG_SHAPE: &str = "this run's test databases do not have the shape it names. An interrupted run can leave the older shape installed: remove it with `cargo run --locked -p polis-api --example older_database_shape -- remove`; to run on the older shape, install it and set POLIS_API_TEST_DATABASE_SHAPE=older"; + +fn config() -> Config { + std::env::var("POLIS_API_TEST_DATABASE_URL") + .unwrap() + .parse() + .unwrap() +} + +/// A database of this test's own, removed when the value is dropped. +struct Database { + admin: Client, + name: String, + client: Client, +} +impl Database { + fn create(label: &str) -> Self { + let mut config = config(); + let mut admin = config.connect(NoTls).unwrap(); + let name = format!("older_shape_{label}_{}", std::process::id()); + admin + .batch_execute(&format!("CREATE DATABASE {name}")) + .unwrap(); + config.dbname(&name); + let mut client = config.connect(NoTls).unwrap(); + client + .batch_execute("SET client_min_messages = warning") + .unwrap(); + Self { + admin, + name, + client, + } + } +} +impl Drop for Database { + fn drop(&mut self) { + // The database cannot be dropped while this connection is open. + if let Ok(closed) = config().connect(NoTls) { + drop(std::mem::replace(&mut self.client, closed)); + } + let _ = self + .admin + .batch_execute(&format!("DROP DATABASE {} WITH (FORCE)", self.name)); + } +} + +/// The type a math document column declared jsonb has once its table exists. +fn declared_jsonb(c: &mut Client, schema: &str) -> String { + c.batch_execute(&format!( + "CREATE SCHEMA {schema}; + CREATE TABLE {schema}.math_main(zid integer, data jsonb);" + )) + .unwrap(); + let stored = c + .query_one( + "SELECT format_type(atttypid, atttypmod) FROM pg_attribute + WHERE attrelid = ($1 || '.math_main')::regclass AND attname = 'data'", + &[&schema], + ) + .unwrap() + .get(0); + c.batch_execute(&format!("DROP SCHEMA {schema} CASCADE")) + .unwrap(); + stored +} + +/// A run says which shape its test databases have, and they have it: both the +/// test database, where tests create their schemas, and a database a test +/// creates. A shape file left installed by an interrupted run fails this in +/// the run that expects the migrations' shape, and a run that expects the +/// older shape fails it when nothing is installed. +#[test] +fn test_databases_have_the_shape_this_run_names() { + let expected = match std::env::var(SETTING).ok().as_deref() { + None | Some("" | "migrations") => "jsonb", + Some("older") => "json", + Some(other) => panic!("{SETTING} must be unset, migrations or older; found {other}"), + }; + let mut c = config().connect(NoTls).unwrap(); + assert_eq!( + declared_jsonb(&mut c, &format!("shape_named_{}", std::process::id())), + expected, + "a table created in the test database: {WRONG_SHAPE}" + ); + let mut created = Database::create("named"); + assert_eq!( + declared_jsonb(&mut created.client, "shape_named"), + expected, + "a table created in a database a test creates: {WRONG_SHAPE}" + ); +} + +/// Every migration file, in order. +fn migrations() -> Vec<(String, String)> { + let directory = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../server/postgres/migrations" + ); + let mut files: Vec<(String, String)> = std::fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap()) + .filter(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + name.starts_with('0') && name.ends_with(".sql") + }) + .map(|entry| { + ( + entry.file_name().to_string_lossy().into_owned(), + std::fs::read_to_string(entry.path()).unwrap(), + ) + }) + .collect(); + files.sort(); + assert!(files.len() >= 36, "the migrations were not found"); + files +} + +/// One migration file. An index built CONCURRENTLY cannot share a +/// transaction with another statement, so a file of those (one statement to +/// a line, as the migration runner requires) is sent a statement at a time. +fn apply(c: &mut Client, sql: &str) -> Result<(), postgres::Error> { + let statements: Vec<&str> = sql + .lines() + .filter(|line| !line.trim().is_empty() && !line.trim_start().starts_with("--")) + .collect(); + if statements + .iter() + .all(|statement| statement.starts_with("CREATE INDEX CONCURRENTLY")) + { + statements + .into_iter() + .try_for_each(|statement| c.batch_execute(statement)) + } else { + c.batch_execute(sql) + } +} + +fn lines(c: &mut Client, sql: &str) -> Vec { + c.query(sql, &[]) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect() +} + +/// Every column of the public tables: table, position, name, type, NOT NULL. +const COLUMNS: &str = + "SELECT c.relname || ' ' || row_number() OVER (PARTITION BY c.oid ORDER BY a.attnum) + || ' ' || a.attname || ' ' || format_type(a.atttypid, a.atttypmod) + || CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END + FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid + WHERE c.relnamespace = 'public'::regnamespace AND c.relkind = 'r' + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY c.relname, a.attnum"; + +/// Everything else the migrations build on those tables, and the functions. +const OTHER_OBJECTS: &str = "SELECT line FROM ( + SELECT 'constraint ' || conrelid::regclass || ' ' || conname || ' ' || pg_get_constraintdef(oid) AS line + FROM pg_constraint WHERE connamespace = 'public'::regnamespace AND contype IN ('p','u','f','c','x') + UNION ALL SELECT 'index ' || tablename || ' ' || indexname || ' ' || indexdef + FROM pg_indexes WHERE schemaname = 'public' + UNION ALL SELECT 'default ' || c.relname || ' ' || a.attname || ' ' || pg_get_expr(d.adbin, d.adrelid) + FROM pg_attrdef d JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum + JOIN pg_class c ON c.oid = d.adrelid WHERE c.relnamespace = 'public'::regnamespace + UNION ALL SELECT 'owned sequence ' || s.relname || ' ' || t.relname || ' ' || a.attname + FROM pg_depend dp JOIN pg_class s ON s.oid = dp.objid AND s.relkind = 'S' + JOIN pg_class t ON t.oid = dp.refobjid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = dp.refobjsubid + WHERE dp.deptype = 'a' AND t.relnamespace = 'public'::regnamespace + UNION ALL SELECT 'trigger ' || tgrelid::regclass || ' ' || tgname || ' ' || pg_get_triggerdef(oid) + FROM pg_trigger WHERE NOT tgisinternal + UNION ALL SELECT 'rule ' || tablename || ' ' || rulename || ' ' || definition + FROM pg_rules WHERE schemaname = 'public' + UNION ALL SELECT 'function ' || p.oid::regprocedure || ' ' || md5(p.prosrc) + FROM pg_proc p WHERE p.pronamespace = 'public'::regnamespace + ) objects ORDER BY line"; + +/// The shape file against the migrations themselves: every migration applies +/// with it installed; the thirteen tables come out with the columns, order, +/// types and nullability the file records; and nothing else differs from a +/// database built without it, except what the file says it changes. +#[test] +fn every_migration_applies_and_the_listed_tables_take_the_older_shape() { + let mut older = Database::create("installed"); + let mut plain = Database::create("plain"); + // Explicit in both, whatever the test server's `template1` holds. + older.client.batch_execute(SHAPE).unwrap(); + plain.client.batch_execute(WITHOUT).unwrap(); + for (name, sql) in migrations() { + for database in [&mut older, &mut plain] { + apply(&mut database.client, &sql) + .unwrap_or_else(|error| panic!("{name} in {}: {error:?}", database.name)); + } + } + + // The columns the file records, as the tables now have them. + let recorded = lines( + &mut older.client, + "SELECT table_name || ' ' || row_number() OVER (PARTITION BY table_name ORDER BY position) + || ' ' || column_name || ' ' || regexp_replace(definition, ' DEFAULT .*', '') + FROM older_database_shape.columns ORDER BY table_name, position", + ); + let tables: Vec = lines( + &mut older.client, + "SELECT DISTINCT table_name FROM older_database_shape.columns ORDER BY 1", + ); + assert_eq!(tables.len(), 13); + let listed = |line: &String| { + tables + .iter() + .any(|table| line.starts_with(&format!("{table} "))) + }; + let built: Vec = lines(&mut older.client, COLUMNS) + .into_iter() + .filter(listed) + .collect(); + // A column a later migration adds stays where that migration puts it, so + // two tables end in another order than the file records. + let later = |table: &str| { + let differing = |lines: &[String]| -> Vec { + lines + .iter() + .filter(|line| line.starts_with(&format!("{table} "))) + .cloned() + .collect() + }; + (differing(&recorded), differing(&built)) + }; + let names = |lines: &[String]| -> Vec { + lines + .iter() + .map(|line| line.split(' ').nth(2).unwrap().to_owned()) + .collect() + }; + let (recorded_conversations, built_conversations) = later("conversations"); + assert_eq!( + names(&built_conversations)[45..], + ["xid_required", "topics_enabled"], + "conversations: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + assert_eq!( + names(&recorded_conversations)[45..], + ["topics_enabled", "xid_required"], + "conversations: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + assert_eq!( + names(&built_conversations)[..45], + names(&recorded_conversations)[..45], + "conversations: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + let (recorded_extended, built_extended) = later("participants_extended"); + assert_eq!( + names(&built_extended), + [ + "uid", + "zid", + "referrer", + "parent_url", + "created", + "encrypted_ip_address", + "encrypted_x_forwarded_for", + "country_iso_code", + "modified", + "show_translation_activated", + "subscribe_email", + "permanent_cookie", + "origin" + ], + "participants_extended: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + assert_eq!( + recorded_extended.len(), + built_extended.len(), + "participants_extended: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + let elsewhere = |line: &&String| { + !line.starts_with("conversations ") && !line.starts_with("participants_extended ") + }; + assert_eq!( + built.iter().filter(elsewhere).collect::>(), + recorded.iter().filter(elsewhere).collect::>(), + "the other eleven tables have exactly the recorded columns, in order: {A_MIGRATION_CHANGED_A_LISTED_TABLE}" + ); + // The type and nullability of every column of the two, wherever it sits. + let unordered = |lines: &[String]| -> Vec { + let mut columns: Vec = lines + .iter() + .map(|line| line.splitn(3, ' ').nth(2).unwrap().to_owned()) + .collect(); + columns.sort(); + columns + }; + assert_eq!( + unordered(&built_conversations), + unordered(&recorded_conversations) + ); + assert_eq!(unordered(&built_extended), unordered(&recorded_extended)); + + // The tables the file does not list are untouched. + let unlisted = |c: &mut Client| -> Vec { + lines(c, COLUMNS) + .into_iter() + .filter(|line| !listed(line)) + .collect() + }; + assert_eq!(unlisted(&mut older.client), unlisted(&mut plain.client)); + + // Constraints, indexes, defaults, owned sequences, triggers, rules and + // functions are the migrations', but for the columns only one side has. + let other = |c: &mut Client| lines(c, OTHER_OBJECTS); + let (with, without) = (other(&mut older.client), other(&mut plain.client)); + let only_with: Vec<&String> = with.iter().filter(|line| !without.contains(line)).collect(); + let only_without: Vec<&String> = without.iter().filter(|line| !with.contains(line)).collect(); + assert_eq!( + only_with, + [ + "default comments curation 0", + "default conversations is_curated false", + "default conversations need_suzinvite false", + "default users test false" + ] + ); + assert_eq!( + only_without, + [ + "default conversations branding_type 1", + "default math_ticks caching_tick 0" + ] + ); + + // A table a test declares with fewer columns keeps its order; only a + // jsonb math document becomes json. One declared with every column of + // the first migration is the migrations' table and is reordered. + older + .client + .batch_execute( + "CREATE SCHEMA declared; SET search_path = declared; + CREATE TABLE math_main(zid integer, math_env text, math_tick bigint, data jsonb); + CREATE TABLE votes(tid integer, zid integer, created bigint, pid integer); + CREATE TABLE participants(pid integer, uid integer, zid integer, vote_count integer, + last_interaction bigint, subscribed integer, last_notified bigint, nsli smallint, + mod integer, created bigint, PRIMARY KEY (zid, pid));", + ) + .unwrap(); + let declared = |c: &mut Client, table: &str| -> Vec { + lines( + c, + &format!( + "SELECT a.attname || ' ' || format_type(a.atttypid, a.atttypmod) + FROM pg_attribute a WHERE a.attrelid = 'declared.{table}'::regclass + AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum" + ), + ) + }; + assert_eq!( + declared(&mut older.client, "math_main"), + [ + "zid integer", + "math_env text", + "math_tick bigint", + "data json" + ] + ); + assert_eq!( + declared(&mut older.client, "votes"), + [ + "tid integer", + "zid integer", + "created bigint", + "pid integer" + ] + ); + assert_eq!( + names( + &declared(&mut older.client, "participants") + .iter() + .map(|line| format!("participants 0 {line}")) + .collect::>() + ), + [ + "pid", + "uid", + "zid", + "created", + "vote_count", + "last_interaction", + "subscribed", + "last_notified", + "mod", + "nsli" + ] + ); + assert_eq!( + lines( + &mut older.client, + "SELECT pg_get_constraintdef(oid) FROM pg_constraint + WHERE conrelid = 'declared.participants'::regclass AND contype = 'p'" + ), + ["PRIMARY KEY (zid, pid)"] + ); +} diff --git a/queue-rs/polis-api/src/participants_read.rs b/queue-rs/polis-api/src/participants_read.rs index 08154ec28..b0c93c667 100644 --- a/queue-rs/polis-api/src/participants_read.rs +++ b/queue-rs/polis-api/src/participants_read.rs @@ -69,59 +69,125 @@ pub async fn handle(app: &App, req: &Request, res: Res) -> Outgoing { if zid <= 0 { return None; } - pool.with(move |c| read(c, zid, uid.0)).await.ok().flatten() + pool.with(move |c| read_as_stored(c, zid, uid.0)) + .await + .ok() + .flatten() }) .await; match result { - Ok(Some(response)) => match serde_json::to_string(&response) { + Ok(Some((response, order))) => match body(&response, &order) { Ok(body) => route::json_text(req, res, 200, body), Err(_) => route::bad_gateway(), }, _ => route::bad_gateway(), } } +/// The keys of a participant in a database built from the migrations: the +/// order of `contract::Participant`. +const MIGRATIONS_ORDER: [&str; 10] = [ + "pid", + "uid", + "zid", + "vote_count", + "last_interaction", + "subscribed", + "last_notified", + "nsli", + "mod", + "created", +]; +/// The answer, and the keys of its participant in the order the table stores +/// its columns (none without a participant). +type Stored = (contract::Response, Vec<&'static str>); +/// The answer's bytes. Node answers with its `SELECT p.*` row, so its keys +/// are the columns of `participants` in their stored order. In a database +/// built from the migrations that is the contract's own order and the +/// contract's own encoding; an installation older than the migrations stores +/// the same columns in another order, and the same values are written in that +/// one. +fn body(response: &contract::Response, order: &[&'static str]) -> serde_json::Result { + let Some(participant) = response else { + return serde_json::to_string(response); + }; + if order == MIGRATIONS_ORDER { + return serde_json::to_string(response); + } + let value = serde_json::to_value(participant)?; + let mut body = String::from("{"); + for (index, key) in order.iter().enumerate() { + if index > 0 { + body.push(','); + } + body.push_str(&serde_json::to_string(key)?); + body.push(':'); + body.push_str(&serde_json::to_string(&value[*key])?); + } + body.push('}'); + Ok(body) +} /// Duplicate projected vote_count replaces its original value without moving /// that property's position. Multiple rows or schema extensions stay on Node. +/// The handler reads through `read_as_stored`; this is its typed result. +#[cfg(all(test, feature = "db-tests"))] pub fn read( c: &mut postgres::Client, zid: i32, uid: i32, ) -> anyhow::Result> { + Ok(read_as_stored(c, zid, uid)?.map(|(response, _)| response)) +} +/// `read`, with the order the table stores its columns in. A schema extension +/// is a column added or missing; the order of the columns is not one, and the +/// keys of the answer follow it, as Node's do. +fn read_as_stored(c: &mut postgres::Client, zid: i32, uid: i32) -> anyhow::Result> { let rows=c.query("SELECT p.*, (SELECT CASE WHEN COUNT(*)=0 AND p.vote_count IS NULL THEN NULL ELSE COUNT(*)::integer END FROM votes v WHERE v.zid=p.zid AND v.pid=p.pid) AS vote_count FROM participants p WHERE uid=$1 AND zid=$2 LIMIT 2",&[&uid,&zid])?; if rows.len() > 1 { return Ok(None); } let Some(r) = rows.first() else { - return Ok(Some(None)); + return Ok(Some((None, vec![]))); + }; + // The table's columns, then the projected count under the name of one of + // them. + let names: Vec<&str> = r.columns().iter().map(|c| c.name()).collect(); + let Some((&"vote_count", table)) = names.split_last() else { + return Ok(None); }; - let expected = [ - "pid", - "uid", - "zid", - "vote_count", - "last_interaction", - "subscribed", - "last_notified", - "nsli", - "mod", - "created", - "vote_count", - ]; - if r.columns().iter().map(|c| c.name()).collect::>() != expected { + let mut order = Vec::with_capacity(table.len()); + for name in table { + let Some(known) = MIGRATIONS_ORDER.iter().find(|known| *known == name) else { + return Ok(None); + }; + if order.contains(known) { + return Ok(None); + } + order.push(*known); + } + if order.len() != MIGRATIONS_ORDER.len() { return Ok(None); } - Ok(Some(Some(contract::Participant { - pid: r.try_get(0)?, - uid: r.try_get(1)?, - zid: r.try_get(2)?, - vote_count: r.try_get(10)?, - last_interaction: r.try_get::<_, Option>(4)?.map(|n| n.to_string()), - subscribed: r.try_get(5)?, - last_notified: r.try_get::<_, Option>(6)?.map(|n| n.to_string()), - nsli: r.try_get(7)?, - moderation: r.try_get(8)?, - created: r.try_get::<_, Option>(9)?.map(|n| n.to_string()), - }))) + Ok(Some(( + Some(contract::Participant { + pid: r.try_get("pid")?, + uid: r.try_get("uid")?, + zid: r.try_get("zid")?, + vote_count: r.try_get(table.len())?, + last_interaction: r + .try_get::<_, Option>("last_interaction")? + .map(|n| n.to_string()), + subscribed: r.try_get("subscribed")?, + last_notified: r + .try_get::<_, Option>("last_notified")? + .map(|n| n.to_string()), + nsli: r.try_get("nsli")?, + moderation: r.try_get("mod")?, + created: r + .try_get::<_, Option>("created")? + .map(|n| n.to_string()), + }), + order, + ))) } #[cfg(all(test, feature = "db-tests"))] #[path = "participants_read_tests.rs"] diff --git a/queue-rs/polis-api/src/participants_read_tests.rs b/queue-rs/polis-api/src/participants_read_tests.rs index 6e0fc1e08..a6edc97fd 100644 --- a/queue-rs/polis-api/src/participants_read_tests.rs +++ b/queue-rs/polis-api/src/participants_read_tests.rs @@ -7,7 +7,7 @@ async fn participant_projection_counts_history_preserves_bigints_and_sheds_locks let setup_schema = schema.clone(); tokio::task::spawn_blocking(move||{ let mut c=postgres::Client::connect(&setup_url,postgres::NoTls).unwrap(); - c.batch_execute(&format!("CREATE SCHEMA {setup_schema}; SET search_path={setup_schema}; CREATE TABLE participants(pid integer,uid integer,zid integer,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,nsli smallint,mod integer,created bigint); CREATE TABLE votes(zid integer,pid integer,vote smallint); INSERT INTO participants VALUES(0,1,1,999,9223372036854775807,2,NULL,3,1,1700000000000),(1,2,1,NULL,0,0,NULL,0,0,NULL);")).unwrap(); + c.batch_execute(&format!("CREATE SCHEMA {setup_schema}; SET search_path={setup_schema}; CREATE TABLE participants(pid integer,uid integer,zid integer,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,nsli smallint,mod integer,created bigint); CREATE TABLE votes(zid integer,pid integer,vote smallint); INSERT INTO participants(pid,uid,zid,vote_count,last_interaction,subscribed,last_notified,nsli,mod,created) VALUES(0,1,1,999,9223372036854775807,2,NULL,3,1,1700000000000),(1,2,1,NULL,0,0,NULL,0,0,NULL);")).unwrap(); let vote=crate::votes::StorageConvention::CURRENT.encode(crate::votes::VoteChoice::Agree) as i16; c.execute("INSERT INTO votes(zid,pid,vote) VALUES(1,0,$1),(1,0,$1)",&[&vote]).unwrap(); }).await.unwrap(); @@ -94,3 +94,98 @@ async fn participant_projection_counts_history_preserves_bigints_and_sheds_locks .await .unwrap(); } + +/// Node answers with its `SELECT p.*` row, so the keys of its answer are the +/// columns of `participants` in their stored order. A database built from the +/// migrations and an installation whose database predates them store those +/// columns in different orders. The answer follows the stored order in both, +/// and a table with another set of columns is handed back. +#[test] +fn participant_keys_follow_the_stored_column_order_as_node_does() { + let url = std::env::var("POLIS_API_TEST_DATABASE_URL").unwrap(); + let mut c = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + let cases = [ + ( + "pid integer,uid integer,zid integer,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,nsli smallint,mod integer,created bigint", + Some( + r#"{"pid":0,"uid":1,"zid":1,"vote_count":2,"last_interaction":"9223372036854775807","subscribed":2,"last_notified":null,"nsli":3,"mod":1,"created":"1700000000000"}"#, + ), + ), + ( + "pid integer,uid integer,zid integer,created bigint,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,mod integer,nsli smallint", + Some( + r#"{"pid":0,"uid":1,"zid":1,"created":"1700000000000","vote_count":2,"last_interaction":"9223372036854775807","subscribed":2,"last_notified":null,"mod":1,"nsli":3}"#, + ), + ), + // Another set of columns is a schema this reader does not model. + ( + "pid integer,uid integer,zid integer,created bigint,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,mod integer,nsli smallint,added integer", + None, + ), + ( + "pid integer,uid integer,zid integer,created bigint,vote_count integer,last_interaction bigint,subscribed integer,last_notified bigint,mod integer", + None, + ), + ]; + for (index, (columns, expected)) in cases.into_iter().enumerate() { + let schema = format!("participants_read_order_{}_{index}", std::process::id()); + let nsli = if columns.contains("nsli") { + ("nsli,", "3,") + } else { + ("", "") + }; + // The table is created under another name and renamed, so that it + // keeps the order written here in every test database. + c.batch_execute(&format!( + "CREATE SCHEMA {schema}; SET search_path={schema}; + CREATE TABLE participants_as_written({columns}); + ALTER TABLE participants_as_written RENAME TO participants; + CREATE TABLE votes(zid integer,pid integer,vote smallint); + INSERT INTO participants({}pid,uid,zid,vote_count,last_interaction,subscribed,last_notified,mod,created) + VALUES({}0,1,1,999,9223372036854775807,2,NULL,1,1700000000000);", + nsli.0, nsli.1 + )) + .unwrap(); + let vote = + crate::votes::StorageConvention::CURRENT.encode(crate::votes::VoteChoice::Agree) as i16; + c.execute( + "INSERT INTO votes(zid,pid,vote) VALUES(1,0,$1),(1,0,$1)", + &[&vote], + ) + .unwrap(); + let answer = read_as_stored(&mut c, 1, 1) + .unwrap() + .map(|(response, order)| body(&response, &order).unwrap()); + assert_eq!(answer.as_deref(), expected, "{columns}"); + // No participant is `null` whatever the order. + if expected.is_some() { + let (response, order) = read_as_stored(&mut c, 1, 99).unwrap().unwrap(); + assert_eq!(body(&response, &order).unwrap(), "null"); + } + c.batch_execute(&format!("DROP SCHEMA {schema} CASCADE")) + .unwrap(); + } + // In the migrations' order the bytes are the contract's own encoding. + let participant = contract::Participant { + pid: Some(0), + uid: Some(1), + zid: Some(1), + vote_count: None, + last_interaction: Some("0".into()), + subscribed: Some(0), + last_notified: None, + nsli: Some(0), + moderation: Some(0), + created: None, + }; + let reversed: Vec<&'static str> = MIGRATIONS_ORDER.iter().rev().copied().collect(); + let participant = Some(participant); + assert_eq!( + body(&participant, &MIGRATIONS_ORDER).unwrap(), + serde_json::to_string(&participant).unwrap() + ); + assert_eq!( + body(&participant, &reversed).unwrap(), + r#"{"created":null,"mod":0,"nsli":0,"last_notified":null,"subscribed":0,"last_interaction":"0","vote_count":null,"zid":1,"uid":1,"pid":0}"# + ); +} diff --git a/queue-rs/polis-api/src/votes_read.rs b/queue-rs/polis-api/src/votes_read.rs index 8530c18fe..8f90e0d9a 100644 --- a/queue-rs/polis-api/src/votes_read.rs +++ b/queue-rs/polis-api/src/votes_read.rs @@ -187,7 +187,9 @@ pub async fn handle(app: &App, req: &Request, res: Res) -> Outgoing { } } /// No order is promised by either Node query. More than one row is therefore -/// handed back, as are noncanonical capability aliases and schema extensions. +/// handed back, as are noncanonical capability aliases and schema extensions +/// (a column added or missing). The order of the columns is not an extension: +/// the keys of the answer follow it, as Node's do. /// A LIMIT bounds materialization without inventing a page or a sort order. pub fn read( c: &mut postgres::Client, @@ -216,46 +218,49 @@ pub fn read( return Ok(None); } let row = &rows[0]; - let expected = if mine { - vec![ - "zid", - "pid", - "tid", - "vote", - "weight_x_32767", - "created", - "high_priority", - ] - } else { - vec!["zid", "pid", "tid", "vote", "weight_x_32767", "modified"] - }; - if row.columns().iter().map(|c| c.name()).collect::>() != expected { + // Node answers with its `SELECT *` rows: the keys are the table's columns + // in their stored order, without `zid`, followed by the keys Node adds. A + // database built from the migrations and an installation older than them + // store the columns of `votes` in different orders, and Node's bytes + // differ accordingly, so the keys here follow the stored order too. The + // set of columns must still be exactly the one modelled. + let time = if mine { "created" } else { "modified" }; + let mut expected = vec!["zid", "pid", "tid", "vote", "weight_x_32767", time]; + if mine { + expected.push("high_priority"); + } + expected.sort_unstable(); + let stored: Vec<&str> = row.columns().iter().map(|c| c.name()).collect(); + let mut sorted = stored.clone(); + sorted.sort_unstable(); + if sorted != expected { return Ok(None); } let mut out = Obj::new(); - out.set("pid", Js::Num(f64::from(row.try_get::<_, i32>("pid")?))); - out.set("tid", Js::Num(f64::from(row.try_get::<_, i32>("tid")?))); - let raw = row.try_get::<_, Option>("vote")?; - let wire = crate::votes::StorageConvention::CURRENT.wire_preserving_invalid(raw); - out.set( - "vote", - wire.map(|v| Js::Num(f64::from(v))).unwrap_or(Js::Null), - ); - out.set( - "weight_x_32767", - row.try_get::<_, Option>("weight_x_32767")? - .map(|v| Js::Num(f64::from(v))) - .unwrap_or(Js::Null), - ); - let time = if mine { "created" } else { "modified" }; - out.set( - time, - row.try_get::<_, Option>(time)? - .map(|v| Js::Str(v.to_string())) - .unwrap_or(Js::Null), - ); + for name in stored { + let value = match name { + // Node removes it before it answers. + "zid" => continue, + "pid" | "tid" => Js::Num(f64::from(row.try_get::<_, i32>(name)?)), + "vote" => { + let raw = row.try_get::<_, Option>(name)?; + let wire = crate::votes::StorageConvention::CURRENT.wire_preserving_invalid(raw); + wire.map(|v| Js::Num(f64::from(v))).unwrap_or(Js::Null) + } + "weight_x_32767" => row + .try_get::<_, Option>(name)? + .map(|v| Js::Num(f64::from(v))) + .unwrap_or(Js::Null), + "created" | "modified" => row + .try_get::<_, Option>(name)? + .map(|v| Js::Str(v.to_string())) + .unwrap_or(Js::Null), + "high_priority" => Js::Bool(row.try_get(name)?), + _ => return Ok(None), + }; + out.set(name, value); + } if mine { - out.set("high_priority", Js::Bool(row.try_get("high_priority")?)); out.set("weight", Js::Null); } out.set("conversation_id", Js::Str(cid.into())); diff --git a/queue-rs/polis-api/src/votes_read_tests.rs b/queue-rs/polis-api/src/votes_read_tests.rs index 413992761..172cb9f2f 100644 --- a/queue-rs/polis-api/src/votes_read_tests.rs +++ b/queue-rs/polis-api/src/votes_read_tests.rs @@ -10,7 +10,10 @@ async fn database_rows_convention_null_history_bounds_and_isolated_pressure() { c.batch_execute(&format!("CREATE SCHEMA {setup_schema};SET search_path={setup_schema};CREATE TABLE zinvites(zid integer,zinvite text);CREATE TABLE votes_latest_unique(zid integer,pid integer,tid integer,vote smallint,weight_x_32767 smallint,modified bigint);CREATE TABLE votes(zid integer,pid integer,tid integer,vote smallint,weight_x_32767 smallint,created bigint,high_priority boolean);INSERT INTO zinvites VALUES(1,'fixture');")).unwrap(); let stored=crate::votes::StorageConvention::CURRENT.encode(crate::votes::VoteChoice::Agree) as i16; c.execute("INSERT INTO votes_latest_unique VALUES(1,0,0,$1,23,9223372036854775807)",&[&stored]).unwrap(); - c.execute("INSERT INTO votes VALUES(1,0,0,$1,23,9223372036854775807,true)",&[&stored]).unwrap(); + // The columns are named: this table has every column of the migrations' + // `votes`, so a test database in the shape of an older installation + // stores them in that installation's order. + c.execute("INSERT INTO votes(zid,pid,tid,vote,weight_x_32767,created,high_priority) VALUES(1,0,0,$1,23,9223372036854775807,true)",&[&stored]).unwrap(); }).await.unwrap(); let make_pool = || { let mut c: postgres::Config = url.parse().unwrap(); @@ -143,3 +146,93 @@ async fn database_rows_convention_null_history_bounds_and_isolated_pressure() { .await .unwrap(); } + +/// Node answers with its `SELECT *` rows, so the keys of its answer are the +/// columns of `votes` in their stored order. A database built from the +/// migrations and an installation whose database predates them store those +/// columns in different orders. The answer follows the stored order in both, +/// and a table with another set of columns is handed back. +#[test] +fn history_keys_follow_the_stored_column_order_as_node_does() { + let url = std::env::var("POLIS_API_TEST_DATABASE_URL").unwrap(); + let mut c = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + let stored = + crate::votes::StorageConvention::CURRENT.encode(crate::votes::VoteChoice::Agree) as i16; + let wire = crate::votes::WIRE_AGREE; + let migrations = "zid integer,pid integer,tid integer,vote smallint,weight_x_32767 smallint,created bigint,high_priority boolean"; + let older = "zid integer,pid integer,tid integer,vote smallint,created bigint,weight_x_32767 smallint,high_priority boolean"; + let cases = [ + ( + migrations, + Some(format!( + r#"[{{"pid":0,"tid":4,"vote":{wire},"weight_x_32767":23,"created":"9223372036854775807","high_priority":true,"weight":null,"conversation_id":"fixture"}}]"# + )), + ), + ( + older, + Some(format!( + r#"[{{"pid":0,"tid":4,"vote":{wire},"created":"9223372036854775807","weight_x_32767":23,"high_priority":true,"weight":null,"conversation_id":"fixture"}}]"# + )), + ), + ( + "high_priority boolean,created bigint,weight_x_32767 smallint,vote smallint,tid integer,pid integer,zid integer", + Some(format!( + r#"[{{"high_priority":true,"created":"9223372036854775807","weight_x_32767":23,"vote":{wire},"tid":4,"pid":0,"weight":null,"conversation_id":"fixture"}}]"# + )), + ), + // Another set of columns is a schema this reader does not model. + ( + "zid integer,pid integer,tid integer,vote smallint,created bigint,weight_x_32767 smallint,high_priority boolean,added integer", + None, + ), + ( + "zid integer,pid integer,tid integer,vote smallint,created bigint,weight_x_32767 smallint", + None, + ), + ]; + for (index, (columns, expected)) in cases.into_iter().enumerate() { + let schema = format!("vote_read_order_{}_{index}", std::process::id()); + // The table is created under another name and renamed, so that it + // keeps the order written here in every test database. + c.batch_execute(&format!( + "CREATE SCHEMA {schema}; SET search_path={schema}; + CREATE TABLE zinvites(zid integer,zinvite text); + INSERT INTO zinvites VALUES(1,'fixture'); + CREATE TABLE votes_as_written({columns}); + ALTER TABLE votes_as_written RENAME TO votes;" + )) + .unwrap(); + let high_priority = if columns.contains("high_priority") { + ("high_priority,", "true,") + } else { + ("", "") + }; + c.execute( + &format!( + "INSERT INTO votes({}zid,pid,tid,vote,weight_x_32767,created) VALUES({}1,0,4,$1,23,9223372036854775807)", + high_priority.0, high_priority.1 + ), + &[&stored], + ) + .unwrap(); + let names: Vec = c.query("SELECT * FROM votes", &[]).unwrap()[0] + .columns() + .iter() + .map(|column| column.name().to_owned()) + .collect(); + assert_eq!( + names, + columns + .split(',') + .map(|column| column.split(' ').next().unwrap().to_owned()) + .collect::>(), + "the table keeps the order written in the test" + ); + let answer = read(&mut c, 1, 0, None, true, "fixture") + .unwrap() + .map(|value| crate::js::stringify(&value)); + assert_eq!(answer, expected, "{columns}"); + c.batch_execute(&format!("DROP SCHEMA {schema} CASCADE")) + .unwrap(); + } +} diff --git a/queue-rs/polis-api/tests/older_database_shape.sql b/queue-rs/polis-api/tests/older_database_shape.sql new file mode 100644 index 000000000..9229a88c5 --- /dev/null +++ b/queue-rs/polis-api/tests/older_database_shape.sql @@ -0,0 +1,430 @@ +-- The shape of an installation whose database predates the migrations. +-- +-- A database built from server/postgres/migrations and a database that was +-- already in service before those migrations existed hold the same tables, +-- but not always in the same shape: the older one stores the math documents +-- as json where the migrations build jsonb, keeps some columns in another +-- order, has a few columns the migrations never create and lacks two they do. +-- A statement can depend on any of these without saying so (a function that +-- exists for jsonb only, a `SELECT *` read by position), and no test of this +-- crate would notice, because every test builds its tables the migrations' +-- way. +-- +-- Installing this file in a database makes every table created there +-- afterwards, under one of the thirteen names below, come out in the older +-- shape. The crate's database tests then run a second time, unchanged +-- (ci/local/polis-api.sh). +-- +-- What is reshaped when a table is created: +-- +-- 1. In any table with one of these names, a column this file records as +-- json and that was declared jsonb becomes json. +-- 2. A table declared with every column the first migration gives it is +-- taken to be the migrations' table. Its columns are put in the older +-- order, the columns only an older installation has are added in their +-- places, the columns it lacks are dropped, and a column that is +-- nullable there loses its NOT NULL. Constraints, defaults and owned +-- sequences are kept. +-- +-- What is not: +-- +-- * A table a test declares with fewer columns keeps the order the test +-- gave it (its rows are usually inserted by position); only rule 1 +-- applies to it. +-- * A column added by a later migration is appended where that migration +-- puts it. Two tables therefore differ from an older installation in +-- their last columns: `conversations` (topics_enabled and xid_required +-- are in the other order there) and `participants_extended` +-- (permanent_cookie and origin come before modified there). +-- * Defaults of existing columns, constraint and index names, triggers and +-- function bodies are the migrations'. +-- * Tables created with CREATE TABLE AS, and tables altered after they are +-- created, are left as they are. +-- +-- When a migration adds a column to one of the thirteen tables, add the +-- column below too, as the last row of that table and marked 'later': an +-- installation older than the migrations receives it from the same +-- migration, at the end of the table. 'first' is only for the columns +-- 000000_initial.sql creates: a table a test declares is reordered when it +-- has every 'first' column. The tests of this file fail until the row is +-- there (src/older_database_shape_tests.rs). +-- +-- This is for test databases. It needs a superuser (event triggers do). + +DROP EVENT TRIGGER IF EXISTS older_database_shape; +DROP SCHEMA IF EXISTS older_database_shape CASCADE; +CREATE SCHEMA older_database_shape; + +-- Every column of the thirteen tables as an older installation has it, in its +-- order there. `migration` says where the migrations build the column: +-- 'first' (000000_initial.sql), 'later' (a later migration adds it) or +-- 'never' (only an older installation has it). +CREATE TABLE older_database_shape.columns ( + position integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + table_name text NOT NULL, + column_name text NOT NULL, + definition text NOT NULL, + migration text NOT NULL CHECK (migration IN ('first', 'later', 'never')), + UNIQUE (table_name, column_name) +); +INSERT INTO older_database_shape.columns (table_name, column_name, definition, migration) VALUES + -- users + ('users', 'uid', 'integer NOT NULL', 'first'), + ('users', 'hname', 'character varying(746)', 'first'), + ('users', 'pwhash', 'character varying(128)', 'never'), + ('users', 'username', 'character varying(128)', 'first'), + ('users', 'email', 'character varying(256)', 'first'), + ('users', 'is_owner', 'boolean', 'first'), + ('users', 'zinvite', 'character varying(300)', 'first'), + ('users', 'oinvite', 'character varying(300)', 'first'), + ('users', 'created', 'bigint', 'first'), + ('users', 'tut', 'smallint', 'first'), + ('users', 'site_id', 'character varying(256) NOT NULL', 'first'), + ('users', 'site_owner', 'boolean', 'first'), + ('users', 'test', 'boolean DEFAULT false', 'never'), + -- conversations + ('conversations', 'zid', 'integer NOT NULL', 'first'), + ('conversations', 'topic', 'character varying(1000)', 'first'), + ('conversations', 'description', 'character varying(50000)', 'first'), + ('conversations', 'is_anon', 'boolean', 'first'), + ('conversations', 'is_active', 'boolean', 'first'), + ('conversations', 'is_draft', 'boolean', 'first'), + ('conversations', 'is_public', 'boolean', 'first'), + ('conversations', 'email_domain', 'character varying(200)', 'first'), + ('conversations', 'owner', 'integer', 'first'), + ('conversations', 'participant_count', 'integer', 'first'), + ('conversations', 'created', 'bigint', 'first'), + ('conversations', 'strict_moderation', 'boolean', 'first'), + ('conversations', 'profanity_filter', 'boolean', 'first'), + ('conversations', 'spam_filter', 'boolean', 'first'), + ('conversations', 'context', 'character varying(1000)', 'first'), + ('conversations', 'modified', 'bigint', 'first'), + ('conversations', 'owner_sees_participation_stats', 'boolean', 'first'), + ('conversations', 'course_id', 'integer', 'first'), + ('conversations', 'link_url', 'character varying(9999)', 'first'), + ('conversations', 'upvotes', 'integer NOT NULL', 'first'), + ('conversations', 'parent_url', 'character varying(9999)', 'first'), + ('conversations', 'vis_type', 'integer NOT NULL', 'first'), + ('conversations', 'write_type', 'integer NOT NULL', 'first'), + ('conversations', 'bgcolor', 'character varying(20)', 'first'), + ('conversations', 'help_type', 'integer NOT NULL', 'first'), + ('conversations', 'socialbtn_type', 'integer NOT NULL', 'first'), + ('conversations', 'style_btn', 'character varying(500)', 'first'), + ('conversations', 'auth_needed_to_vote', 'boolean', 'first'), + ('conversations', 'auth_needed_to_write', 'boolean', 'first'), + ('conversations', 'auth_opt_fb', 'boolean', 'first'), + ('conversations', 'auth_opt_tw', 'boolean', 'first'), + ('conversations', 'auth_opt_allow_3rdparty', 'boolean', 'first'), + ('conversations', 'help_bgcolor', 'character varying(20)', 'first'), + ('conversations', 'help_color', 'character varying(20)', 'first'), + ('conversations', 'is_data_open', 'boolean', 'first'), + ('conversations', 'is_curated', 'boolean DEFAULT false', 'never'), + ('conversations', 'dataset_explanation', 'character varying(50000)', 'never'), + ('conversations', 'write_hint_type', 'integer NOT NULL', 'first'), + ('conversations', 'subscribe_type', 'integer NOT NULL', 'first'), + ('conversations', 'org_id', 'integer', 'first'), + ('conversations', 'need_suzinvite', 'boolean DEFAULT false', 'never'), + ('conversations', 'use_xid_whitelist', 'boolean', 'first'), + ('conversations', 'prioritize_seed', 'boolean', 'first'), + ('conversations', 'importance_enabled', 'boolean NOT NULL', 'later'), + ('conversations', 'treevite_enabled', 'boolean', 'later'), + ('conversations', 'topics_enabled', 'boolean NOT NULL', 'later'), + ('conversations', 'xid_required', 'boolean NOT NULL', 'later'), + -- participants + ('participants', 'pid', 'integer NOT NULL', 'first'), + ('participants', 'uid', 'integer NOT NULL', 'first'), + ('participants', 'zid', 'integer NOT NULL', 'first'), + ('participants', 'created', 'bigint', 'first'), + ('participants', 'vote_count', 'integer NOT NULL', 'first'), + ('participants', 'last_interaction', 'bigint NOT NULL', 'first'), + ('participants', 'subscribed', 'integer NOT NULL', 'first'), + ('participants', 'last_notified', 'bigint', 'first'), + ('participants', 'mod', 'integer NOT NULL', 'first'), + ('participants', 'nsli', 'smallint NOT NULL', 'first'), + -- participants_extended + ('participants_extended', 'uid', 'integer NOT NULL', 'first'), + ('participants_extended', 'zid', 'integer NOT NULL', 'first'), + ('participants_extended', 'referrer', 'character varying(9999)', 'first'), + ('participants_extended', 'parent_url', 'character varying(9999)', 'first'), + ('participants_extended', 'created', 'bigint', 'first'), + ('participants_extended', 'permanent_cookie', 'character varying(32)', 'later'), + ('participants_extended', 'origin', 'character varying(9999)', 'later'), + ('participants_extended', 'encrypted_ip_address', 'character varying(9999)', 'never'), + ('participants_extended', 'encrypted_x_forwarded_for', 'character varying(9999)', 'never'), + ('participants_extended', 'country_iso_code', 'character varying(10)', 'never'), + ('participants_extended', 'modified', 'bigint NOT NULL', 'first'), + ('participants_extended', 'show_translation_activated', 'boolean', 'first'), + ('participants_extended', 'subscribe_email', 'character varying(256)', 'first'), + -- comments + ('comments', 'tid', 'integer NOT NULL', 'first'), + ('comments', 'zid', 'integer NOT NULL', 'first'), + ('comments', 'pid', 'integer NOT NULL', 'first'), + ('comments', 'txt', 'character varying(1000) NOT NULL', 'first'), + ('comments', 'created', 'bigint', 'first'), + ('comments', 'velocity', 'real', 'first'), + ('comments', 'mod', 'integer NOT NULL', 'first'), + ('comments', 'active', 'boolean NOT NULL', 'first'), + ('comments', 'modified', 'bigint', 'first'), + ('comments', 'uid', 'integer NOT NULL', 'first'), + ('comments', 'tweet_id', 'bigint', 'first'), + ('comments', 'quote_src_url', 'character varying(1000)', 'first'), + ('comments', 'anon', 'boolean NOT NULL', 'first'), + ('comments', 'is_seed', 'boolean NOT NULL', 'first'), + ('comments', 'curation', 'smallint NOT NULL DEFAULT 0', 'never'), + ('comments', 'is_meta', 'boolean NOT NULL', 'first'), + ('comments', 'lang_confidence', 'real', 'first'), + ('comments', 'lang', 'character varying(10)', 'first'), + ('comments', 'original_id', 'uuid', 'later'), + -- votes + ('votes', 'zid', 'integer NOT NULL', 'first'), + ('votes', 'pid', 'integer NOT NULL', 'first'), + ('votes', 'tid', 'integer NOT NULL', 'first'), + ('votes', 'vote', 'smallint', 'first'), + ('votes', 'created', 'bigint', 'first'), + ('votes', 'weight_x_32767', 'smallint', 'first'), + ('votes', 'high_priority', 'boolean NOT NULL', 'later'), + -- xids + ('xids', 'uid', 'integer NOT NULL', 'first'), + ('xids', 'owner', 'integer NOT NULL', 'first'), + ('xids', 'xid', 'text NOT NULL', 'first'), + ('xids', 'created', 'bigint', 'first'), + ('xids', 'x_profile_image_url', 'character varying(3000)', 'first'), + ('xids', 'x_name', 'character varying(746)', 'first'), + ('xids', 'modified', 'bigint NOT NULL', 'first'), + ('xids', 'x_email', 'character varying(256)', 'first'), + ('xids', 'zid', 'integer', 'later'), + ('xids', 'pid', 'integer', 'later'), + -- reports + ('reports', 'rid', 'bigint NOT NULL', 'first'), + ('reports', 'report_id', 'character varying(300) NOT NULL', 'first'), + ('reports', 'zid', 'integer NOT NULL', 'first'), + ('reports', 'created', 'bigint', 'first'), + ('reports', 'modified', 'bigint', 'first'), + ('reports', 'label_x_neg', 'character varying(999)', 'first'), + ('reports', 'label_y_neg', 'character varying(999)', 'first'), + ('reports', 'label_y_pos', 'character varying(999)', 'first'), + ('reports', 'label_x_pos', 'character varying(999)', 'first'), + ('reports', 'label_group_0', 'character varying(999)', 'first'), + ('reports', 'label_group_1', 'character varying(999)', 'first'), + ('reports', 'label_group_2', 'character varying(999)', 'first'), + ('reports', 'label_group_3', 'character varying(999)', 'first'), + ('reports', 'label_group_4', 'character varying(999)', 'first'), + ('reports', 'label_group_5', 'character varying(999)', 'first'), + ('reports', 'label_group_6', 'character varying(999)', 'first'), + ('reports', 'label_group_7', 'character varying(999)', 'first'), + ('reports', 'label_group_8', 'character varying(999)', 'first'), + ('reports', 'label_group_9', 'character varying(999)', 'first'), + ('reports', 'report_name', 'character varying(999)', 'first'), + ('reports', 'mod_level', 'smallint NOT NULL', 'later'), + -- site_domain_whitelist + ('site_domain_whitelist', 'site_id', 'character varying(256) NOT NULL', 'first'), + ('site_domain_whitelist', 'domain_whitelist', 'character varying(999)', 'first'), + ('site_domain_whitelist', 'modified', 'bigint NOT NULL', 'first'), + ('site_domain_whitelist', 'created', 'bigint NOT NULL', 'first'), + ('site_domain_whitelist', 'domain_whitelist_override_key', 'character varying(999)', 'first'), + -- math_main + ('math_main', 'zid', 'integer NOT NULL', 'first'), + ('math_main', 'math_env', 'character varying(999) NOT NULL', 'first'), + ('math_main', 'data', 'json NOT NULL', 'first'), + ('math_main', 'last_vote_timestamp', 'bigint NOT NULL', 'first'), + ('math_main', 'modified', 'bigint', 'first'), + ('math_main', 'math_tick', 'bigint NOT NULL', 'first'), + ('math_main', 'caching_tick', 'bigint NOT NULL', 'first'), + -- math_ticks + ('math_ticks', 'zid', 'integer', 'first'), + ('math_ticks', 'math_tick', 'bigint NOT NULL', 'first'), + ('math_ticks', 'math_env', 'character varying(999) NOT NULL', 'first'), + ('math_ticks', 'modified', 'bigint NOT NULL', 'first'), + -- math_bidtopid + ('math_bidtopid', 'zid', 'integer NOT NULL', 'first'), + ('math_bidtopid', 'math_env', 'character varying(999) NOT NULL', 'first'), + ('math_bidtopid', 'data', 'json NOT NULL', 'first'), + ('math_bidtopid', 'modified', 'bigint', 'first'), + ('math_bidtopid', 'math_tick', 'bigint NOT NULL', 'first'), + -- math_ptptstats + ('math_ptptstats', 'zid', 'integer NOT NULL', 'first'), + ('math_ptptstats', 'math_env', 'character varying(999) NOT NULL', 'first'), + ('math_ptptstats', 'data', 'json NOT NULL', 'first'), + ('math_ptptstats', 'modified', 'bigint', 'first'), + ('math_ptptstats', 'math_tick', 'bigint NOT NULL', 'first'); + +-- Columns the migrations build and an older installation does not have. +CREATE TABLE older_database_shape.absent_columns ( + table_name text NOT NULL, + column_name text NOT NULL, + PRIMARY KEY (table_name, column_name) +); +INSERT INTO older_database_shape.absent_columns (table_name, column_name) VALUES + ('conversations', 'branding_type'), + ('math_ticks', 'caching_tick'); + +-- Gives one newly created, still empty table the older shape. +CREATE FUNCTION older_database_shape.reshape(target regclass) RETURNS void +LANGUAGE plpgsql AS $function$ +DECLARE + tbl text; + col record; + present text[]; + wanted text[]; + first_moved integer; + moved text; + constraint_names text[]; + constraint_definitions text[]; + added text; +BEGIN + SELECT relname INTO tbl FROM pg_class WHERE oid = target; + + -- 1. json where the table was declared with jsonb. + FOR col IN + SELECT s.column_name + FROM older_database_shape.columns s + JOIN pg_attribute a ON a.attrelid = target AND a.attname = s.column_name + WHERE s.table_name = tbl AND s.definition ~ '^json( |$)' + AND a.attnum > 0 AND NOT a.attisdropped + AND a.atttypid = 'pg_catalog.jsonb'::regtype + LOOP + EXECUTE format('ALTER TABLE %s ALTER COLUMN %I TYPE json', target, col.column_name); + END LOOP; + + -- 2. Only a table with every column the first migration gives it is + -- reshaped further. + IF EXISTS ( + SELECT FROM older_database_shape.columns s + WHERE s.table_name = tbl AND s.migration = 'first' + AND NOT EXISTS ( + SELECT FROM pg_attribute a + WHERE a.attrelid = target AND a.attname = s.column_name + AND a.attnum > 0 AND NOT a.attisdropped) + ) THEN + RETURN; + END IF; + + FOR col IN + SELECT d.column_name + FROM older_database_shape.absent_columns d + JOIN pg_attribute a ON a.attrelid = target AND a.attname = d.column_name + WHERE d.table_name = tbl AND a.attnum > 0 AND NOT a.attisdropped + LOOP + EXECUTE format('ALTER TABLE %s DROP COLUMN %I', target, col.column_name); + END LOOP; + + FOR col IN + SELECT s.column_name + FROM older_database_shape.columns s + JOIN pg_attribute a ON a.attrelid = target AND a.attname = s.column_name + WHERE s.table_name = tbl AND s.definition !~ ' NOT NULL' + AND a.attnum > 0 AND NOT a.attisdropped AND a.attnotnull + AND NOT EXISTS ( + SELECT FROM pg_index i + WHERE i.indrelid = target AND i.indisprimary AND a.attnum = ANY (i.indkey)) + LOOP + EXECUTE format('ALTER TABLE %s ALTER COLUMN %I DROP NOT NULL', target, col.column_name); + END LOOP; + + -- The order wanted: the older installation's, for the columns the table + -- has and the ones only an older installation has; then any column this + -- file does not know, as declared. + present := ARRAY( + SELECT a.attname::text FROM pg_attribute a + WHERE a.attrelid = target AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum); + wanted := ARRAY( + SELECT s.column_name FROM older_database_shape.columns s + WHERE s.table_name = tbl + AND (s.migration = 'never' OR s.column_name = ANY (present)) + ORDER BY s.position) + || ARRAY( + SELECT u.column_name FROM unnest(present) WITH ORDINALITY AS u(column_name, n) + WHERE NOT EXISTS ( + SELECT FROM older_database_shape.columns s + WHERE s.table_name = tbl AND s.column_name = u.column_name) + ORDER BY u.n); + first_moved := 1; + WHILE first_moved <= cardinality(present) AND present[first_moved] = wanted[first_moved] LOOP + first_moved := first_moved + 1; + END LOOP; + IF first_moved > cardinality(wanted) THEN + RETURN; + END IF; + + -- A column is moved by adding it again at the end and dropping the + -- original. Its constraints would go with the original, so every + -- constraint of the table is set aside first and put back afterwards + -- under its own name. + SELECT coalesce(array_agg(c.conname::text ORDER BY c.contype = 'f', c.oid), '{}'), + coalesce(array_agg(pg_get_constraintdef(c.oid) ORDER BY c.contype = 'f', c.oid), '{}') + INTO constraint_names, constraint_definitions + FROM pg_constraint c + WHERE c.conrelid = target AND c.contype IN ('p', 'u', 'f', 'c', 'x'); + FOR i IN REVERSE cardinality(constraint_names) .. 1 LOOP + EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', target, constraint_names[i]); + END LOOP; + + FOREACH moved IN ARRAY wanted[first_moved:] LOOP + IF moved = ANY (present) THEN + SELECT format_type(a.atttypid, a.atttypmod) AS data_type, + a.attnotnull AS not_null, + pg_get_expr(d.adbin, d.adrelid) AS default_value, + pg_get_serial_sequence(target::text, a.attname::text) AS owned_sequence, + a.attidentity <> '' OR a.attgenerated <> '' AS computed + INTO col + FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = target AND a.attname = moved + AND a.attnum > 0 AND NOT a.attisdropped; + IF col.computed THEN + RAISE EXCEPTION 'older_database_shape: %.% is an identity or generated column and cannot be moved', target, moved; + END IF; + IF col.owned_sequence IS NOT NULL THEN + EXECUTE format('ALTER SEQUENCE %s OWNED BY NONE', col.owned_sequence); + END IF; + EXECUTE format('ALTER TABLE %s RENAME COLUMN %I TO older_database_shape_moving', target, moved); + EXECUTE format('ALTER TABLE %s ADD COLUMN %I %s%s%s', target, moved, col.data_type, + CASE WHEN col.default_value IS NULL THEN '' ELSE ' DEFAULT ' || col.default_value END, + CASE WHEN col.not_null THEN ' NOT NULL' ELSE '' END); + EXECUTE format('ALTER TABLE %s DROP COLUMN older_database_shape_moving', target); + IF col.owned_sequence IS NOT NULL THEN + EXECUTE format('ALTER SEQUENCE %s OWNED BY %s.%I', col.owned_sequence, target, moved); + END IF; + ELSE + SELECT s.definition INTO added + FROM older_database_shape.columns s + WHERE s.table_name = tbl AND s.column_name = moved; + EXECUTE format('ALTER TABLE %s ADD COLUMN %I %s', target, moved, added); + END IF; + END LOOP; + + FOR i IN 1 .. cardinality(constraint_names) LOOP + EXECUTE format('ALTER TABLE %s ADD CONSTRAINT %I %s', target, constraint_names[i], constraint_definitions[i]); + END LOOP; +END +$function$; + +CREATE FUNCTION older_database_shape.on_create_table() RETURNS event_trigger +LANGUAGE plpgsql AS $function$ +DECLARE + created record; +BEGIN + FOR created IN + SELECT c.objid + FROM pg_event_trigger_ddl_commands() c + JOIN pg_class r ON r.oid = c.objid + WHERE c.classid = 'pg_catalog.pg_class'::regclass + AND c.command_tag = 'CREATE TABLE' AND c.object_type = 'table' + AND r.relkind = 'r' + AND r.relnamespace <> 'older_database_shape'::regnamespace + AND r.relname IN (SELECT s.table_name FROM older_database_shape.columns s) + LOOP + PERFORM older_database_shape.reshape(created.objid::regclass); + END LOOP; +END +$function$; + +-- A migration may create its tables as another role; the trigger runs as that +-- role and has to be able to read the lists above. +GRANT USAGE ON SCHEMA older_database_shape TO PUBLIC; +GRANT SELECT ON ALL TABLES IN SCHEMA older_database_shape TO PUBLIC; + +CREATE EVENT TRIGGER older_database_shape ON ddl_command_end + WHEN TAG IN ('CREATE TABLE', 'CREATE SCHEMA') + EXECUTE FUNCTION older_database_shape.on_create_table();