From 0c0d83884ef3451039bca6d216c156cf8232d999 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Tue, 3 Nov 2026 10:56:49 -0800 Subject: [PATCH] polis-api: Treevite waves lock tests wait for what they test, not for the runner Two database tests in post_treevite_waves_tests.rs failed intermittently with PostgreSQL timeouts, on hosted runners and on a quiet x86 box: - allocator_lock_and_real_fk_locks_do_not_cycle timed out (57014) in fixture DDL, before either lock probe ran: setup used the serving connection with its 500 ms statement budget. - concurrent_creates_and_owner_budget_are_serialized raced: the second create could reach the owner lock after the first had committed, or time out waiting for it, so the test sometimes observed no serialization at all. Setup and lock probes now use a test-only connection with 30 s / 10 s statement and lock timeouts; connect() and the real writer pool keep the serving settings. The concurrent test holds the owner advisory lock in an observer, sees both creates waiting on it before releasing, and joins both workers before asserting. Test-only; no handler, schema or serving change. Under 8-process load plus 8 CPU workers, 600 pairs: concurrent creates passed 348 before and 600 after. Handler mutations still fail the tests: a real lock cycle or a member-row FOR UPDATE fails the allocator test; removing the owner lock, or taking it after the cap check, fails the concurrent test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ss9NfEMZS9Qm4FZ2npXDrx --- .../src/post_treevite_waves_tests.rs | 125 +++++++++++++----- 1 file changed, 89 insertions(+), 36 deletions(-) diff --git a/queue-rs/polis-api/src/post_treevite_waves_tests.rs b/queue-rs/polis-api/src/post_treevite_waves_tests.rs index 1aa687048..30a3f46ea 100644 --- a/queue-rs/polis-api/src/post_treevite_waves_tests.rs +++ b/queue-rs/polis-api/src/post_treevite_waves_tests.rs @@ -79,7 +79,10 @@ mod database { config, name, }; - let mut c = f.connect(); + // Schema creation is test setup, not a request through the writer + // pool. Concurrent database/DDL work on CI can exceed its 500 ms + // statement budget before the lock-order test has even started. + let mut c = f.long_timeout_connect(); c.batch_execute("create table users(uid int primary key,site_id text); create table conversations(zid int primary key,owner int,participant_count int default 0); create table participants(zid int,pid int,uid int, primary key(zid,pid)); insert into users values(1,'a'),(2,'a'),(3,'b'); insert into conversations(zid,owner) values(1,1),(2,1),(3,3); insert into participants values(1,0,3),(1,1,3);").unwrap(); c.batch_execute(include_str!( "../../../server/postgres/migrations/000013_create_treevite.sql" @@ -93,6 +96,16 @@ mod database { .unwrap(); c } + fn long_timeout_connect(&self) -> Client { + let mut c = self.config.connect(postgres::NoTls).unwrap(); + // Test-only statement and lock timeouts (30 s / 10 s), not the + // serving settings: for setup DDL and for correctness probes that + // park transactions until the observer sees a lock wait. connect() + // and the real writer pool keep the short serving timeouts. + c.batch_execute("set statement_timeout=30000; set lock_timeout=10000") + .unwrap(); + c + } fn pool(&self) -> Arc { db::Pool::writer( polis_queue_adapter::jobs::transport::Connector::Plain(Box::new( @@ -560,54 +573,89 @@ mod database { ); } - #[test] - fn concurrent_creates_and_owner_budget_are_serialized() { - let f = Fixture::new(); - let gate = Arc::new(std::sync::Barrier::new(2)); - let handles = (0..2) - .map(|_| { - let mut c = f.connect(); - let gate = gate.clone(); + fn concurrent_creates( + f: &Fixture, + requests: Vec<(i32, Fields, Caps)>, + ) -> Vec> { + let mut observer = f.long_timeout_connect(); + observer.batch_execute("begin; select pg_advisory_xact_lock(hashtextextended(jsonb_build_array('treevite:owner-codes',1)::text,0))").unwrap(); + let holder: i32 = observer + .query_one("select pg_backend_pid()", &[]) + .unwrap() + .get(0); + let mut pids = Vec::new(); + let handles = requests + .into_iter() + .map(|(zid, fields, caps)| { + let mut c = f.long_timeout_connect(); + pids.push( + c.query_one("select pg_backend_pid()", &[]) + .unwrap() + .get::<_, i32>(0), + ); std::thread::spawn(move || { - gate.wait(); - wave(&mut c, 1, &fields(3, 0, None)).wave + create(&mut c, zid, 1, &[], &fields, caps, DEADLINE).unwrap() }) }) .collect::>(); - let mut numbers = handles - .into_iter() - .map(|t| t.join().unwrap()) - .collect::>(); + // A thread barrier only starts callers together. Observe both requests + // actually contending on the owner's transaction lock before release; + // removing that lock must fail this probe, even on a serial scheduler. + let end = Instant::now() + Duration::from_secs(5); + let observed = loop { + let blocked = pids.iter().all(|pid| { + observer + .query_one( + "select $1::int = any(pg_blocking_pids($2::int))", + &[&holder, pid], + ) + .unwrap() + .get::<_, bool>(0) + }); + if blocked { + break true; + } + if Instant::now() >= end { + break false; + } + std::thread::sleep(Duration::from_millis(1)); + }; + observer.batch_execute("commit").unwrap(); + // Join both workers before asserting, so a failure cannot drop the + // fixture database while its sibling is still executing a transaction. + let results = handles.into_iter().map(|h| h.join()).collect::>(); + assert!(observed, "both creates must wait on the owner lock"); + results.into_iter().map(|r| r.unwrap()).collect() + } + + #[test] + fn concurrent_creates_and_owner_budget_are_serialized() { + let f = Fixture::new(); + let mut numbers = concurrent_creates( + &f, + (0..2).map(|_| (1, fields(3, 0, None), caps())).collect(), + ) + .into_iter() + .map(|r| r.unwrap().wave) + .collect::>(); numbers.sort(); assert_eq!(numbers, vec![1, 2]); assert_eq!(f.counts(), (2, 6)); - let gate = Arc::new(std::sync::Barrier::new(2)); - let handles = (1..=2) - .map(|zid| { - let mut c = f.connect(); - let gate = gate.clone(); - std::thread::spawn(move || { - gate.wait(); - create( - &mut c, + let results = concurrent_creates( + &f, + (1..=2) + .map(|zid| { + ( zid, - 1, - &[], - &fields(3, 0, Some(0)), + fields(3, 0, Some(0)), Caps { conversation: 20, owner: 9, }, - DEADLINE, ) - .unwrap() }) - }) - .collect::>(); - let results = handles - .into_iter() - .map(|h| h.join().unwrap()) - .collect::>(); + .collect(), + ); assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); assert_eq!( results @@ -839,7 +887,9 @@ mod database { #[test] fn allocator_lock_and_real_fk_locks_do_not_cycle() { let f = Fixture::new(); - let mut c = f.connect(); + // Seed the root wave and install the allocator before applying request + // budgets. No competing transaction exists during this setup. + let mut c = f.long_timeout_connect(); let root = wave(&mut c, 1, &fields(1, 0, Some(0))); c.batch_execute("update treevite_invites set invite_used_by_pid=0,status=1") .unwrap(); @@ -855,6 +905,9 @@ mod database { .next() .unwrap(); c.batch_execute(&format!("CREATE OR REPLACE FUNCTION{function} CREATE TRIGGER pid_auto BEFORE INSERT ON participants FOR EACH ROW EXECUTE FUNCTION pid_auto();")).unwrap(); + // The actual participation/lock probe retains the serving SQL limits + // and both original elapsed-time assertions. + c = f.connect(); let mut admission = f.connect(); admission .batch_execute("begin; insert into participants(zid,uid) values(1,3)")