From de23d980cd62c793cf72d17edafe729bb077f994 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 26 Oct 2026 13:16:32 +0800 Subject: [PATCH 1/3] feat(policy)!: gate maintenance behind the maintain and reclaim actions Add the `maintain` and `reclaim` Cedar actions. `maintain` covers optimize, index builds, the full-text rebuild and the repair report and is judged on the branch maintained; `reclaim` covers cleanup and takes no branch scope. Maintenance has no actor-less public engine entry point any more: `optimize_as`, `cleanup_as`, `ensure_indices_as`, `ensure_indices_on_as`, `rebuild_full_text_indices_on_as` and the read-only `repair_report_as` each enforce their action before any effect. The full-text rebuild admits `maintain` or `change` through `PolicyChecker::check_any`. Repair loses its publishing mode, its options and its hard-coded actor strings. The CLI, DST, GQT and bench callers move to the `_as` forms; the direct CLI passes the actor it resolves for direct writes. --- ...ll-text-rebuild-change-grant.deprecated.md | 6 + ...nce-entry-points-take-an-actor.breaking.md | 12 + .../maintenance-policy-actions.added.md | 10 + crates/omnigraph-api-types/src/lib.rs | 3 + crates/omnigraph-bench/src/dataset_worker.rs | 5 +- .../src/legacy/branch_merge_oracle.rs | 2 +- .../src/commands/maintenance/mod.rs | 11 +- crates/omnigraph-cli/src/output.rs | 13 +- .../tests/integration/cli_data.rs | 5 +- crates/omnigraph-dst/src/concurrent.rs | 15 +- crates/omnigraph-dst/src/harness.rs | 17 +- crates/omnigraph-dst/tests/scenarios.rs | 54 +- crates/omnigraph-gqt-core/src/lib.rs | 2 +- crates/omnigraph-policy/src/lib.rs | 271 +++++++++- crates/omnigraph/src/db/graph_coordinator.rs | 7 +- crates/omnigraph/src/db/mod.rs | 2 +- crates/omnigraph/src/db/omnigraph.rs | 94 ++-- .../omnigraph/src/db/omnigraph/collector.rs | 18 +- crates/omnigraph/src/db/omnigraph/optimize.rs | 42 +- crates/omnigraph/src/db/omnigraph/repair.rs | 162 +----- .../omnigraph/src/db/omnigraph/table_ops.rs | 35 +- crates/omnigraph/src/db/upgrade/tests.rs | 6 +- crates/omnigraph/src/engine/plan_source.rs | 2 +- crates/omnigraph/src/runtime_cache.rs | 2 +- crates/omnigraph/tests/bound_storage.rs | 13 +- .../omnigraph/tests/detached_commit_matrix.rs | 47 +- crates/omnigraph/tests/failpoints.rs | 272 +++++----- crates/omnigraph/tests/forbidden_apis.rs | 118 +++- .../tests/helpers/expand_projection.rs | 2 +- crates/omnigraph/tests/helpers/mod.rs | 2 +- .../omnigraph/tests/integration/branching.rs | 79 ++- crates/omnigraph/tests/integration/changes.rs | 19 +- .../tests/integration/changes_cost.rs | 125 +---- .../tests/integration/composite_flow.rs | 13 +- .../tests/integration/consistency.rs | 4 +- .../omnigraph/tests/integration/end_to_end.rs | 6 +- .../omnigraph/tests/integration/engine_v2.rs | 4 +- .../tests/integration/engine_v2_memory.rs | 10 +- .../integration/engine_v2_plan_replay.rs | 6 +- .../tests/integration/literal_filters.rs | 2 +- .../tests/integration/maintenance.rs | 505 ++++++++++-------- .../tests/integration/merge_cleanup.rs | 4 +- .../tests/integration/point_in_time.rs | 5 +- .../omnigraph/tests/integration/recovery.rs | 8 +- .../tests/integration/repro_issue_563.rs | 2 +- .../tests/integration/rrf_prefilter_gate.rs | 14 +- .../tests/integration/scalar_indexes.rs | 4 +- .../tests/integration/traversal_adaptive.rs | 6 +- .../tests/integration/warm_read_cost.rs | 22 +- .../omnigraph/tests/integration/write_cost.rs | 16 +- .../omnigraph/tests/policy_engine_chassis.rs | 306 +++++++++-- crates/omnigraph/tests/schema_apply.rs | 27 +- crates/omnigraph/tests/search.rs | 35 +- .../omnigraph/tests/system_column_upgrade.rs | 6 +- docs/dev/control-plane.md | 1 + docs/dev/writes.md | 9 + docs/user/operations/maintenance.md | 15 +- docs/user/operations/policy.md | 18 +- skills/omnigraph/references/server-policy.md | 6 +- 59 files changed, 1566 insertions(+), 961 deletions(-) create mode 100644 changelog.d/full-text-rebuild-change-grant.deprecated.md create mode 100644 changelog.d/maintenance-entry-points-take-an-actor.breaking.md create mode 100644 changelog.d/maintenance-policy-actions.added.md diff --git a/changelog.d/full-text-rebuild-change-grant.deprecated.md b/changelog.d/full-text-rebuild-change-grant.deprecated.md new file mode 100644 index 000000000..b17d94953 --- /dev/null +++ b/changelog.d/full-text-rebuild-change-grant.deprecated.md @@ -0,0 +1,6 @@ +- The full-text rebuild accepts `maintain` or `change` on its branch in this + release; a later release accepts `maintain` only. Grant `maintain` to the + accounts that rebuild full-text indexes. See + [rebuild full-text indexes][full-text-rebuild-change-grant]. + +[full-text-rebuild-change-grant]: ../docs/user/operations/maintenance.md#rebuild-full-text-indexes diff --git a/changelog.d/maintenance-entry-points-take-an-actor.breaking.md b/changelog.d/maintenance-entry-points-take-an-actor.breaking.md new file mode 100644 index 000000000..d6f96b6fc --- /dev/null +++ b/changelog.d/maintenance-entry-points-take-an-actor.breaking.md @@ -0,0 +1,12 @@ +- Rust callers of the engine's maintenance methods move to their actor-taking + forms: `optimize_as(actor)`, `cleanup_as(options, actor)`, + `ensure_indices_as(actor)`, `ensure_indices_on_as(branch, actor)` and + `rebuild_full_text_indices_on_as(branch, actor)`; pass `None` where no policy + is installed. `repair` is now the read-only `repair_report_as(actor)`: + `RepairOptions`, the `healed` and `forced` actions and + `RepairStats::graph_manifest_version` are gone, and `repair --json` no longer + prints the always-null `graph_manifest_version`. An exhaustive match on + `PolicyAction` adds arms for `Maintain` and `Reclaim`. See + [repair][maintenance-entry-points-repair]. + +[maintenance-entry-points-repair]: ../docs/user/operations/maintenance.md#repair diff --git a/changelog.d/maintenance-policy-actions.added.md b/changelog.d/maintenance-policy-actions.added.md new file mode 100644 index 000000000..d7698ffce --- /dev/null +++ b/changelog.d/maintenance-policy-actions.added.md @@ -0,0 +1,10 @@ +- **Maintenance has its own Cedar actions.** `maintain` covers optimize, index + builds, the full-text rebuild and the repair report on the branch they + maintain, and `reclaim` covers cleanup with no branch scope; `change` implies + neither, so automation can compact a graph without the right to write data or + delete history. An embedded host that installs a policy checker enforces them + on every maintenance entry point, optimize and the index writers record the + requesting actor on their commit, and existing policies grant neither action. + See [policy actions][maintenance-policy-actions]. + +[maintenance-policy-actions]: ../docs/user/operations/policy.md#actions diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs index f07f23307..ebe8b3a1c 100644 --- a/crates/omnigraph-api-types/src/lib.rs +++ b/crates/omnigraph-api-types/src/lib.rs @@ -2013,6 +2013,9 @@ fn access_slot(action: omnigraph_policy::PolicyAction) -> Option { P::InvokeQuery => Some(AccessSlot::GraphWide), // Server- or operator-scoped: no graph's access report names them. P::Admin | P::GraphList | P::ConfigManage => None, + // Maintenance runs only through the direct CLI and embedded callers; + // no served operation performs it, so the report does not name it. + P::Maintain | P::Reclaim => None, } } diff --git a/crates/omnigraph-bench/src/dataset_worker.rs b/crates/omnigraph-bench/src/dataset_worker.rs index f66ae24af..3999d6e98 100644 --- a/crates/omnigraph-bench/src/dataset_worker.rs +++ b/crates/omnigraph-bench/src/dataset_worker.rs @@ -803,7 +803,10 @@ pub(crate) async fn build_dataset( Engine::V2, )?; if plan.needs_indices { - session.ensure_indices().await.map_err(|e| e.to_string())?; + session + .ensure_indices_as(None) + .await + .map_err(|e| e.to_string())?; } (session, case, Some(exact)) } diff --git a/crates/omnigraph-bench/src/legacy/branch_merge_oracle.rs b/crates/omnigraph-bench/src/legacy/branch_merge_oracle.rs index b5151f265..78466e2be 100644 --- a/crates/omnigraph-bench/src/legacy/branch_merge_oracle.rs +++ b/crates/omnigraph-bench/src/legacy/branch_merge_oracle.rs @@ -28,7 +28,7 @@ pub async fn initialize_local_fixture( let optimized_user_tables = match plan.compaction_recency { CompactionRecency::Optimized => { let outcomes = db - .optimize() + .optimize_as(None) .await .map_err(|error| fixture_error(format!("optimize fixture main branch: {error}")))?; let intended_keys = (0..plan.node_tables()) diff --git a/crates/omnigraph-cli/src/commands/maintenance/mod.rs b/crates/omnigraph-cli/src/commands/maintenance/mod.rs index a6557054c..f39beb150 100644 --- a/crates/omnigraph-cli/src/commands/maintenance/mod.rs +++ b/crates/omnigraph-cli/src/commands/maintenance/mod.rs @@ -3,7 +3,7 @@ use std::io::Write; use color_eyre::eyre::{Result, bail}; -use omnigraph::db::{CleanupPolicyOptions, Omnigraph, RepairAction, RepairOptions}; +use omnigraph::db::{CleanupPolicyOptions, Omnigraph, RepairAction}; use crate::app::Completion; use crate::helpers::{confirm_destructive, parse_duration_arg, resolve_actor}; @@ -42,13 +42,14 @@ pub(crate) async fn run_optimize( let OptimizeArgs { json } = args; let uri = crate::scope::direct_graph_uri(scope.target, scope.graph).await?; let binding = crate::scope::capture_storage_binding(scope.target)?.for_uri(&uri)?; + let actor = resolve_actor(scope.operator, scope.actor, stderr)?; crate::admission::ensure_graph_with_binding(&binding, stderr).await?; let db = Omnigraph::open_with_storage( binding.uri(), omnigraph::storage::storage_for_binding(&binding)?, ) .await?; - let stats = db.optimize().await?; + let stats = db.optimize_as(actor.as_deref()).await?; write_optimize(&uri, &stats, json, stdout)?; Ok(Completion::success()) } @@ -93,13 +94,14 @@ pub(crate) async fn run_repair( let RepairArgs { json } = args; let uri = crate::scope::direct_graph_uri(scope.target, scope.graph).await?; let binding = crate::scope::capture_storage_binding(scope.target)?.for_uri(&uri)?; + let actor = resolve_actor(scope.operator, scope.actor, stderr)?; crate::admission::ensure_graph_with_binding(&binding, stderr).await?; let db = Omnigraph::open_with_storage( binding.uri(), omnigraph::storage::storage_for_binding(&binding)?, ) .await?; - let stats = db.repair(RepairOptions::default()).await?; + let stats = db.repair_report_as(actor.as_deref()).await?; let refused_count = stats .datasets .iter() @@ -159,13 +161,14 @@ pub(crate) async fn run_cleanup( keep_versions: keep, older_than: older_than_dur, }; + let actor = resolve_actor(scope.operator, scope.actor, stderr)?; crate::admission::ensure_graph_with_binding(&binding, stderr).await?; let db = Omnigraph::open_with_storage( binding.uri(), omnigraph::storage::storage_for_binding(&binding)?, ) .await?; - let stats = db.cleanup(options.clone()).await?; + let stats = db.cleanup_as(options.clone(), actor.as_deref()).await?; write_cleanup(&uri, &options, &policy_desc, &stats, json, stdout)?; Ok(Completion::success()) } diff --git a/crates/omnigraph-cli/src/output.rs b/crates/omnigraph-cli/src/output.rs index aa8280127..6455bd7d5 100644 --- a/crates/omnigraph-cli/src/output.rs +++ b/crates/omnigraph-cli/src/output.rs @@ -133,7 +133,6 @@ pub(crate) fn write_repair( if json { let value = serde_json::json!({ "uri": uri, - "graph_manifest_version": stats.graph_manifest_version, "datasets": stats.datasets.iter().map(|s| serde_json::json!({ "type_key": s.type_key, "published_dataset_version": s.published_dataset_version, @@ -1867,7 +1866,7 @@ mod tests { #[tokio::test] async fn maintenance_optimize_and_repair_reports_preserve_all_result_fields() { use omnigraph::db::{ - Omnigraph, PendingIndex, RepairAction, RepairClassification, RepairOptions, SkipReason, + Omnigraph, PendingIndex, RepairAction, RepairClassification, SkipReason, }; // These engine result types are non_exhaustive. Obtain one valid value // from a tiny graph, then set its public presentation fields; no engine @@ -1881,7 +1880,7 @@ mod tests { let db = Omnigraph::init(&uri, "node Person {\n slug: String @key\n}\n") .await .unwrap(); - let mut compacted = db.optimize().await.unwrap().pop().unwrap(); + let mut compacted = db.optimize_as(None).await.unwrap().pop().unwrap(); compacted.type_key = "node:Person".into(); compacted.fragments_removed = 3; compacted.fragments_added = 1; @@ -1933,7 +1932,7 @@ mod tests { |json, writer| super::write_optimize("graph", &[], json, writer), ); - let mut repair = db.repair(RepairOptions::default()).await.unwrap(); + let mut repair = db.repair_report_as(None).await.unwrap(); let mut clean = repair.datasets.pop().unwrap(); clean.type_key = "node:Person".into(); clean.published_dataset_version = 4; @@ -1950,7 +1949,6 @@ mod tests { refused.operations = vec!["Append".into(), "Delete".into()]; refused.error = Some("foreign write".into()); repair.datasets = vec![clean, refused]; - repair.graph_manifest_version = Some(9); assert_maintenance_report( concat!( "repair graph — 2 datasets\n", @@ -1958,7 +1956,7 @@ mod tests { " edge type 'Knows' refused foreign_drift 4 → 6 [Append, Delete] (foreign write)\n", ), serde_json::json!({ - "uri":"graph", "graph_manifest_version":9, + "uri":"graph", "datasets":[ {"type_key":"node:Person","published_dataset_version":4,"lance_head_version":4,"classification":"no_drift","action":"no_op","operations":[],"error":null}, {"type_key":"edge:Knows","published_dataset_version":4,"lance_head_version":6,"classification":"foreign_drift","action":"refused","operations":["Append","Delete"],"error":"foreign write"}, @@ -1967,10 +1965,9 @@ mod tests { |json, writer| super::write_repair("graph", &repair, json, writer), ); repair.datasets.clear(); - repair.graph_manifest_version = None; assert_maintenance_report( "repair graph — 0 datasets\n", - serde_json::json!({"uri":"graph","graph_manifest_version":null,"datasets":[]}), + serde_json::json!({"uri":"graph","datasets":[]}), |json, writer| super::write_repair("graph", &repair, json, writer), ); } diff --git a/crates/omnigraph-cli/tests/integration/cli_data.rs b/crates/omnigraph-cli/tests/integration/cli_data.rs index 939fe3de9..9d110089a 100644 --- a/crates/omnigraph-cli/tests/integration/cli_data.rs +++ b/crates/omnigraph-cli/tests/integration/cli_data.rs @@ -1211,7 +1211,8 @@ fn repair_json_reports_noop_on_clean_graph() { assert!(payload.get("confirm").is_none()); assert!(payload.get("force").is_none()); - assert_eq!(payload["graph_manifest_version"], Value::Null); + // The report publishes nothing, so it names no graph manifest version. + assert!(payload.get("graph_manifest_version").is_none()); assert!(payload.get("manifest_version").is_none()); assert!(payload.get("tables").is_none()); let datasets = payload["datasets"].as_array().unwrap(); @@ -1282,7 +1283,7 @@ fn repair_reports_foreign_drift_and_refuses_removed_flags() { .arg(&graph), ); let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); - assert_eq!(payload["graph_manifest_version"], Value::Null); + assert!(payload.get("graph_manifest_version").is_none()); let person = payload["datasets"] .as_array() .unwrap() diff --git a/crates/omnigraph-dst/src/concurrent.rs b/crates/omnigraph-dst/src/concurrent.rs index a05a5274a..98fc5f188 100644 --- a/crates/omnigraph-dst/src/concurrent.rs +++ b/crates/omnigraph-dst/src/concurrent.rs @@ -1753,12 +1753,15 @@ fn maintenance_life( let mut occ_retries = 0usize; loop { let result = match kind { - 0 => Box::pin(db.ensure_indices()).await.map(|_| ()), - 1 => Box::pin(db.optimize()).await.map(|_| ()), - _ => Box::pin(db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - })) + 0 => Box::pin(db.ensure_indices_as(None)).await.map(|_| ()), + 1 => Box::pin(db.optimize_as(None)).await.map(|_| ()), + _ => Box::pin(db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + )) .await .map(|_| ()), }; diff --git a/crates/omnigraph-dst/src/harness.rs b/crates/omnigraph-dst/src/harness.rs index 109c1df1c..9f4e65730 100644 --- a/crates/omnigraph-dst/src/harness.rs +++ b/crates/omnigraph-dst/src/harness.rs @@ -1634,12 +1634,15 @@ async fn exec_op(db: &Session, branch: &str, op: &Op) -> OmniResult<()> { ) .await .map(|_| ()), - Op::Optimize => Box::pin(db.optimize()).await.map(|_| ()), + Op::Optimize => Box::pin(db.optimize_as(None)).await.map(|_| ()), Op::Cleanup => { - let stats = Box::pin(db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - })) + let stats = Box::pin(db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + )) .await?; let deferred: BTreeMap = stats .iter() @@ -1655,7 +1658,7 @@ async fn exec_op(db: &Session, branch: &str, op: &Op) -> OmniResult<()> { .insert(db.uri().to_string(), deferred); Ok(()) } - Op::EnsureIndices => Box::pin(db.ensure_indices()).await.map(|_| ()), + Op::EnsureIndices => Box::pin(db.ensure_indices_as(None)).await.map(|_| ()), Op::SchemaAddProperty { count } => Box::pin(db.apply_schema(&schema_with_extras(*count))) .await .map(|_| ()), @@ -6432,7 +6435,7 @@ impl UniverseScenario for Scenario { tokio::time::resume(); let lively = tokio::time::timeout( std::time::Duration::from_secs(120), - Box::pin(db.ensure_indices()), + Box::pin(db.ensure_indices_as(None)), ) .await; tokio::time::pause(); diff --git a/crates/omnigraph-dst/tests/scenarios.rs b/crates/omnigraph-dst/tests/scenarios.rs index 8266b52b3..c0cc84306 100644 --- a/crates/omnigraph-dst/tests/scenarios.rs +++ b/crates/omnigraph-dst/tests/scenarios.rs @@ -1045,7 +1045,7 @@ async fn dst_memory_graph_end_to_end() { db.load_jsonl(TEST_DATA, LoadMode::Overwrite) .await .expect("load test data"); - db.ensure_indices().await.expect("ensure indices"); + db.ensure_indices_as(None).await.expect("ensure indices"); let persons = count_rows(&db, "node:Person").await; assert!(persons > 0, "loaded graph must have Person rows"); @@ -3668,7 +3668,7 @@ fn reborn_branch_cache_poison_body() { .expect("op1"); db.branch_create("b0").await.expect("op2"); world_read(&db).await; // ARMING READ 1 (the universe's op-2 verify) - db.optimize().await.expect("op3"); + db.optimize_as(None).await.expect("op3"); m!( db, "b0", @@ -3682,7 +3682,7 @@ fn reborn_branch_cache_poison_body() { Box::pin(db.branch_merge("b0", "main")) .await .expect("op6 merge"); - db.ensure_indices().await.expect("op7"); + db.ensure_indices_as(None).await.expect("op7"); db.branch_delete("b0").await.expect("op8"); m!( db, @@ -3716,7 +3716,7 @@ fn reborn_branch_cache_poison_body() { &[] ) .expect("op12"); - db.ensure_indices().await.expect("op13"); + db.ensure_indices_as(None).await.expect("op13"); m!( db, "main", @@ -3757,10 +3757,13 @@ fn reborn_branch_cache_poison_body() { &[("$age", 20)] ) .expect("op20"); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .expect("op21"); m!( @@ -3779,10 +3782,13 @@ fn reborn_branch_cache_poison_body() { &[("$age", 14), ("$ver", 8)] ) .expect("op23"); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .expect("op24"); db.branch_create("b0").await.expect("op25"); @@ -5197,7 +5203,7 @@ fn dst_classa_ablation_matrix_body() { db.branch_create("b0").await.expect("bc b0 life1"); } if o { - db.optimize().await.expect("optimize"); + db.optimize_as(None).await.expect("optimize"); } if r { upsert(&db, "b0", "w6", 45, 2).await; @@ -5210,20 +5216,20 @@ fn dst_classa_ablation_matrix_body() { upsert(&db, "main", "w1", 19, 3).await; upsert(&db, "main", "w7", 7, 6).await; if c { - db.cleanup(omnigraph::db::CleanupPolicyOptions { + db.cleanup_as(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None, - }) + }, None) .await .expect("cleanup 1"); } upsert(&db, "main", "w7", 17, 7).await; upsert(&db, "main", "w5", 14, 8).await; if c { - db.cleanup(omnigraph::db::CleanupPolicyOptions { + db.cleanup_as(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None, - }) + }, None) .await .expect("cleanup 2"); } @@ -5284,26 +5290,26 @@ fn dst_classa_ablation_matrix_body() { m!(db, "main", "set_age_v", &[("$name", "w5")], &[("$age", 70), ("$ver", 1)]).expect("op0"); m!(db, "main", "insert_person", &[("$name", "w6")], &[("$age", 7)]).expect("op1"); db.branch_create("b0").await.expect("op2"); - db.optimize().await.expect("op3"); + db.optimize_as(None).await.expect("op3"); m!(db, "b0", "set_age_v", &[("$name", "w6")], &[("$age", 45), ("$ver", 2)]).expect("op4"); m!(db, "b0", "remove_friendships_from", &[("$from", "w6")], &[]).expect("op5"); Box::pin(db.branch_merge("b0", "main")).await.expect("op6 merge"); - db.ensure_indices().await.expect("op7"); + db.ensure_indices_as(None).await.expect("op7"); db.branch_delete("b0").await.expect("op8"); m!(db, "main", "remove_friendships_from", &[("$from", "w2")], &[]).expect("op9"); m!(db, "main", "insert_person_v", &[("$name", "w1")], &[("$age", 19), ("$ver", 3)]).expect("op10"); m!(db, "main", "set_age_v", &[("$name", "w2")], &[("$age", 3), ("$ver", 4)]).expect("op11"); m!(db, "main", "add_friend", &[("$from", "Charlie"), ("$to", "w6")], &[]).expect("op12"); - db.ensure_indices().await.expect("op13"); + db.ensure_indices_as(None).await.expect("op13"); m!(db, "main", "insert_person", &[("$name", "w1")], &[("$age", 52)]).expect("op15"); m!(db, "main", "remove_friendships_from", &[("$from", "w3")], &[]).expect("op16"); m!(db, "main", "set_age_v", &[("$name", "w3")], &[("$age", 24), ("$ver", 5)]).expect("op18"); m!(db, "main", "set_age_v", &[("$name", "w7")], &[("$age", 7), ("$ver", 6)]).expect("op19"); m!(db, "main", "insert_person", &[("$name", "w7")], &[("$age", 20)]).expect("op20"); - db.cleanup(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None }).await.expect("op21"); + db.cleanup_as(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None }, None).await.expect("op21"); m!(db, "main", "set_age_v", &[("$name", "w7")], &[("$age", 17), ("$ver", 7)]).expect("op22"); m!(db, "main", "set_age_v", &[("$name", "w5")], &[("$age", 14), ("$ver", 8)]).expect("op23"); - db.cleanup(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None }).await.expect("op24"); + db.cleanup_as(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None }, None).await.expect("op24"); db.branch_create("b0").await.expect("op25"); let empty_merge = Box::pin(db.branch_merge("b0", "main")).await; println!("faithful op26 empty merge: {:?}", empty_merge.map(|_| "ok")); @@ -5373,7 +5379,7 @@ fn dst_liveness_oracle_survives_cross_thread_work() { tokio::time::resume(); let lively = tokio::time::timeout( std::time::Duration::from_secs(120), - Box::pin(db.ensure_indices()), + Box::pin(db.ensure_indices_as(None)), ) .await; tokio::time::pause(); @@ -6338,7 +6344,7 @@ fn dst_optimize_races_branch_delete_minimal_two_thread_negative() { let _ = mutate_main(&db, MUTATION_QUERIES, "set_age", ¶ms).await; } - match Box::pin(db.optimize()).await { + match Box::pin(db.optimize_as(None)).await { Ok(_) => {} Err(err) => { let rendered = format!("{err:?}"); diff --git a/crates/omnigraph-gqt-core/src/lib.rs b/crates/omnigraph-gqt-core/src/lib.rs index 69f117dc1..02b8f3afc 100644 --- a/crates/omnigraph-gqt-core/src/lib.rs +++ b/crates/omnigraph-gqt-core/src/lib.rs @@ -4294,7 +4294,7 @@ pub async fn seed_case(session: &Session, seed: &Seed, needs_indices: bool) -> R seed.load(session).await?; if needs_indices { session - .ensure_indices() + .ensure_indices_as(None) .await .map_err(|e| format!("ensure_indices failed: {e}"))?; } diff --git a/crates/omnigraph-policy/src/lib.rs b/crates/omnigraph-policy/src/lib.rs index 3e1c18422..2b2383b7f 100644 --- a/crates/omnigraph-policy/src/lib.rs +++ b/crates/omnigraph-policy/src/lib.rs @@ -30,6 +30,16 @@ pub enum PolicyAction { /// Drop an unpublished graph commit (`commit drop ""`). Target-branch /// scoped: the branch the proposal would have landed on. CommitDrop, + /// Content-preserving maintenance: optimize, index builds, the full-text + /// rebuild and the repair report. Branch scoped like `change`: the scope + /// is the branch whose tables the operation maintains, `main` for + /// optimize and repair. It grants no right to change data or to delete + /// history. + Maintain, + /// Reclaim storage: cleanup, which permanently deletes table versions and + /// files no retained commit needs. A run covers every branch's history, + /// so the action is graph scoped and takes no branch scope. + Reclaim, /// Reserved action; no current engine or HTTP operation uses this grant. Admin, /// List the server's graph registry. This is server-scoped, with no branch @@ -57,6 +67,8 @@ impl PolicyAction { Self::BranchMerge => "branch_merge", Self::CommitPublish => "commit_publish", Self::CommitDrop => "commit_drop", + Self::Maintain => "maintain", + Self::Reclaim => "reclaim", Self::Admin => "admin", Self::GraphList => "graph_list", Self::InvokeQuery => "invoke_query", @@ -65,7 +77,10 @@ impl PolicyAction { } fn uses_branch_scope(self) -> bool { - matches!(self, Self::Read | Self::Export | Self::Change) + matches!( + self, + Self::Read | Self::Export | Self::Change | Self::Maintain + ) } fn uses_target_branch_scope(self) -> bool { @@ -98,6 +113,8 @@ impl PolicyAction { | Self::BranchMerge | Self::CommitPublish | Self::CommitDrop + | Self::Maintain + | Self::Reclaim | Self::Admin | Self::InvokeQuery => PolicyResourceKind::Graph, } @@ -160,6 +177,8 @@ impl FromStr for PolicyAction { "branch_merge" => Ok(Self::BranchMerge), "commit_publish" => Ok(Self::CommitPublish), "commit_drop" => Ok(Self::CommitDrop), + "maintain" => Ok(Self::Maintain), + "reclaim" => Ok(Self::Reclaim), "admin" => Ok(Self::Admin), "graph_list" => Ok(Self::GraphList), "invoke_query" => Ok(Self::InvokeQuery), @@ -942,6 +961,8 @@ namespace Omnigraph { action "branch_merge" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "commit_publish" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "commit_drop" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; + action "maintain" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; + action "reclaim" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "admin" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "invoke_query" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; @@ -1095,6 +1116,35 @@ pub trait PolicyChecker: Send + Sync { scope: &ResourceScope, actor: &str, ) -> Result<(), PolicyError>; + + /// Gate for an operation that any one of `actions` admits on `scope`, + /// such as the full-text rebuild, which `maintain` and its older grant + /// `change` both admit. Each action is decided as [`Self::check`] decides + /// it, in order, and the first allowed one admits the operation. An + /// evaluation failure is returned at once rather than passed over for + /// the next action, so a broken rule never widens access. When every + /// action is denied, the denial names each one. + fn check_any( + &self, + actions: &[PolicyAction], + scope: &ResourceScope, + actor: &str, + ) -> Result<(), PolicyError> { + if actions.is_empty() { + return Err(PolicyError::Internal( + "no policy action to check".to_string(), + )); + } + let mut denials = Vec::with_capacity(actions.len()); + for &action in actions { + match self.check(action, scope, actor) { + Ok(()) => return Ok(()), + Err(PolicyError::Denied(message)) => denials.push(message), + Err(internal) => return Err(internal), + } + } + Err(PolicyError::Denied(denials.join("; "))) + } } impl PolicyChecker for PolicyEngine { @@ -1951,4 +2001,223 @@ rules: assert!(PolicyEngine::load_cluster_from_source(&mixed).is_err()); } } + + #[test] + fn every_action_round_trips_through_its_name() { + use clap::ValueEnum; + for &action in PolicyAction::value_variants() { + let name = action.as_str(); + assert_eq!(action.to_string(), name); + assert_eq!(name.parse::().unwrap(), action, "{name}"); + assert_eq!( + serde_json::to_value(action).unwrap(), + serde_json::Value::String(name.to_string()) + ); + assert_eq!( + serde_json::from_value::(serde_json::json!(name)).unwrap(), + action + ); + } + assert_eq!( + "maintain".parse::().unwrap(), + PolicyAction::Maintain + ); + assert_eq!( + "reclaim".parse::().unwrap(), + PolicyAction::Reclaim + ); + assert!("maintenance".parse::().is_err()); + } + + /// A rule granting any one action compiles under the Cedar schema and + /// allows that action, so no action can be added to the enum without its + /// declaration. + #[test] + fn every_action_is_declared_in_the_cedar_schema() { + use clap::ValueEnum; + for &action in PolicyAction::value_variants() { + let source = format!( + "version: 1\ngroups:\n g: [act]\nrules:\n - id: r\n allow:\n actors: {{ group: g }}\n actions: [{}]\n", + action.as_str() + ); + let engine = match action.resource_kind() { + PolicyResourceKind::Graph => PolicyEngine::load_graph_from_source(&source, "graph"), + PolicyResourceKind::Server => PolicyEngine::load_server_from_source(&source), + PolicyResourceKind::Cluster => PolicyEngine::load_cluster_from_source(&source), + } + .unwrap_or_else(|error| panic!("{action}: {error}")); + let decision = engine + .authorize( + "act", + &PolicyRequest { + action, + branch: None, + target_branch: None, + }, + ) + .unwrap(); + assert!(decision.allowed, "{action}: {}", decision.message); + } + } + + #[test] + fn maintain_is_branch_scoped_and_reclaim_takes_no_branch_scope() { + let policy: PolicyConfig = yaml_serde::from_str( + r#" +version: 1 +groups: + automation: [act-bot] + writers: [act-writer] +protected_branches: [main] +rules: + - id: automation-maintains-unprotected + allow: + actors: { group: automation } + actions: [maintain] + branch_scope: unprotected + - id: writers-change + allow: + actors: { group: writers } + actions: [change] + branch_scope: any +"#, + ) + .unwrap(); + let engine = PolicyCompiler::compile(&policy, "graph").unwrap(); + let decide = |actor: &str, action, branch: Option<&str>| { + engine + .authorize( + actor, + &PolicyRequest { + action, + branch: branch.map(str::to_string), + target_branch: None, + }, + ) + .unwrap() + .allowed + }; + assert!(decide("act-bot", PolicyAction::Maintain, Some("feature"))); + assert!(!decide("act-bot", PolicyAction::Maintain, Some("main"))); + // Maintenance grants neither data writes nor reclamation. + assert!(!decide("act-bot", PolicyAction::Change, Some("feature"))); + assert!(!decide("act-bot", PolicyAction::Reclaim, None)); + // And `change` grants no maintenance. + assert!(!decide( + "act-writer", + PolicyAction::Maintain, + Some("feature") + )); + assert!(!decide("act-writer", PolicyAction::Reclaim, None)); + + let reclaim = r#" +version: 1 +groups: + automation: [act-bot] +rules: + - id: automation-reclaims + allow: + actors: { group: automation } + actions: [reclaim] +"#; + let engine = PolicyEngine::load_graph_from_source(reclaim, "graph").unwrap(); + let allowed = engine + .authorize( + "act-bot", + &PolicyRequest { + action: PolicyAction::Reclaim, + branch: None, + target_branch: None, + }, + ) + .unwrap(); + assert!(allowed.allowed); + assert_eq!( + PolicyAction::Reclaim.resource_kind(), + PolicyResourceKind::Graph + ); + for scope in ["branch_scope", "target_branch_scope"] { + let scoped = reclaim.replace( + "actions: [reclaim]", + &format!("actions: [reclaim]\n {scope}: any"), + ); + let error = PolicyEngine::load_graph_from_source(&scoped, "graph").unwrap_err(); + assert!( + error.to_string().contains("unsupported action 'reclaim'"), + "{scope}: {error}" + ); + } + let target_scoped_maintain = reclaim.replace("[reclaim]", "[maintain]").replace( + "actions: [maintain]", + "actions: [maintain]\n target_branch_scope: any", + ); + assert!(PolicyEngine::load_graph_from_source(&target_scoped_maintain, "graph").is_err()); + } + + #[test] + fn check_any_admits_on_any_allowed_action_and_names_every_denial() { + use super::{PolicyChecker, PolicyError, ResourceScope}; + + let policy = PolicyEngine::load_graph_from_source( + r#" +version: 1 +groups: + maintainers: [act-maintainer] + writers: [act-writer] + readers: [act-reader] +rules: + - id: maintainers + allow: + actors: { group: maintainers } + actions: [maintain] + - id: writers + allow: + actors: { group: writers } + actions: [change] +"#, + "graph", + ) + .unwrap(); + let either = [PolicyAction::Maintain, PolicyAction::Change]; + let scope = ResourceScope::Branch("main".to_string()); + for actor in ["act-maintainer", "act-writer"] { + policy.check_any(&either, &scope, actor).unwrap(); + } + match policy.check_any(&either, &scope, "act-reader") { + Err(PolicyError::Denied(message)) => { + assert!( + message.contains("'maintain'") && message.contains("'change'"), + "{message}" + ); + } + other => panic!("expected a denial naming both actions, got {other:?}"), + } + assert!(matches!( + policy.check_any(&[], &scope, "act-maintainer"), + Err(PolicyError::Internal(_)) + )); + + /// Fails to evaluate `maintain` and allows everything else. + struct BrokenMaintain; + impl PolicyChecker for BrokenMaintain { + fn check( + &self, + action: PolicyAction, + _scope: &ResourceScope, + _actor: &str, + ) -> Result<(), PolicyError> { + match action { + PolicyAction::Maintain => Err(PolicyError::Internal("broken".to_string())), + _ => Ok(()), + } + } + } + assert!( + matches!( + BrokenMaintain.check_any(&either, &scope, "act"), + Err(PolicyError::Internal(message)) if message == "broken" + ), + "an evaluation failure must not fall through to the next action" + ); + } } diff --git a/crates/omnigraph/src/db/graph_coordinator.rs b/crates/omnigraph/src/db/graph_coordinator.rs index f2d9d1cfe..ae540613d 100644 --- a/crates/omnigraph/src/db/graph_coordinator.rs +++ b/crates/omnigraph/src/db/graph_coordinator.rs @@ -896,7 +896,10 @@ impl GraphCoordinator { /// each immutable table identity matches what the caller observed before /// writing; the diagnostic alias is checked as part of the expectation. /// Mismatches surface as `OmniError::Manifest` with - /// `ManifestConflictDetails::PublishedDatasetVersionMismatch`. + /// `ManifestConflictDetails::PublishedDatasetVersionMismatch`. It publishes + /// over whatever head the branch has (`PublishPrecondition::Any`), so no + /// production writer uses it; tests and failpoint fixtures do. + #[cfg(any(test, feature = "failpoints"))] pub(crate) async fn commit_updates_with_actor_with_expected( &mut self, updates: &[DatasetUpdate], @@ -915,6 +918,7 @@ impl GraphCoordinator { /// no separate write, no `commit_graph.refresh()` to pick a parent (the /// publisher resolves it under the CAS). The in-memory commit cache is then /// updated from the intent + the resolved parent without a re-read. + #[cfg(any(test, feature = "failpoints"))] async fn commit_changes_with_actor_with_expected( &mut self, changes: &[ManifestChange], @@ -1104,6 +1108,7 @@ pub(crate) fn commit_not_found(snapshot_id: &SnapshotId) -> OmniError { } /// Wrap each `DatasetUpdate` as a `ManifestChange::Update` for the publisher. +#[cfg(any(test, feature = "failpoints"))] fn updates_to_changes(updates: &[DatasetUpdate]) -> Vec { updates .iter() diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 85f3d3415..2e19d14af 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -22,7 +22,7 @@ pub use omnigraph::{ DatasetRepairStats, EXPORT_CHUNK_MAX_BYTES, ExportCut, FullTextIndexRebuildResult, GraphCreateReconciliation, InitOptions, MergeOutcome, MergeResult, Omnigraph, OpenMode, PendingIndex, PreparedGraphCreate, PreparedSchemaApply, PreparedSchemaSettlement, ReadCheck, - RebuiltFullTextIndex, RepairAction, RepairClassification, RepairOptions, RepairStats, + RebuiltFullTextIndex, RepairAction, RepairClassification, RepairStats, RetainedManifestVersions, SYSTEM_COLUMNS_PREFLIGHT, SchemaApplyReconciliation, SchemaApplyResult, SchemaApplySettlement, SchemaContractDigest, SchemaNonPublicationProof, SkipReason, StagingVerdict, SystemColumnUpgradeFinding, SystemColumnUpgradeOptions, diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 2a5b62ff7..db0e0e3c0 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -53,9 +53,7 @@ pub(crate) use export::{ pub use optimize::{CleanupPolicyOptions, DatasetCleanupStats, DatasetOptimizeStats, SkipReason}; use prepared_create::initial_schema_ir; pub use prepared_create::{GraphCreateReconciliation, PreparedGraphCreate}; -pub use repair::{ - DatasetRepairStats, RepairAction, RepairClassification, RepairOptions, RepairStats, -}; +pub use repair::{DatasetRepairStats, RepairAction, RepairClassification, RepairStats}; pub use schema_apply::{ PreparedSchemaApply, PreparedSchemaSettlement, SchemaApplyReconciliation, SchemaApplySettlement, SchemaContractDigest, SchemaNonPublicationProof, @@ -1130,6 +1128,19 @@ impl Omnigraph { action: omnigraph_policy::PolicyAction, scope: &omnigraph_policy::ResourceScope, actor: Option<&str>, + ) -> Result<()> { + self.enforce_any(&[action], scope, actor) + } + + /// [`Self::enforce`] for an operation that any one of `actions` admits on + /// `scope`, decided by [`omnigraph_policy::PolicyChecker::check_any`]: the + /// first allowed action admits it, an evaluation failure refuses it, and a + /// refusal names every denied action. + pub(crate) fn enforce_any( + &self, + actions: &[omnigraph_policy::PolicyAction], + scope: &omnigraph_policy::ResourceScope, + actor: Option<&str>, ) -> Result<()> { let (branch, target_branch) = scope.to_branch_pair(); for name in [branch, target_branch].into_iter().flatten() { @@ -1147,7 +1158,7 @@ impl Omnigraph { )); }; checker - .check(action, scope, actor) + .check_any(actions, scope, actor) .map_err(|err| OmniError::Policy(err.to_string())) } @@ -3528,14 +3539,23 @@ impl Omnigraph { /// under the final gates. /// Returns the declared indexes that could not be materialized on this /// pass (today: vector properties with no trainable vectors yet). They are - /// deferred, not errors; a later `ensure_indices`/`optimize` builds them - /// once the property is trainable. Reads stay correct (brute-force) meanwhile. - pub async fn ensure_indices(&self) -> Result> { - table_ops::ensure_indices(self).await + /// deferred, not errors; a later `ensure_indices_as`/`optimize_as` builds + /// them once the property is trainable. Reads stay correct (brute-force) + /// meanwhile. + /// + /// Works on the handle's current branch. Requires `maintain` on that + /// branch before any effect and records `actor` in the publication. + pub async fn ensure_indices_as(&self, actor: Option<&str>) -> Result> { + table_ops::ensure_indices_as(self, actor).await } - pub async fn ensure_indices_on(&self, branch: &str) -> Result> { - table_ops::ensure_indices_on(self, branch).await + /// [`Self::ensure_indices_as`] on `branch`, under `maintain` on `branch`. + pub async fn ensure_indices_on_as( + &self, + branch: &str, + actor: Option<&str>, + ) -> Result> { + table_ops::ensure_indices_on_as(self, branch, actor).await } /// Fully rebuild all declared or existing supported full-text indexes from @@ -3545,19 +3565,13 @@ impl Omnigraph { /// Rebuilt full-text indexes use the engine's default analyzer, including /// indexes originally created externally with custom tokenizer settings. /// - /// Unlike [`Self::ensure_indices_on`], this replaces existing FTS indexes - /// even when their row coverage is complete. It establishes the supported - /// analyzer generation after an upgrade; it does not rewrite old history. - /// With a policy installed, use the actor-aware counterpart. - pub async fn rebuild_full_text_indices_on( - &self, - branch: &str, - ) -> Result { - self.rebuild_full_text_indices_on_as(branch, None).await - } - - /// Actor-aware full-text rebuilding. Requires `Change` on the selected - /// branch before any effects and records the actor in the graph publication. + /// Unlike [`Self::ensure_indices_on_as`], this replaces existing FTS + /// indexes even when their row coverage is complete. It establishes the + /// supported analyzer generation after an upgrade; it does not rewrite old + /// history. + /// + /// Requires `maintain` or `change` on the selected branch before any + /// effect and records the actor in the graph publication. pub async fn rebuild_full_text_indices_on_as( &self, branch: &str, @@ -3584,26 +3598,34 @@ impl Omnigraph { } /// Compact small Lance fragments into fewer larger ones across every - /// backing dataset for a node or edge type on `main`. See [`optimize`] for details. - pub async fn optimize(&self) -> Result> { - optimize::optimize_all_datasets(self).await + /// backing dataset for a node or edge type on `main`. See [`optimize`] for + /// details. Requires `maintain` on `main` before any effect and records + /// `actor` in the publication. + pub async fn optimize_as( + &self, + actor: Option<&str>, + ) -> Result> { + optimize::optimize_all_datasets(self, actor).await } - /// Classify and explicitly repair uncovered graph-manifest/Lance-HEAD drift. See - /// [`repair`] for the distinction between safe maintenance drift and - /// suspicious/unverifiable drift. - pub async fn repair(&self, options: repair::RepairOptions) -> Result { - repair::repair_all_datasets(self, options).await + /// Report, per table on `main`, how its Lance linear history stands + /// against its registration. See [`repair`] for the classifications. The + /// report publishes nothing and adopts no drift. Requires `maintain` on + /// `main`. + pub async fn repair_report_as(&self, actor: Option<&str>) -> Result { + repair::repair_report(self, actor).await } /// Remove Lance manifests (and the fragments they uniquely own) per the /// given [`optimize::CleanupPolicyOptions`]. Destructive to version - /// history. See [`optimize`] for details. - pub async fn cleanup( + /// history. See [`optimize`] for details. Requires `reclaim` before any + /// effect; a run covers every branch, so the check names none. + pub async fn cleanup_as( &self, options: optimize::CleanupPolicyOptions, + actor: Option<&str>, ) -> Result> { - optimize::cleanup_all_datasets(self, options).await + optimize::cleanup_all_datasets(self, options, actor).await } /// The tracing collector's plan for `options`, deleting nothing and taking @@ -6185,7 +6207,7 @@ edge WorksAt: Person -> Company ); // The reconciler materializes it (Person has a row). - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let snapshot = db.snapshot().await; let ds = db .storage() @@ -6217,7 +6239,7 @@ edge WorksAt: Person -> Company db.apply_schema(&desired).await.unwrap(); // After the rewrite the reconciler restores index coverage. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let snapshot = db.snapshot().await; let ds = db .storage() diff --git a/crates/omnigraph/src/db/omnigraph/collector.rs b/crates/omnigraph/src/db/omnigraph/collector.rs index f803fd81d..24c168196 100644 --- a/crates/omnigraph/src/db/omnigraph/collector.rs +++ b/crates/omnigraph/src/db/omnigraph/collector.rs @@ -2262,7 +2262,7 @@ mod borrowed_origin_tests { load_company(&db, "initial-company").await; db.branch_create("legacy").await.unwrap(); load_person(&db, "borrowed", 41).await; - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let source_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); let entry = source_snapshot.dataset("node:Person").unwrap().clone(); let mut source = source_snapshot @@ -2376,7 +2376,7 @@ mod borrowed_origin_tests { assert!(!company_old.is_empty()); load_company(&db, "unrelated-current").await; load_person(&db, "main-current", 99).await; - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let options = super::super::optimize::CleanupPolicyOptions { keep_versions: Some(1), older_than: None, @@ -2391,7 +2391,7 @@ mod borrowed_origin_tests { source_plan.would_remove().contains(&source_version), "borrowed source manifest is reclaimable" ); - let rows = db.cleanup(options).await.unwrap(); + let rows = db.cleanup_as(options, None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let source_store = source.object_store(None).await.unwrap(); for path in source_data.iter().chain(&source_indices) { @@ -2504,7 +2504,7 @@ mod borrowed_origin_tests { .any(|table| table.would_remove().contains(&version)), "old source pin must be reclaimable without the native tag's borrowed files" ); - let rows = db.cleanup(options).await.unwrap(); + let rows = db.cleanup_as(options, None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); drop(db); // forbidden-api-allow: test-only cold native read of a tagged fork absent from graph registrations @@ -2745,7 +2745,7 @@ mod overlay_retention_tests { .any(|object| object.path == overlay_path) ); assert!(expected.missing_paths().await.unwrap().is_empty()); - let rows = db.cleanup(options).await.unwrap(); + let rows = db.cleanup_as(options, None).await.unwrap(); let row = rows .iter() .find(|row| row.type_key == entry.type_key) @@ -2793,7 +2793,7 @@ mod overlay_retention_tests { .any(|object| object.path == overlay_path), "the independent pre-cleanup probe must include overlays" ); - let rows = db.cleanup(options.clone()).await.unwrap(); + let rows = db.cleanup_as(options.clone(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let store = overlay.object_store(None).await.unwrap(); assert!(store.exists(&overlay_path).await.unwrap()); @@ -2842,7 +2842,7 @@ mod overlay_retention_tests { } else { load_person(&db, 99).await; } - let rows = db.cleanup(options).await.unwrap(); + let rows = db.cleanup_as(options, None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( !store.exists(&overlay_path).await.unwrap(), @@ -3020,7 +3020,7 @@ mod native_table_tag_tests { !plan.would_remove().contains(&version), "an exact native table tag must exclude its version from cleanup: {plan:?}", ); - let rows = db.cleanup(options.clone()).await.unwrap(); + let rows = db.cleanup_as(options.clone(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!(store.exists(&manifest_path).await.unwrap()); // forbidden-api-allow: test-only cold native tag read after the graph collector ran @@ -3056,7 +3056,7 @@ mod native_table_tag_tests { .any(|plan| plan.location == location && plan.would_remove().contains(&version)), "removing the tag must make the old version reclaimable" ); - let rows = db.cleanup(options).await.unwrap(); + let rows = db.cleanup_as(options, None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( !store.exists(&manifest_path).await.unwrap(), diff --git a/crates/omnigraph/src/db/omnigraph/optimize.rs b/crates/omnigraph/src/db/omnigraph/optimize.rs index dd5517308..e1ff20c91 100644 --- a/crates/omnigraph/src/db/omnigraph/optimize.rs +++ b/crates/omnigraph/src/db/omnigraph/optimize.rs @@ -216,7 +216,15 @@ decide_seam! { /// with an exact pin CAS, so one public Optimize produces at most one graph /// commit. The publisher owns main's `__manifest` fragment layout; maintenance /// only removes obsolete automatic-cleanup configuration from that dataset. -pub async fn optimize_all_datasets(db: &Omnigraph) -> Result> { +pub async fn optimize_all_datasets( + db: &Omnigraph, + actor: Option<&str>, +) -> Result> { + db.enforce( + omnigraph_policy::PolicyAction::Maintain, + &omnigraph_policy::ResourceScope::Branch("main".to_string()), + actor, + )?; let _export_exclusion = db.reserve_export_destructive_control()?; db.ensure_schema_state_valid().await?; @@ -324,14 +332,14 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result, ) -> Result> { + db.enforce( + omnigraph_policy::PolicyAction::Reclaim, + &omnigraph_policy::ResourceScope::Graph, + actor, + )?; if options.keep_versions.is_none() && options.older_than.is_none() { return Err(OmniError::manifest( "cleanup requires at least one of keep_versions or older_than", @@ -1677,10 +1691,13 @@ mod tests { let bytes = store.inner.get(&path).await.unwrap().bytes().await.unwrap(); store.delete(&path).await.unwrap(); let rows = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!( @@ -1694,10 +1711,13 @@ mod tests { ); store.inner.put(&path, bytes.into()).await.unwrap(); let rows = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); diff --git a/crates/omnigraph/src/db/omnigraph/repair.rs b/crates/omnigraph/src/db/omnigraph/repair.rs index ff0ac8ee8..d2d06fca2 100644 --- a/crates/omnigraph/src/db/omnigraph/repair.rs +++ b/crates/omnigraph/src/db/omnigraph/repair.rs @@ -1,27 +1,17 @@ -//! Explicit repair for uncovered published-dataset/Lance-HEAD drift. +//! The repair report: uncovered published-dataset/Lance-HEAD drift. //! //! Pending pins converge automatically: the next writer or `cleanup` promotes //! them. This module is for the different case: a dataset's Lance HEAD is //! ahead of the version recorded in `__manifest` and no published pin explains -//! the movement. `repair` classifies that uncovered drift from Lance -//! transactions and only auto-publishes maintenance-only drift when the -//! operator confirms. +//! the movement. The report classifies that drift from Lance transactions. It +//! publishes nothing: no read or write resolves a table's linear HEAD, so +//! drift never needs adopting. use lance::Dataset; use lance::dataset::transaction::Operation; use super::*; -/// Options for [`Omnigraph::repair`]. -#[derive(Debug, Clone, Copy, Default)] -pub struct RepairOptions { - /// Preview by default. With `confirm`, verified maintenance drift is - /// published to `__manifest`. - pub confirm: bool, - /// Also publish suspicious/unverifiable drift. Requires `confirm`. - pub force: bool, -} - /// Classification of a dataset's graph-manifest/Lance-HEAD state. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -65,20 +55,15 @@ impl std::fmt::Display for RepairClassification { } } -/// What repair did for a backing dataset. +/// What the repair report says about a backing dataset. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum RepairAction { - /// Nothing to do. + /// Nothing to report: no drift, or foreign drift no read or write resolves. NoOp, - /// Drift was reported but not published because this was a preview. + /// Drift the report classified and left in place. Preview, - /// Verified maintenance drift was published to `__manifest`. - Healed, - /// Suspicious/unverifiable drift was published because `force` was set. - Forced, - /// Drift was left untouched because it was not safe to publish without - /// `force`. + /// The table could not be judged; its `error` says why. Refused, } @@ -88,8 +73,6 @@ impl RepairAction { match self { Self::NoOp => "no_op", Self::Preview => "preview", - Self::Healed => "healed", - Self::Forced => "forced", Self::Refused => "refused", } } @@ -101,13 +84,13 @@ impl std::fmt::Display for RepairAction { } } -/// Per-dataset repair outcome. +/// One dataset's row of the repair report. #[derive(Debug, Clone)] #[non_exhaustive] pub struct DatasetRepairStats { /// Qualified graph type key, or `__manifest` for the system dataset. pub type_key: String, - /// Published dataset version before repair. + /// The published dataset version the registration pins. pub published_dataset_version: u64, pub lance_head_version: u64, pub classification: RepairClassification, @@ -116,13 +99,11 @@ pub struct DatasetRepairStats { pub error: Option, } -/// Whole-graph repair outcome. +/// The repair report for the whole graph. #[derive(Debug, Clone)] #[non_exhaustive] pub struct RepairStats { pub datasets: Vec, - /// New graph-manifest version if repair published any dataset pins. - pub graph_manifest_version: Option, } struct ClassificationResult { @@ -132,31 +113,27 @@ struct ClassificationResult { } struct RepairTableTask { - identity: crate::db::manifest::TableIdentity, table_key: String, full_path: String, pinned_data_version: u64, - pinned_native_ref: Option, entry: crate::db::manifest::DatasetEntry, } -pub async fn repair_all_datasets(db: &Omnigraph, options: RepairOptions) -> Result { - if options.force && !options.confirm { - return Err(OmniError::manifest("repair --force requires --confirm")); - } - +/// The repair report on `main`, under `maintain` on `main`. +pub(super) async fn repair_report(db: &Omnigraph, actor: Option<&str>) -> Result { + db.enforce( + omnigraph_policy::PolicyAction::Maintain, + &omnigraph_policy::ResourceScope::Branch("main".to_string()), + actor, + )?; db.ensure_schema_state_valid().await?; - // Repair may adopt physical HEADs into graph authority. Bind the entire - // attempt to one accepted view and revalidate after schema -> main -> table - // gates so concurrent graph movement refuses before publication. + // Bind the whole report to one accepted view, revalidated after the + // schema -> main -> table gates. The accepted catalog, the identity/path + // pairs and every raw Lance read stay under those gates, so a concurrent + // schema apply cannot pair an old dataset path with a replacement's alias, + // and a cleanup cannot remove a version the report is reading. let authority_txn = db.open_write_txn(None).await?; - - // Repair publishes manifest authority, so it joins the canonical writer - // envelope. The accepted catalog, identity/path pairs, raw Lance reads, and - // final publish all remain under schema -> main -> sorted-table gates. This - // prevents a concurrent drop/re-add from pairing the old dataset path with - // the replacement's same public alias and new identity. let _schema_permit = db.write_queue().acquire_schema_shared().await; db.refresh_coordinator_only().await?; let catalog = db.load_accepted_catalog_with_schema_gate_held().await?; @@ -175,41 +152,26 @@ pub async fn repair_all_datasets(db: &Omnigraph, options: RepairOptions) -> Resu .filter_map(|table_key| { let entry = snapshot.dataset(&table_key)?; Some(RepairTableTask { - identity: entry.identity, table_key, full_path: format!("{}/{}", db.root_uri, entry.dataset_path), pinned_data_version: entry.published_dataset_version, - pinned_native_ref: entry.native_dataset_branch.clone(), entry: entry.clone(), }) }) .collect::>(); - if table_tasks.is_empty() { - return Ok(RepairStats { - datasets: Vec::new(), - graph_manifest_version: None, - }); - } - let mut tables = Vec::with_capacity(table_tasks.len()); - let mut updates = Vec::new(); - let mut expected = crate::db::manifest::ExpectedTableVersions::new(); - let mut any_forced = false; - for task in table_tasks { let RepairTableTask { - identity, table_key, full_path, pinned_data_version, - pinned_native_ref, entry, } = task; // `classify_drift` inspects raw Lance transaction history // (`read_transaction_by_version`), a Lance-only maintenance read the // staged-write trait does not surface. The raw borrow is an enumerated, - // read-only escape; repair never takes ownership or moves Lance HEAD. + // read-only escape; the report never takes ownership or moves Lance HEAD. let handle = db.storage().open_dataset_head(&full_path, None).await?; if let Some(row) = judge_against_last_linear_version(&table_key, &entry, &handle)? { tables.push(row); @@ -262,90 +224,18 @@ pub async fn repair_all_datasets(db: &Omnigraph, options: RepairOptions) -> Resu } let classification = classify_drift(ds, pinned_data_version, lance_head_version).await; - let action = match ( - options.confirm, - options.force, - classification.classification, - ) { - (false, _, _) => RepairAction::Preview, - (true, _, RepairClassification::VerifiedMaintenance) => RepairAction::Healed, - (true, true, RepairClassification::Suspicious | RepairClassification::Unverifiable) => { - any_forced = true; - RepairAction::Forced - } - (true, _, RepairClassification::Suspicious | RepairClassification::Unverifiable) => { - RepairAction::Refused - } - (true, _, RepairClassification::NoDrift) => RepairAction::NoOp, - // Reported above, before drift is classified; never adopted. - (true, _, RepairClassification::BlockedPromotion) => RepairAction::Refused, - (true, _, RepairClassification::ForeignDrift) => RepairAction::NoOp, - }; - - if matches!(action, RepairAction::Healed | RepairAction::Forced) { - let state = db.storage().table_state(&full_path, &handle).await?; - updates.push(crate::db::DatasetUpdate { - identity, - type_key: table_key.clone(), - published_dataset_version: state.version, - native_dataset_branch: None, - entity_count: state.row_count, - version_metadata: state - .version_metadata - .with_last_linear_version(entry.version_metadata.last_linear_version()), - }); - expected.insert( - identity, - crate::db::manifest::TableVersionExpectation { - table_key: table_key.clone(), - table_version: pinned_data_version, - native_ref: crate::db::manifest::NativeRefPin::Exact(pinned_native_ref), - }, - ); - } - tables.push(DatasetRepairStats { type_key: table_key, published_dataset_version: pinned_data_version, lance_head_version, classification: classification.classification, - action, + action: RepairAction::Preview, operations: classification.operations, error: classification.error, }); } - let manifest_version = if updates.is_empty() { - None - } else { - let actor = if any_forced { - Some("omnigraph:repair:force") - } else { - Some("omnigraph:repair") - }; - let PublishedSnapshot { - graph_manifest_version: manifest_version, - .. - } = db - .coordinator - .write() - .await - .commit_updates_with_actor_with_expected(&updates, &expected, actor) - .await?; - db.runtime_cache.invalidate_all().await; - if updates - .iter() - .any(|update| update.type_key.starts_with("edge:")) - { - db.invalidate_graph_index().await; - } - Some(manifest_version) - }; - - Ok(RepairStats { - datasets: tables, - graph_manifest_version: manifest_version, - }) + Ok(RepairStats { datasets: tables }) } /// The detached-only rule: the recorded last linear version is the baseline diff --git a/crates/omnigraph/src/db/omnigraph/table_ops.rs b/crates/omnigraph/src/db/omnigraph/table_ops.rs index 68875383a..4e0619bba 100644 --- a/crates/omnigraph/src/db/omnigraph/table_ops.rs +++ b/crates/omnigraph/src/db/omnigraph/table_ops.rs @@ -35,19 +35,26 @@ pub(super) async fn graph_index_for_resolved( .await } -pub(super) async fn ensure_indices(db: &Omnigraph) -> Result> { +pub(super) async fn ensure_indices_as( + db: &Omnigraph, + actor: Option<&str>, +) -> Result> { let current_branch = db .coordinator .read() .await .current_branch() .map(str::to_string); - ensure_indices_for_branch(db, current_branch.as_deref()).await + ensure_indices_for_branch(db, current_branch.as_deref(), actor).await } -pub(super) async fn ensure_indices_on(db: &Omnigraph, branch: &str) -> Result> { +pub(super) async fn ensure_indices_on_as( + db: &Omnigraph, + branch: &str, + actor: Option<&str>, +) -> Result> { let branch = normalize_branch_name(branch)?; - ensure_indices_for_branch(db, branch.as_deref()).await + ensure_indices_for_branch(db, branch.as_deref(), actor).await } /// The indexes replaced by one successfully published full-text rebuild. @@ -86,8 +93,13 @@ pub(super) async fn rebuild_full_text_indices_on_as( ) -> Result { let branch = normalize_branch_name(branch)?; let public_branch = branch.as_deref().unwrap_or("main"); - db.enforce( - omnigraph_policy::PolicyAction::Change, + // `maintain` is the rebuild's grant; `change`, its grant before + // `maintain` existed, still admits it so existing policies keep working. + db.enforce_any( + &[ + omnigraph_policy::PolicyAction::Maintain, + omnigraph_policy::PolicyAction::Change, + ], &omnigraph_policy::ResourceScope::Branch(public_branch.to_string()), actor, )?; @@ -186,12 +198,19 @@ pub(super) async fn failpoint_publish_table_head_without_index_rebuild_for_test( .await } -pub(super) async fn ensure_indices_for_branch( +/// The index writer on `branch` (`None` is main), under `maintain` on it. +async fn ensure_indices_for_branch( db: &Omnigraph, branch: Option<&str>, + actor: Option<&str>, ) -> Result> { + db.enforce( + omnigraph_policy::PolicyAction::Maintain, + &omnigraph_policy::ResourceScope::Branch(branch.unwrap_or("main").to_string()), + actor, + )?; Ok( - maintain_indices_for_branch(db, branch, IndexMaintenanceMode::Ensure, None) + maintain_indices_for_branch(db, branch, IndexMaintenanceMode::Ensure, actor) .await? .pending, ) diff --git a/crates/omnigraph/src/db/upgrade/tests.rs b/crates/omnigraph/src/db/upgrade/tests.rs index 39ab4cb9d..30fa83bb8 100644 --- a/crates/omnigraph/src/db/upgrade/tests.rs +++ b/crates/omnigraph/src/db/upgrade/tests.rs @@ -2752,7 +2752,7 @@ async fn cleanup_after_upgrade_with_keep_four_and_older_than() { .collect(); assert_eq!(links.len(), 1, "{links:?}"); for (round, options) in [week, keep(4)].into_iter().enumerate() { - let stats = graph.cleanup(options).await.unwrap(); + let stats = graph.cleanup_as(options, None).await.unwrap(); let reclaimed = |table_key: &str| { if round == 0 && table_key == PERSON { links.len() as u64 @@ -2800,7 +2800,7 @@ async fn merge_with_legacy_base_pins_and_collector_keeps_it() { "the merge base of feature and main pins Person version {}: {roots:?}", source.a_root ); - let stats = graph.cleanup(keep(1)).await.unwrap(); + let stats = graph.cleanup_as(keep(1), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); session(&graph) @@ -2897,7 +2897,7 @@ async fn leftover_merge_input_tag_on_bookkeeping_version_resolves() { roots.is_superset(&[source.b_root, source.c_root].into()), "{roots:?}" ); - let stats = graph.cleanup(keep(1)).await.unwrap(); + let stats = graph.cleanup_as(keep(1), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); assert_eq!( open(root, None) diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index fcd88b8a7..f4bb53522 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -926,7 +926,7 @@ query people() { match { $p: Person } return { count($p) as n } } ) .await .unwrap(); - session.ensure_indices().await.unwrap(); + session.ensure_indices_as(None).await.unwrap(); let settings = SessionSettings::default(); let (view, catalog) = db .capture_read_view(ReadTarget::branch("main")) diff --git a/crates/omnigraph/src/runtime_cache.rs b/crates/omnigraph/src/runtime_cache.rs index be6267f34..3e3c00b6e 100644 --- a/crates/omnigraph/src/runtime_cache.rs +++ b/crates/omnigraph/src/runtime_cache.rs @@ -648,7 +648,7 @@ edge Likes: Person -> Person {} db.load_jsonl(DATA, crate::loader::LoadMode::Overwrite) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); // Fresh handle: cold in-memory caches, the artifact stays on the store. let db = crate::db::Omnigraph::open(dir.path().to_str().unwrap()) .await diff --git a/crates/omnigraph/tests/bound_storage.rs b/crates/omnigraph/tests/bound_storage.rs index 4641a513d..a4f989498 100644 --- a/crates/omnigraph/tests/bound_storage.rs +++ b/crates/omnigraph/tests/bound_storage.rs @@ -303,11 +303,14 @@ async fn bound_handle_workload_child() { .contains("Feature") ); - db.optimize().await.unwrap(); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(2), - older_than: None, - }) + db.optimize_as(None).await.unwrap(); + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(2), + older_than: None, + }, + None, + ) .await .unwrap(); let reopened = session(Omnigraph::open_with_storage(&uri, storage()).await.unwrap()); diff --git a/crates/omnigraph/tests/detached_commit_matrix.rs b/crates/omnigraph/tests/detached_commit_matrix.rs index d0a5c2792..2f7e9dc31 100644 --- a/crates/omnigraph/tests/detached_commit_matrix.rs +++ b/crates/omnigraph/tests/detached_commit_matrix.rs @@ -414,15 +414,18 @@ fn rfc0067_matrix_child_process() { .block_on(async move { let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); let outcome: omnigraph::error::Result<()> = match op.as_str() { - "cleanup" => db.cleanup(reclaim_everything()).await.map(|_| ()), + "cleanup" => db.cleanup_as(reclaim_everything(), None).await.map(|_| ()), "cleanup_keep" => db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .map(|_| ()), - "ensure_indices" => db.ensure_indices().await.map(|_| ()), + "ensure_indices" => db.ensure_indices_as(None).await.map(|_| ()), "merge" => db .branch_merge(&format!("src_{name}"), "main") .await @@ -437,9 +440,12 @@ fn rfc0067_matrix_child_process() { .unwrap(); db.apply_prepared_schema_as(&intent, None).await.map(|_| ()) } - "optimize" => db.optimize().await.map(|_| ()), + "optimize" => db.optimize_as(None).await.map(|_| ()), "load" => load_two(&db, &name).await, - "fts_rebuild" => db.rebuild_full_text_indices_on("main").await.map(|_| ()), + "fts_rebuild" => db + .rebuild_full_text_indices_on_as("main", None) + .await + .map(|_| ()), "system_column_upgrade" => db .upgrade_system_columns(upgrade_execute()) .await @@ -696,7 +702,7 @@ async fn run_cell( ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); } // A clear cell clears a photo a put stored first. if writer == Writer::BlobClear { @@ -736,8 +742,10 @@ async fn run_cell( let outcome: omnigraph::error::Result<()> = match writer { Writer::Insert => insert(&db, &write_name).await, Writer::MultiTable => insert_and_friend(&db, &write_name).await, - Writer::Cleanup => Box::pin(db.cleanup(reclaim_everything())).await.map(|_| ()), - Writer::EnsureIndices => db.ensure_indices().await.map(|_| ()), + Writer::Cleanup => Box::pin(db.cleanup_as(reclaim_everything(), None)) + .await + .map(|_| ()), + Writer::EnsureIndices => db.ensure_indices_as(None).await.map(|_| ()), Writer::Merge => db .branch_merge(&format!("src_{write_name}"), "main") .await @@ -747,9 +755,12 @@ async fn run_cell( .apply_prepared_schema_as(prepared.as_ref().unwrap(), None) .await .map(|_| ()), - Writer::Optimize => db.optimize().await.map(|_| ()), + Writer::Optimize => db.optimize_as(None).await.map(|_| ()), Writer::Load => load_two(&db, &write_name).await, - Writer::FtsRebuild => db.rebuild_full_text_indices_on("main").await.map(|_| ()), + Writer::FtsRebuild => db + .rebuild_full_text_indices_on_as("main", None) + .await + .map(|_| ()), Writer::SystemColumnUpgrade => db .upgrade_system_columns(upgrade_execute()) .await @@ -879,7 +890,7 @@ async fn run_cell( } Recovery::Cleanup => { let fresh = helpers::session(Omnigraph::open(&root).await.unwrap()); - Box::pin(fresh.cleanup(reclaim_everything())) + Box::pin(fresh.cleanup_as(reclaim_everything(), None)) .await .unwrap_or_else(|error| panic!("{cell}: recovery cleanup failed: {error}")); } @@ -974,7 +985,7 @@ async fn run_cell( if writer == Writer::Optimize && recovery != Recovery::ReadOnly { let fresh = helpers::session(Omnigraph::open(&root).await.unwrap()); for run in 1..=2 { - fresh.optimize().await.unwrap_or_else(|error| { + fresh.optimize_as(None).await.unwrap_or_else(|error| { panic!("{cell}: optimize run {run} after recovery failed: {error}") }); assert_pin_detached_and_head_unmoved(&cell, &fresh, &person_uri, head_before).await; @@ -984,12 +995,12 @@ async fn run_cell( if writer == Writer::EnsureIndices && recovery != Recovery::ReadOnly { let fresh = helpers::session(Omnigraph::open(&root).await.unwrap()); fresh - .ensure_indices() + .ensure_indices_as(None) .await .unwrap_or_else(|error| panic!("{cell}: index pass after recovery failed: {error}")); assert_pin_detached_and_head_unmoved(&cell, &fresh, &person_uri, head_before).await; fresh - .ensure_indices() + .ensure_indices_as(None) .await .unwrap_or_else(|error| panic!("{cell}: second index pass failed: {error}")); drop(fresh); @@ -998,7 +1009,7 @@ async fn run_cell( let fresh = helpers::session(Omnigraph::open(&root).await.unwrap()); for run in 1..=2 { let rebuilt = fresh - .rebuild_full_text_indices_on("main") + .rebuild_full_text_indices_on_as("main", None) .await .unwrap_or_else(|error| { panic!("{cell}: full-text rebuild run {run} after recovery failed: {error}") diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index d0e1a6822..77fd7d211 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -322,7 +322,7 @@ fn rfc023_external_writer_process() { } "collector_advance_merge_source" => { db.load("source", &payload, mode).await.unwrap(); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); } "collector_retired" => { @@ -514,7 +514,7 @@ async fn branch_start_refuses_a_commit_an_interrupted_cleanup_emptied() { { let _fp = catalog::CLEANUP_SWEEP_PRE_MANIFEST_DELETE.fire_always(); - let _ = db.cleanup(keep_one()).await; + let _ = db.cleanup_as(keep_one(), None).await; } assert!( detached_versions(&person_uri).await.contains(&eve_person), @@ -546,7 +546,7 @@ async fn branch_start_refuses_a_commit_an_interrupted_cleanup_emptied() { assert_eq!(db.branch_list().await.unwrap(), vec!["main".to_string()]); assert_eq!(version_main(&db).await.unwrap(), main_version); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let person = stats .iter() .find(|row| row.type_key == "node:Person") @@ -623,7 +623,7 @@ async fn branch_delete_cleanup_failure_converges_on_retry() { ); { let _fp = catalog::CLEANUP_TABLE_GC.fire_always(); - main.cleanup(keep_one()).await.unwrap(); + main.cleanup_as(keep_one(), None).await.unwrap(); } assert!( detached_versions(&person_uri) @@ -631,7 +631,7 @@ async fn branch_delete_cleanup_failure_converges_on_retry() { .contains(&former_staging), "failed cleanup preserves the deleted branch's staging" ); - main.cleanup(keep_one()).await.unwrap(); + main.cleanup_as(keep_one(), None).await.unwrap(); assert!( !detached_versions(&person_uri) .await @@ -723,7 +723,7 @@ async fn recreate_over_a_deleted_branch_writes_without_cleanup() { "writes do not collect the former staging" ); assert!(remaining.contains(&second_staging)); - main.cleanup(keep_one()).await.unwrap(); + main.cleanup_as(keep_one(), None).await.unwrap(); let remaining = detached_versions(&person_uri).await; assert!(!remaining.contains(&first_staging)); assert!( @@ -780,7 +780,7 @@ async fn branch_delete_acknowledges_with_staging_awaiting_cleanup() { .contains(&former_staging), "delete and reopen defer collection of the branch's staging" ); - main.cleanup(keep_one()).await.unwrap(); + main.cleanup_as(keep_one(), None).await.unwrap(); assert!( !detached_versions(&person_uri) .await @@ -850,7 +850,7 @@ async fn branch_recreate_completes_while_old_staging_awaits_cleanup() { let remaining = detached_versions(&person_uri).await; assert!(remaining.contains(&first_staging)); assert!(remaining.contains(&second_staging)); - main.cleanup(keep_one()).await.unwrap(); + main.cleanup_as(keep_one(), None).await.unwrap(); let remaining = detached_versions(&person_uri).await; assert!(!remaining.contains(&first_staging)); assert!(remaining.contains(&second_staging)); @@ -923,10 +923,13 @@ async fn branch_write_ignores_unavailable_cleanup_classifier() { helpers::count_rows_branch(&db, "feature", "node:Person").await, 5 ); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let branches = lance::Dataset::open(&person_uri) @@ -970,10 +973,13 @@ async fn cleanup_isolates_single_table_failure() { // One table's version GC fails once; the sweep must isolate it. let _fp = catalog::CLEANUP_TABLE_GC.fire_once_at(1); let stats = db - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .expect("a single table's GC failure must not abort cleanup"); @@ -1024,10 +1030,13 @@ async fn cleanup_isolates_reconcile_failure() { // not abort. { let _fp = catalog::CLEANUP_RECONCILE_FORK.fire_once_at(1); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .expect("a reconcile force-delete failure must not abort cleanup"); } @@ -1044,10 +1053,13 @@ async fn cleanup_isolates_reconcile_failure() { ); } // A second cleanup with no injected failure converges. - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); { @@ -1096,10 +1108,13 @@ async fn reconcile_skips_fork_when_fresh_recheck_is_unavailable_then_converges() // branch is live but unreadable) → cleanup must SKIP it, not delete. { let _fp = catalog::CLASSIFY_FRESH_READ.fire_always(); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let ds = lance::Dataset::open(&person_uri).await.unwrap(); @@ -1114,10 +1129,13 @@ async fn reconcile_skips_fork_when_fresh_recheck_is_unavailable_then_converges() } // Read succeeds now → cleanup confirms the orphan and reclaims it (converges). - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); { @@ -2666,7 +2684,7 @@ async fn merge_target_delete_waits_and_source_delete_does_not_issue_643() { .unwrap() .unwrap_or_else(|error| panic!("deleting the target after the merge: {error}")); } else { - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert_eq!( helpers::count_rows_branch(&db, "t", "node:Person").await, @@ -3059,7 +3077,7 @@ async fn cancelled_merge_leaves_the_target_whole_issue_643() { ) .await .unwrap_or_else(|error| panic!("{seam}: a later write on the target: {error}")); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!( rows.iter().all(|row| row.error.is_none()), "{seam}: {rows:?}" @@ -3705,7 +3723,7 @@ async fn ensure_indices_stage_btree_failure_leaves_existing_tables_writable() { &catalog::ENSURE_INDICES_POST_STAGE_PRE_COMMIT_BTREE, ); let writer_a_db = std::sync::Arc::clone(&db); - let writer_a = tokio::spawn(async move { writer_a_db.ensure_indices().await }); + let writer_a = tokio::spawn(async move { writer_a_db.ensure_indices_as(None).await }); rendezvous.wait_until_reached().await; db.load( @@ -4043,15 +4061,9 @@ async fn schema_apply_retry_preserves_the_original_empty_create() { assert_eq!(entry.published_dataset_version, if changed { 2 } else { 1 }); assert_eq!(entry.version_metadata.staged_version().is_some(), changed); assert_eq!(entry.version_metadata.last_linear_version(), Some(1)); - recovered - .repair(omnigraph::db::RepairOptions { - confirm: true, - force: false, - }) - .await - .unwrap(); - recovered.ensure_indices().await.unwrap(); - let cleaned = recovered.cleanup(keep_one()).await.unwrap(); + recovered.repair_report_as(None).await.unwrap(); + recovered.ensure_indices_as(None).await.unwrap(); + let cleaned = recovered.cleanup_as(keep_one(), None).await.unwrap(); assert!(cleaned.iter().all(|row| row.error.is_none()), "{cleaned:?}"); drop(recovered); let reopened = helpers::session(Omnigraph::open(&uri).await.unwrap()); @@ -4475,7 +4487,7 @@ async fn optimize_pre_publish_failure_leaves_no_residue() { .version; { let _failpoint = catalog::OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT.fire_always(); - let err = db.optimize().await.unwrap_err(); + let err = db.optimize_as(None).await.unwrap_err(); assert!( err.to_string().contains( "injected failpoint triggered: optimize.post_phase_b_pre_manifest_commit" @@ -4502,7 +4514,7 @@ async fn optimize_pre_publish_failure_leaves_no_residue() { assert_eq!(helpers::count_rows(&db, "node:Person").await, 4); assert_eq!(helpers::count_rows(&db, "node:Company").await, 4); let stats = db - .optimize() + .optimize_as(None) .await .expect("the retry re-plans from the pins"); assert!( @@ -4533,7 +4545,7 @@ async fn optimize_lost_publish_acknowledgement_leaves_published_detached_pins() let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); let before = person_head_and_published(&db, "main").await; let _failpoint = catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT.fire_once_at(1); - db.optimize() + db.optimize_as(None) .await .expect_err("a lost publish acknowledgement surfaces as an error"); before @@ -4549,7 +4561,7 @@ async fn optimize_lost_publish_acknowledgement_leaves_published_detached_pins() before.1 ); assert_person_pin_detached(&recovered, "main", before.0).await; - let stats = recovered.optimize().await.unwrap(); + let stats = recovered.optimize_as(None).await.unwrap(); assert!( stats .iter() @@ -4585,7 +4597,7 @@ node Embedding { ) .await .unwrap(); - let pending = db.ensure_indices().await.unwrap(); + let pending = db.ensure_indices_as(None).await.unwrap(); assert!( pending .iter() @@ -4598,7 +4610,7 @@ node Embedding { ) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); let embedding = snapshot.open_dataset("node:Embedding").await.unwrap(); assert!(embedding.has_unindexed_fragments().await.unwrap()); @@ -4635,7 +4647,7 @@ node Embedding { .version; { let _failpoint = catalog::OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT.fire_once_at(1); - let error = db.optimize().await.unwrap_err(); + let error = db.optimize_as(None).await.unwrap_err(); assert!( !matches!(error, OmniError::RecoveryRequired { .. }), "{error}" @@ -4663,7 +4675,7 @@ node Embedding { ); drop(db); let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); let work = stats .iter() .find(|stat| stat.type_key == "node:Work") @@ -4724,7 +4736,7 @@ async fn optimize_partial_table_effect_leaves_no_residue() { .version() .version; let _failpoint = catalog::OPTIMIZE_POST_TABLE_EFFECT.fire_once_at(1); - let error = db.optimize().await.unwrap_err(); + let error = db.optimize_as(None).await.unwrap_err(); assert!( !matches!(error, OmniError::RecoveryRequired { .. }), "a partial batch before publication needs no recovery: {error}" @@ -4768,7 +4780,7 @@ async fn optimize_partial_table_effect_leaves_no_residue() { recovered.list_commits(Some("main")).await.unwrap().len(), commits_before ); - let stats = recovered.optimize().await.unwrap(); + let stats = recovered.optimize_as(None).await.unwrap(); assert_eq!( stats .iter() @@ -4816,7 +4828,7 @@ async fn optimize_refuses_when_graph_authority_moves_before_its_gates() { &catalog::OPTIMIZE_POST_AUTHORITY_CAPTURE_PRE_GATES, ); let optimize_task_db = std::sync::Arc::clone(&optimize_db); - let optimize = tokio::spawn(async move { optimize_task_db.optimize().await }); + let optimize = tokio::spawn(async move { optimize_task_db.optimize_as(None).await }); rendezvous.wait_until_reached().await; // Advance the graph head underneath it with an ordinary committed write, so @@ -4912,7 +4924,7 @@ async fn optimize_holds_main_gate_through_disjoint_table_effects() { let uri_opt = uri.clone(); let optimize = tokio::spawn(async move { let db = helpers::session(Omnigraph::open(&uri_opt).await.unwrap()); - db.optimize().await + db.optimize_as(None).await }); rendezvous.wait_until_reached().await; @@ -4956,7 +4968,7 @@ async fn optimize_holds_main_gate_through_disjoint_table_effects() { 1, "queued table-disjoint Company insert must persist", ); - db.optimize() + db.optimize_as(None) .await .expect("graph must remain healthy / re-optimizable"); } @@ -4996,7 +5008,7 @@ async fn optimize_serializes_concurrent_delete_across_handles() { let uri_opt = uri.clone(); let optimize = tokio::spawn(async move { let db = helpers::session(Omnigraph::open(&uri_opt).await.unwrap()); - db.optimize().await + db.optimize_as(None).await }); rendezvous.wait_until_reached().await; @@ -5056,7 +5068,7 @@ async fn optimize_serializes_concurrent_delete_across_handles() { 3, "the concurrent delete must persist (alice removed)", ); - db.optimize() + db.optimize_as(None) .await .expect("graph must remain healthy / re-optimizable"); } @@ -5410,7 +5422,7 @@ async fn ensure_indices_phase_b_failure_does_not_leak_sidecar_when_no_work_neede ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); } // Setup: trigger the failpoint. Steady-state ensure_indices @@ -5420,7 +5432,7 @@ async fn ensure_indices_phase_b_failure_does_not_leak_sidecar_when_no_work_neede { let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); let _failpoint = catalog::ENSURE_INDICES_POST_PHASE_B_PRE_MANIFEST_COMMIT.fire_always(); - let err = db.ensure_indices().await.unwrap_err(); + let err = db.ensure_indices_as(None).await.unwrap_err(); assert!( err.to_string().contains( "injected failpoint triggered: ensure_indices.post_phase_b_pre_manifest_commit" @@ -7337,7 +7349,7 @@ async fn ensure_indices_failure_before_publish_leaves_no_residue() { let version = helpers::version_main(&db).await.unwrap(); { let _failpoint = catalog::ENSURE_INDICES_POST_TABLE_EFFECT.fire_always(); - let err = db.ensure_indices().await.unwrap_err(); + let err = db.ensure_indices_as(None).await.unwrap_err(); assert!( err.to_string() .contains("injected failpoint triggered: ensure_indices.post_table_effect"), @@ -7348,7 +7360,7 @@ async fn ensure_indices_failure_before_publish_leaves_no_residue() { assert_eq!(person_head_and_published(&db, "main").await, before); assert_eq!(helpers::version_main(&db).await.unwrap(), version); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); assert_person_pin_detached(&db, "main", before.0).await; assert_eq!( person_head_and_published(&db, "main").await.1, @@ -7409,7 +7421,7 @@ async fn full_text_rebuild_on_a_branch_stages_on_the_inherited_table() { db.load_jsonl(SEARCH_DATA, LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db.branch_create("rebuild").await.unwrap(); let main_before = doc_head_and_published(&db, "main").await; let main_entry = helpers::snapshot_main(&db) @@ -7421,7 +7433,7 @@ async fn full_text_rebuild_on_a_branch_stages_on_the_inherited_table() { let main_pin = pinned_version(&db, "main", "node:Doc").await; let branch_version = helpers::version_branch(&db, "rebuild").await.unwrap(); - db.rebuild_full_text_indices_on("rebuild") + db.rebuild_full_text_indices_on_as("rebuild", None) .await .expect("a branch rebuild stages on the inherited table and never forks"); assert!(sidecar_operation_ids(dir.path()).is_empty()); @@ -7518,7 +7530,7 @@ async fn assert_optimize_detached_pin_survives_next_write(uri: &str) { { let _failpoint = catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT.fire_once_at(1); let error = db - .optimize() + .optimize_as(None) .await .expect_err("compaction published before its acknowledgement was lost"); assert!( @@ -8604,10 +8616,13 @@ async fn rfc_0067_cleanup_reaps_aged_surplus_detached_manifests() { } } for _ in 0..2 { - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::ZERO), - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::ZERO), + }, + None, + ) .await .unwrap(); } @@ -8672,10 +8687,13 @@ async fn rfc_0067_zero_age_cleanup_keeps_a_fresh_unpublished_detached_manifest() "the interrupted write must strand at least one detached manifest" ); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::ZERO), - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::ZERO), + }, + None, + ) .await .unwrap(); let after = detached_files(); @@ -8691,10 +8709,13 @@ async fn rfc_0067_zero_age_cleanup_keeps_a_fresh_unpublished_detached_manifest() .set_times(std::fs::FileTimes::new().set_modified(std::time::SystemTime::UNIX_EPOCH)) .unwrap(); } - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::ZERO), - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::ZERO), + }, + None, + ) .await .unwrap(); let aged = detached_files(); @@ -8716,10 +8737,13 @@ fn rfc_0067_external_cleanup_process() { .unwrap() .block_on(async { let db = Omnigraph::open(&uri).await.unwrap(); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::ZERO), - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::ZERO), + }, + None, + ) .await .unwrap(); }); @@ -9330,20 +9354,20 @@ async fn live_handle_writes_after_every_write_family_seam_failure() { db.apply_schema(&render_schema(&schema_props, age_indexed)) .await .expect("ensure-row schema setup"); - db.ensure_indices() + db.ensure_indices_as(None) .await .map(|_| ()) .map_err(|error| error.to_string()) } Driver::Optimize => db - .optimize() + .optimize_as(None) .await .map(|_| ()) .map_err(|error| error.to_string()), - Driver::Cleanup => Box::pin(db.cleanup(omnigraph::db::CleanupPolicyOptions { + Driver::Cleanup => Box::pin(db.cleanup_as(omnigraph::db::CleanupPolicyOptions { keep_versions: Some(1), older_than: None, - })) + }, None)) .await .map(|_| ()) .map_err(|error| error.to_string()), @@ -9419,10 +9443,13 @@ async fn rfc_0067_merge_diff_keeps_every_chunk() { after_rows - before_rows ); db.branch_delete("feature").await.unwrap(); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::ZERO), - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::ZERO), + }, + None, + ) .await .unwrap(); let person_uri = node_table_uri(&db, "Person").await; @@ -9462,10 +9489,13 @@ async fn rfc_0067_cleanup_window_keeps_version_gc_on_a_written_table() { for pass in 0..2 { let stats = db - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(std::time::Duration::from_secs(7 * 24 * 60 * 60)), - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(std::time::Duration::from_secs(7 * 24 * 60 * 60)), + }, + None, + ) .await .unwrap(); let person = stats @@ -9543,7 +9573,7 @@ async fn collector_keeps_live_staging_and_reclaims_it_once_the_head_moves() { ); assert!(staging.bytes > 0); } - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let row = stats .iter() .find(|row| row.type_key == "node:Person") @@ -9570,7 +9600,7 @@ async fn collector_keeps_live_staging_and_reclaims_it_once_the_head_moves() { staging.verdict ); } - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let row = stats .iter() .find(|row| row.type_key == "node:Person") @@ -9662,7 +9692,7 @@ async fn collector_keeps_pins_published_by_a_late_branch_in_another_process() { let expected_rows = count_rows(&db, "node:Person").await + 1; let cleaner = helpers::session(Omnigraph::open(db.uri()).await.unwrap()); let (guard, hold) = catalog::CLEANUP_COLLECTOR_POST_SNAPSHOT.hold(); - let cleanup = tokio::spawn(async move { cleaner.cleanup(keep_one()).await }); + let cleanup = tokio::spawn(async move { cleaner.cleanup_as(keep_one(), None).await }); let reached = hold.clone(); tokio::task::spawn_blocking(move || reached.wait_until_reached()) .await @@ -9705,7 +9735,7 @@ async fn collector_keeps_pins_published_by_a_late_branch_in_another_process() { path.display() ); } - let retry = reader.cleanup(keep_one()).await.unwrap(); + let retry = reader.cleanup_as(keep_one(), None).await.unwrap(); assert!(retry.iter().all(|row| row.error.is_none()), "{retry:?}"); assert_eq!(count_rows(&reader, "node:Person").await, expected_rows - 1); continue; @@ -9752,7 +9782,7 @@ async fn collector_refuses_mixed_branch_captures_before_deleting() { count_rows(&db, "node:Person").await + usize::from(action == "collector_mixed"); let cleaner = helpers::session(Omnigraph::open(db.uri()).await.unwrap()); let (guard, hold) = catalog::CLEANUP_COLLECTOR_MID_BRANCH_READ.hold(); - let cleanup = tokio::spawn(async move { cleaner.cleanup(keep_one()).await }); + let cleanup = tokio::spawn(async move { cleaner.cleanup_as(keep_one(), None).await }); let reached = hold.clone(); tokio::task::spawn_blocking(move || reached.wait_until_reached()) .await @@ -9790,7 +9820,7 @@ async fn collector_refuses_mixed_branch_captures_before_deleting() { let pin = pinned_version(&reader, "main", "node:Person").await; assert!(detached_versions(&person_uri).await.contains(&pin)); assert_eq!(count_rows(&reader, "node:Person").await, expected_rows); - let retry = reader.cleanup(keep_one()).await.unwrap(); + let retry = reader.cleanup_as(keep_one(), None).await.unwrap(); assert!(retry.iter().all(|row| row.error.is_none()), "{retry:?}"); assert!(detached_versions(&person_uri).await.contains(&pin)); } @@ -9808,7 +9838,7 @@ async fn collector_pass_interrupted_mid_sweep_counts_nothing_and_retries_whole() let stats = { let _fp = catalog::CLEANUP_SWEEP_BETWEEN_MANIFESTS.fail_once_at(1); - db.cleanup(keep_one()).await.unwrap() + db.cleanup_as(keep_one(), None).await.unwrap() }; let row = stats .iter() @@ -9824,7 +9854,7 @@ async fn collector_pass_interrupted_mid_sweep_counts_nothing_and_retries_whole() row.manifests_removed, 1, "the pass stopped after its first manifest: {row:?}" ); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let row = stats .iter() .find(|row| row.type_key == "node:Person") @@ -9891,7 +9921,7 @@ async fn assert_collector_retry_treats_an_index_directory_an_interrupted_pass_re let stats = { let _fp = catalog::CLEANUP_SWEEP_PRE_FILE_DELETE.fire_always(); - db.cleanup(keep_one()).await.unwrap() + db.cleanup_as(keep_one(), None).await.unwrap() }; let row = stats .iter() @@ -9924,7 +9954,7 @@ async fn assert_collector_retry_treats_an_index_directory_an_interrupted_pass_re let stats = { let _fp = catalog::CLEANUP_SWEEP_PRE_MANIFEST_DELETE.fire_always(); - db.cleanup(keep_one()).await.unwrap() + db.cleanup_as(keep_one(), None).await.unwrap() }; let row = stats .iter() @@ -9945,7 +9975,7 @@ async fn assert_collector_retry_treats_an_index_directory_an_interrupted_pass_re ); } - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); let row = stats .iter() @@ -10031,7 +10061,7 @@ async fn assert_collector_retry_after_a_pass_stopped_between_manifests_finishes_ let stats = { let _fp = catalog::CLEANUP_SWEEP_BETWEEN_MANIFESTS.fire_always(); - db.cleanup(keep_one()).await.unwrap() + db.cleanup_as(keep_one(), None).await.unwrap() }; let row = stats .iter() @@ -10079,7 +10109,7 @@ async fn assert_collector_retry_after_a_pass_stopped_between_manifests_finishes_ plan.sweep_paths ); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); let row = stats .iter() @@ -10134,7 +10164,7 @@ async fn assert_collector_retry_after_a_pass_stopped_before_the_dead_manifests_r let before = detached_versions(&person_uri).await; { let _fp = catalog::ENSURE_INDICES_POST_PHASE_B_PRE_MANIFEST_COMMIT.fire_always(); - let outcome = db.ensure_indices().await; + let outcome = db.ensure_indices_as(None).await; assert!( outcome.is_err(), "the seam fails the index build before publish" @@ -10172,7 +10202,7 @@ async fn assert_collector_retry_after_a_pass_stopped_before_the_dead_manifests_r let stats = { let _fp = catalog::CLEANUP_SWEEP_PRE_DEAD_MANIFEST_DELETE.fire_always(); - db.cleanup(keep_one()).await.unwrap() + db.cleanup_as(keep_one(), None).await.unwrap() }; let row = stats .iter() @@ -10213,7 +10243,7 @@ async fn assert_collector_retry_after_a_pass_stopped_before_the_dead_manifests_r plan.dead_paths ); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); let row = stats .iter() @@ -10358,7 +10388,7 @@ async fn branch_merge_in_doubt_keeps_input_tags_until_cleanup() { "the acknowledged source and target guards must survive an in-doubt fast-forward" ); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( !native @@ -10892,7 +10922,7 @@ async fn unattached_live_chain_pins_are_rooted_and_survive_cleanup() { report.retained_unattached_tags ); - let rows = db.cleanup(keep_one_at_any_age()).await.unwrap(); + let rows = db.cleanup_as(keep_one_at_any_age(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let remaining = detached_versions(&person_uri).await; assert!( @@ -10937,7 +10967,7 @@ async fn unattached_live_chain_pins_are_rooted_and_survive_cleanup() { let plan = main_plan(&report, "node:Person"); assert!(plan.roots.contains(&grandchild_pin)); assert!(plan.roots.contains(&root_pin)); - let rows = db.cleanup(keep_one_at_any_age()).await.unwrap(); + let rows = db.cleanup_as(keep_one_at_any_age(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let remaining = detached_versions(&person_uri).await; assert!(remaining.contains(&grandchild_pin)); @@ -11013,7 +11043,9 @@ async fn cleanup_releases_expired_unattached_tags_descendants_first() { let interrupted = { let _fp = catalog::CLEANUP_POST_UNATTACHED_RELEASE.fail_once_at(1); - db.cleanup(keep_one_at_any_age()).await.unwrap_err() + db.cleanup_as(keep_one_at_any_age(), None) + .await + .unwrap_err() }; assert!( interrupted @@ -11027,7 +11059,7 @@ async fn cleanup_releases_expired_unattached_tags_descendants_first() { "the child's tag went first; the root's stays for the retry" ); - let rows = db.cleanup(keep_one_at_any_age()).await.unwrap(); + let rows = db.cleanup_as(keep_one_at_any_age(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!(unattached_tags(&uri).await.is_empty()); let at = |id: &str| ReadTarget::snapshot(SnapshotId::new(id.to_string())); @@ -11045,7 +11077,7 @@ async fn cleanup_releases_expired_unattached_tags_descendants_first() { person_count_at(&db, ReadTarget::branch("main")).await, main_before + 1 ); - let rows = db.cleanup(keep_one_at_any_age()).await.unwrap(); + let rows = db.cleanup_as(keep_one_at_any_age(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); } @@ -11109,7 +11141,7 @@ async fn abandoned_merge_input_tags_expire_after_target_advances() { ) .await .unwrap(); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( captured_path.is_file(), @@ -11122,7 +11154,7 @@ async fn abandoned_merge_input_tags_expire_after_target_advances() { ) .await .unwrap(); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( !captured_path.exists(), @@ -11174,7 +11206,7 @@ async fn cleanup_keeps_a_late_retired_merge_base_provider() { db.branch_create("target").await.unwrap(); let cleaner = helpers::session(Omnigraph::open(&uri).await.unwrap()); let parked = helpers::failpoint::Rendezvous::park_first(&catalog::CLEANUP_TABLE_GC); - let cleanup = tokio::spawn(async move { cleaner.cleanup(keep_one()).await }); + let cleanup = tokio::spawn(async move { cleaner.cleanup_as(keep_one(), None).await }); parked.wait_until_reached().await; let child_uri = uri.clone(); let child = tokio::task::spawn_blocking(move || { @@ -11238,7 +11270,7 @@ async fn planning_source_failures_keep_their_error_class() { ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let lookup = params(&[("$name", "Planned")]); for seam in [ &catalog::QUERY_INDEX_FACTS_PRE_LOAD, diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index 0b73be466..fff04ba6b 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -190,7 +190,6 @@ const SENTINEL: &str = "// forbidden-api-allow:"; enum WriteProtocol { Exact(&'static str), Composed(&'static str), - ManifestAdoption, NativeRefControl, PhysicalOnly, EphemeralScratch, @@ -205,7 +204,6 @@ impl WriteProtocol { match self { Self::Exact(name) => format!("exact adapter ({name})"), Self::Composed(name) => format!("composed protocol ({name})"), - Self::ManifestAdoption => "manifest adoption".into(), Self::NativeRefControl => "native ref control".into(), Self::PhysicalOnly => "physical-only maintenance".into(), Self::EphemeralScratch => "ephemeral scratch".into(), @@ -264,13 +262,11 @@ write_surfaces! { "db/omnigraph.rs" => WriteProtocol::Exact("unpublished commit publication: the proposal's recorded changes replayed through the manifest publisher under its captured authority, required to allocate the proposal's own id") => ["publish_commit_as"], "db/omnigraph.rs" => WriteProtocol::Composed("proposal tag removal confirmed by a readback, then merge-input tag release") => ["drop_commit_as"], "db/omnigraph.rs" => INDICES_V9 => [ - "ensure_indices", "ensure_indices_on", - "rebuild_full_text_indices_on", "rebuild_full_text_indices_on_as", + "ensure_indices_as", "ensure_indices_on_as", "rebuild_full_text_indices_on_as", ], "db/omnigraph.rs" => WriteProtocol::TestOnly => ["failpoint_publish_table_head_without_index_rebuild_for_test", "init_with_legacy_system_columns_for_tests"], - "db/omnigraph.rs" => OPTIMIZE_V9 => ["optimize"], - "db/omnigraph.rs" => WriteProtocol::ManifestAdoption => ["repair"], - "db/omnigraph.rs" => WriteProtocol::PhysicalOnly => ["cleanup"], + "db/omnigraph.rs" => OPTIMIZE_V9 => ["optimize_as"], + "db/omnigraph.rs" => WriteProtocol::PhysicalOnly => ["cleanup_as"], "db/omnigraph.rs" => WriteProtocol::NativeRefControl => ["branch_create", "branch_create_as", "branch_create_from", "branch_create_from_as", "branch_delete", "branch_delete_as"], } @@ -296,6 +292,9 @@ const READ_ONLY_SURFACES: &[(&str, &str)] = &[ ("db/omnigraph.rs", "open_read_only"), ("db/omnigraph.rs", "refresh"), ("db/omnigraph.rs", "cleanup_plan"), + // The repair report: it publishes nothing, but `maintain` gates it like + // the maintenance writers (`maintenance_entry_points_take_an_actor`). + ("db/omnigraph.rs", "repair_report_as"), ("db/omnigraph.rs", "cleanup_plan_missing_paths"), ("db/omnigraph.rs", "cleanup_plan_path_snapshot"), ("db/omnigraph.rs", "open_read_only_with_storage"), @@ -1091,7 +1090,6 @@ durable_calls! { ("db/omnigraph/table_ops.rs", ".commit_changes_with_intent_and_expected(", 1, WriteProtocol::Exact("shared publisher")), ("db/omnigraph/schema_apply.rs", ".commit_changes_with_intent_and_expected(", 1, SCHEMA_V9), ("db/omnigraph/schema_apply/settlement.rs", ".commit_changes_with_intent_and_expected(", 1, WriteProtocol::Exact("schema settlement: strict numeric neutral lineage publication")), - ("db/omnigraph/repair.rs", ".commit_updates_with_actor_with_expected(", 1, WriteProtocol::ManifestAdoption), ("db/graph_coordinator.rs", ".commit_changes_with_intent_and_expected(", 1, WriteProtocol::Exact("publisher gateway")), ("db/graph_coordinator.rs", ".commit_changes_with_lineage_and_precondition(", 1, WriteProtocol::Exact("lowest manifest publisher gateway")), ("omnigraph-catalog/lib.rs", ".publish_with_precondition(", 1, WriteProtocol::Exact("lowest manifest publisher gateway")), @@ -1141,7 +1139,8 @@ durable_calls! { ("db/omnigraph/collector.rs", ".dataset()", 17, WriteProtocol::ReadOnlyAccess), ("db/omnigraph/collector.rs", ".delete(", 1, WriteProtocol::Composed("detached-only sweep through the table's own Lance store: freed files, then the published manifests no retained `__manifest` version pins (links before tips) and the dead stagings; an interrupted pass is re-swept by the next")), ("db/omnigraph/schema_apply.rs", ".dataset()", 1, SCHEMA_V9), - ("db/omnigraph/repair.rs", ".dataset()", 1, WriteProtocol::ManifestAdoption), + // The repair report reads raw transaction history; it publishes nothing. + ("db/omnigraph/repair.rs", ".dataset()", 1, WriteProtocol::ReadOnlyAccess), // The sixth accessor reports deferred FTS coverage from an immutable // snapshot; it only reads index metadata and never stages or publishes. ("db/omnigraph/optimize.rs", ".dataset()", 10, WriteProtocol::Composed("Optimize planning + read-only coverage and native-fork inventory + physical cleanup, including the release of expired proposal tags and of expired unattached tags descendants first, each confirmed by a readback")), @@ -2136,6 +2135,107 @@ fn graph_write_surfaces_are_registered() { ); } +/// The public surfaces of maintenance: the graph writers registered under +/// these protocols, plus the read-only repair report the same gate covers. +fn is_maintenance_protocol(protocol: WriteProtocol) -> bool { + protocol == INDICES_V9 || protocol == OPTIMIZE_V9 || protocol == WriteProtocol::PhysicalOnly +} + +const READ_ONLY_MAINTENANCE_SURFACES: &[(&str, &str)] = &[("db/omnigraph.rs", "repair_report_as")]; + +/// Maintenance has no actor-less public form (invariant 10). Every public +/// optimize, cleanup, index or repair entry point is an `_as` method taking the +/// `actor: Option<&str>` its policy check judges, so neither an embedded nor a +/// direct caller reaches maintenance around the gate; with a policy installed, +/// a `None` actor is refused. `policy_engine_chassis.rs` owns which action and +/// scope each one checks. +#[test] +fn maintenance_entry_points_take_an_actor() { + let src = engine_src_root(); + let read_only = READ_ONLY_SURFACES.iter().copied().collect::>(); + for surface in READ_ONLY_MAINTENANCE_SURFACES { + assert!( + read_only.contains(surface), + "{surface:?} must stay a registered read-only surface" + ); + } + let surfaces = WRITE_SURFACES + .iter() + .filter(|surface| is_maintenance_protocol(surface.protocol)) + .map(|surface| (surface.file, surface.function)) + .chain(READ_ONLY_MAINTENANCE_SURFACES.iter().copied()) + .collect::>(); + assert_eq!( + surfaces, + BTreeSet::from([ + ("db/omnigraph.rs", "cleanup_as"), + ("db/omnigraph.rs", "ensure_indices_as"), + ("db/omnigraph.rs", "ensure_indices_on_as"), + ("db/omnigraph.rs", "optimize_as"), + ("db/omnigraph.rs", "rebuild_full_text_indices_on_as"), + ("db/omnigraph.rs", "repair_report_as"), + ]), + "a maintenance surface was added or removed; give it an actor and a policy case" + ); + for (file, function) in surfaces { + assert!( + function.ends_with("_as"), + "{file}::{function}: maintenance entry points are `_as` forms" + ); + let path = guarded_path(&src, file); + let contents = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + let ast = parse_rust_source(&contents, file); + let signatures = function_signatures(&ast, function); + assert_eq!( + signatures.len(), + 1, + "{file}::{function} must resolve to exactly one function" + ); + let takes_actor = signatures[0].inputs.iter().any(|input| { + let syn::FnArg::Typed(argument) = input else { + return false; + }; + matches!(&*argument.pat, syn::Pat::Ident(name) if name.ident == "actor") + && type_final_ident(&argument.ty).is_some_and(|ident| ident == "Option") + }); + assert!( + takes_actor, + "{file}::{function} must take `actor: Option<&str>` for its policy check" + ); + } +} + +fn function_signatures(ast: &syn::File, function_name: &str) -> Vec { + struct Finder<'a> { + function_name: &'a str, + signatures: Vec, + } + + impl<'ast> Visit<'ast> for Finder<'_> { + fn visit_item_fn(&mut self, node: &'ast syn::ItemFn) { + if node.sig.ident == self.function_name { + self.signatures.push(node.sig.clone()); + } + visit::visit_item_fn(self, node); + } + + fn visit_impl_item_fn(&mut self, node: &'ast syn::ImplItemFn) { + if node.sig.ident == self.function_name { + self.signatures.push(node.sig.clone()); + } + visit::visit_impl_item_fn(self, node); + } + } + + let mut finder = Finder { + function_name, + signatures: Vec::new(), + }; + finder.visit_file(ast); + finder.signatures +} + #[test] fn low_level_coordinator_surfaces_are_registered() { let src = engine_src_root(); diff --git a/crates/omnigraph/tests/helpers/expand_projection.rs b/crates/omnigraph/tests/helpers/expand_projection.rs index 0fe1e877e..8f792dee2 100644 --- a/crates/omnigraph/tests/helpers/expand_projection.rs +++ b/crates/omnigraph/tests/helpers/expand_projection.rs @@ -63,7 +63,7 @@ pub async fn fixture(dir: &tempfile::TempDir, sources: usize, payload_bytes: usi .await .unwrap(); } - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); with_setting(&db, "engine", "v2") } diff --git a/crates/omnigraph/tests/helpers/mod.rs b/crates/omnigraph/tests/helpers/mod.rs index c39350972..af4c39238 100644 --- a/crates/omnigraph/tests/helpers/mod.rs +++ b/crates/omnigraph/tests/helpers/mod.rs @@ -347,7 +347,7 @@ pub async fn init_and_load_at(uri: &str, schema: &str) -> Session { db.load_jsonl(TEST_DATA, LoadMode::Overwrite).await.unwrap(); // Mutation/load publish only exact data effects; physical indexes are // reconciled separately as derived state. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } diff --git a/crates/omnigraph/tests/integration/branching.rs b/crates/omnigraph/tests/integration/branching.rs index 388854f56..bb2379621 100644 --- a/crates/omnigraph/tests/integration/branching.rs +++ b/crates/omnigraph/tests/integration/branching.rs @@ -150,7 +150,7 @@ async fn init_search_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(SEARCH_DATA, LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -2261,10 +2261,13 @@ fn assert_native_version_case( ); let maintenance = Omnigraph::open(uri).await.unwrap(); maintenance - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(100), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(100), + older_than: None, + }, + None, + ) .await .unwrap(); main.mutate( @@ -3623,7 +3626,10 @@ async fn ensure_indices_on_child_branch_keeps_inherited_table_when_no_work_is_ne "the child branch inherits feature's pin" ); - experiment.ensure_indices_on("experiment").await.unwrap(); + experiment + .ensure_indices_on_as("experiment", None) + .await + .unwrap(); let experiment_snap = snapshot_branch(&experiment, "experiment").await.unwrap(); assert_eq!( @@ -4210,10 +4216,13 @@ async fn branch_delete_defers_pin_collection_and_allows_recreate() { "recreated writes leave the former pin intact" ); assert!(detached.contains(&second_pin)); - main.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + main.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let detached = helpers::collector::detached_versions(&person_uri).await; @@ -4409,10 +4418,13 @@ async fn branch_delete_retires_legacy_physical_path_parents() { "{refused}" ); let rows = db - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); @@ -4595,10 +4607,13 @@ async fn branch_delete_retires_native_parent_and_cleanup_preserves_live_child() "{invalid}" ); assert!( - main.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + main.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None + ) .await .is_err(), "{invalid}" @@ -4632,10 +4647,13 @@ async fn branch_delete_retires_native_parent_and_cleanup_preserves_live_child() .await .unwrap(); reopened - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!(manifest.branches().get(&native).await.is_err()); @@ -4649,10 +4667,13 @@ async fn branch_delete_retires_native_parent_and_cleanup_preserves_live_child() ); reopened.branch_delete("experiment").await.unwrap(); reopened - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!(manifest.branches().get(&native).await.is_err()); @@ -4681,7 +4702,7 @@ async fn merged_table_preserves_row_version_for_unchanged_rows() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let main = init_and_load(&dir).await; - main.ensure_indices().await.unwrap(); + main.ensure_indices_as(None).await.unwrap(); main.branch_create("feature").await.unwrap(); let feature = helpers::session(Omnigraph::open(uri).await.unwrap()); @@ -4764,7 +4785,7 @@ async fn merged_table_preserves_row_version_for_unchanged_rows() { async fn edge_tables_have_id_btree_after_ensure_indices() { let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let snap = snapshot_main(&db).await.unwrap(); let ds = snap.open_dataset("edge:Knows").await.unwrap(); @@ -4786,7 +4807,7 @@ async fn merge_delta_only_bumps_changed_rows() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let main = init_and_load(&dir).await; - main.ensure_indices().await.unwrap(); + main.ensure_indices_as(None).await.unwrap(); main.branch_create("feature").await.unwrap(); let feature = helpers::session(Omnigraph::open(uri).await.unwrap()); diff --git a/crates/omnigraph/tests/integration/changes.rs b/crates/omnigraph/tests/integration/changes.rs index fc782044f..80e0815b6 100644 --- a/crates/omnigraph/tests/integration/changes.rs +++ b/crates/omnigraph/tests/integration/changes.rs @@ -17,10 +17,13 @@ use crate::helpers::*; /// name). async fn reclaim_person_history(db: &Session) { let stats = db - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let person = stats @@ -550,7 +553,7 @@ async fn diff_after_merge_reports_actual_changes() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let main = init_and_load(&dir).await; - main.ensure_indices().await.unwrap(); + main.ensure_indices_as(None).await.unwrap(); let v_before_branch = snapshot_id(&main, "main").await.unwrap(); main.branch_create("feature").await.unwrap(); @@ -2056,7 +2059,7 @@ node Document { // touching logical rows. The moved descriptors force the payload // tie-break, which must classify every row as unchanged: the maintenance // commit is an empty block, not a phantom full-table update. - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); assert!( stats .iter() @@ -2158,7 +2161,7 @@ async fn physical_only_commit_over_a_large_fragment_reads_as_empty_issue_705() { ) .await .unwrap(); - let stats = main.optimize().await.unwrap(); + let stats = main.optimize_as(None).await.unwrap(); assert!( stats .iter() @@ -3186,7 +3189,7 @@ async fn change_feed_merge_commit_is_first_parent_relative_with_merged_parent_on let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let main = init_and_load(&dir).await; - main.ensure_indices().await.unwrap(); + main.ensure_indices_as(None).await.unwrap(); let pre_merge_head = snapshot_id(&main, "main").await.unwrap(); main.branch_create("side").await.unwrap(); diff --git a/crates/omnigraph/tests/integration/changes_cost.rs b/crates/omnigraph/tests/integration/changes_cost.rs index c35e9a104..2f82e1716 100644 --- a/crates/omnigraph/tests/integration/changes_cost.rs +++ b/crates/omnigraph/tests/integration/changes_cost.rs @@ -26,14 +26,10 @@ use crate::helpers; -use std::sync::Arc; - use crate::helpers::cost::{IoCounts, assert_flat, assert_grows, cost_harness, measure}; -use lance::Dataset; -use lance::dataset::UpdateBuilder; use omnigraph::IndexCoverage; use omnigraph::changes::ChangeFeedScope; -use omnigraph::db::{Omnigraph, ReadTarget, RepairOptions}; +use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::loader::LoadMode; use omnigraph_compiler::ir::ParamMap; @@ -93,7 +89,7 @@ node Company { // Build full coverage, then append a new parent fragment. // The measured update therefore runs with a normal partial // / stale index state. The false arm never builds an index. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db.load_with_receipt( "main", r#"{"type":"Person","data":{"name":"stale-index-tail","age":1}}"#, @@ -303,123 +299,6 @@ node Document { .await; } -/// Multi-version graph intervals are intentionally outside the candidate -/// contract. The fixture advances one physical dataset twice, then adopts both -/// logical updates in one forced repair graph commit. Two stateless -/// `max_changes=1` pages must each fall back before reading either transaction; -/// the page count cannot multiply a transaction-history walk. -#[tokio::test] -#[ignore = "the multi-version interval is built by `repair`, which promotes at head; phase 4 of RFC \"Detached-only tables\" restates it over a detached chain"] -async fn changes_page_size_one_skips_transaction_history_for_multi_version_intervals() { - cost_harness(async { - let dir = tempfile::tempdir().unwrap(); - let db = Omnigraph::init( - dir.path().to_str().unwrap(), - "node Person {\n name: String @key\n age: I32?\n}", - ) - .await - .unwrap(); - let db = helpers::session(db); - db.load_with_receipt( - "main", - concat!( - r#"{"type":"Person","data":{"name":"alice","age":1}}"#, - "\n", - r#"{"type":"Person","data":{"name":"bob","age":1}}"#, - ), - LoadMode::Merge, - ) - .await - .unwrap(); - let snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); - let entry = snapshot.dataset("node:Person").unwrap(); - let before = entry.published_dataset_version; - let dataset_uri = format!( - "{}/{}", - db.uri().trim_end_matches('/'), - entry.dataset_path.trim_start_matches('/') - ); - - let dataset = Dataset::open(&dataset_uri).await.unwrap(); - let dataset = UpdateBuilder::new(Arc::new(dataset)) - .update_where("name = 'alice'") - .unwrap() - .set("age", "2") - .unwrap() - .build() - .unwrap() - .execute() - .await - .unwrap() - .new_dataset; - let dataset = UpdateBuilder::new(dataset) - .update_where("name = 'bob'") - .unwrap() - .set("age", "3") - .unwrap() - .build() - .unwrap() - .execute() - .await - .unwrap() - .new_dataset; - assert_eq!(dataset.version().version, before + 2); - - let repair = db - .repair(RepairOptions { - confirm: true, - force: true, - }) - .await - .unwrap(); - assert!(repair.graph_manifest_version.is_some()); - let repair_commit = db - .list_commits(None) - .await - .unwrap() - .first() - .expect("repair publishes a graph lineage commit") - .graph_commit_id - .clone(); - - let (first, first_io) = measure(db.commit_changes_page( - &repair_commit, - &ChangeFeedScope::default(), - None, - Some(1), - None, - )) - .await; - let first = first.unwrap(); - assert_eq!(first.block.changes.len(), 1); - let token = first.next_page_token.expect("two updates need two pages"); - assert_eq!( - first_io.candidate_transaction_reads, 0, - "first multi-version page must fall back before transaction-history I/O" - ); - - let (second, second_io) = measure(db.commit_changes_page( - &repair_commit, - &ChangeFeedScope::default(), - Some(&token), - Some(1), - None, - )) - .await; - let second = second.unwrap(); - assert_eq!(second.block.changes.len(), 1); - assert!(second.next_page_token.is_none()); - assert_ne!(first.block.changes[0].id, second.block.changes[0].id); - for io in [first_io, second_io] { - assert_eq!( - io.candidate_transaction_reads, 0, - "every stateless multi-version page must skip transaction-history I/O" - ); - } - }) - .await; -} - /// A delete's commit path proves it from the recorded ids, so its page data /// reads stay flat as the table grows. #[tokio::test] diff --git a/crates/omnigraph/tests/integration/composite_flow.rs b/crates/omnigraph/tests/integration/composite_flow.rs index a9dfac8fd..863ab0271 100644 --- a/crates/omnigraph/tests/integration/composite_flow.rs +++ b/crates/omnigraph/tests/integration/composite_flow.rs @@ -309,7 +309,7 @@ async fn composite_flow_canonical_lifecycle() { // version, and this flow exercises exactly that previously-failing // write below. // ───────────────────────────────────────────────────────────────── - let optimize_stats = db.optimize().await.unwrap(); + let optimize_stats = db.optimize_as(None).await.unwrap(); assert!( !optimize_stats.is_empty(), "optimize must return per-table stats" @@ -362,10 +362,13 @@ async fn composite_flow_canonical_lifecycle() { use omnigraph::db::CleanupPolicyOptions; use std::time::Duration; let _cleanup_stats = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(10), - older_than: Some(Duration::from_secs(3600)), - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(10), + older_than: Some(Duration::from_secs(3600)), + }, + None, + ) .await .unwrap(); diff --git a/crates/omnigraph/tests/integration/consistency.rs b/crates/omnigraph/tests/integration/consistency.rs index c7e3cd8fa..ab37588e4 100644 --- a/crates/omnigraph/tests/integration/consistency.rs +++ b/crates/omnigraph/tests/integration/consistency.rs @@ -1369,7 +1369,7 @@ node Thing { db.load_jsonl(&seed, LoadMode::Overwrite).await.unwrap(); // Build the key index explicitly before exercising the merge loads. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let mut delta = String::new(); for i in (1..=25).chain(51..=55) { @@ -1382,7 +1382,7 @@ node Thing { db.load_jsonl(&delta, LoadMode::Merge) .await .expect("first merge after ensure_indices must succeed"); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db.load_jsonl(&delta, LoadMode::Merge) .await .expect("second merge after ensure_indices must succeed"); diff --git a/crates/omnigraph/tests/integration/end_to_end.rs b/crates/omnigraph/tests/integration/end_to_end.rs index 02e8e5f45..8ed619ced 100644 --- a/crates/omnigraph/tests/integration/end_to_end.rs +++ b/crates/omnigraph/tests/integration/end_to_end.rs @@ -2989,9 +2989,9 @@ async fn ensure_indices_does_not_error_on_repeated_call() { // load commits now enforce required indices; repeated ensure_indices calls // should be a no-op at the manifest level. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let version_after_first = version_main(&db).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let version_after_second = version_main(&db).await.unwrap(); assert_eq!(version_after_first, version_after_load); @@ -3093,7 +3093,7 @@ async fn full_flow_optimize_then_query_update_and_reopen() { .unwrap(); } - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); assert!( stats.iter().any(|s| s.committed), "a multi-fragment table should have compacted in this flow" diff --git a/crates/omnigraph/tests/integration/engine_v2.rs b/crates/omnigraph/tests/integration/engine_v2.rs index 078c32321..cf54ed2f0 100644 --- a/crates/omnigraph/tests/integration/engine_v2.rs +++ b/crates/omnigraph/tests/integration/engine_v2.rs @@ -413,7 +413,7 @@ async fn indexed_single_hop_switches_to_the_csr_between_input_batches() { db.load_jsonl(&lines.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let v2 = with_setting(&db, "engine", "v2"); let pinned = first_column_sorted( @@ -1187,7 +1187,7 @@ edge Likes: Person -> Doc ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); for (source, name, params) in [ (ADULTS, "adults", int_params(&[("min", 18)])), (RANKED, "ranked", params(&[("q", "graph")])), diff --git a/crates/omnigraph/tests/integration/engine_v2_memory.rs b/crates/omnigraph/tests/integration/engine_v2_memory.rs index 02660da69..36b5a0120 100644 --- a/crates/omnigraph/tests/integration/engine_v2_memory.rs +++ b/crates/omnigraph/tests/integration/engine_v2_memory.rs @@ -520,7 +520,7 @@ node Doc { seed.push('\n'); } db.load_jsonl(&seed, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let v2 = with_setting(&db, "engine", "v2"); let source = r#"query fused($q: String) { match { $d: Doc } @@ -608,7 +608,7 @@ edge Knows: Doc -> Doc db.load_jsonl(&lines.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let v2 = with_setting( &with_setting(&db, "engine", "v2"), "rrf_plan", @@ -724,7 +724,7 @@ edge Knows: Person -> Person ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let v2 = with_setting(&db, "engine", "v2"); let queries = r#" query fused($q: String) { @@ -846,7 +846,7 @@ async fn hash_join_output_is_the_refusing_reservation() { db.load_jsonl(&lines.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let v2 = with_setting(&db, "engine", "v2"); let limit = 8 * MIB; @@ -929,7 +929,7 @@ async fn underestimated_hash_build_retries_id_lookup() { db.load_jsonl(&lines.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let v2 = with_setting(&db, "engine", "v2"); let limit = 16 * MIB; let explain = with_query_memory_limit( diff --git a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs index f866b91da..35e4fd92e 100644 --- a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs @@ -65,7 +65,7 @@ async fn incomplete_rank_fuse_row_tiebreak_refuses_replay_issue_659() { .collect::>() .join("\n"); db.load_jsonl(&edges, LoadMode::Append).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let query = r#"query selected_page() { match { $p: Person $p $e:(knows | likes) $q } return { $e.@type as edge_type, $e.@id as edge_id, $q.name as target } @@ -305,7 +305,7 @@ async fn selected_cold_indexed_route_uses_persisted_members_without_building_csr ) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); drop(db); let db = with_traversal( &session(Omnigraph::open(dir.path().to_str().unwrap()).await.unwrap()), @@ -2123,7 +2123,7 @@ async fn a_replay_rechecks_recorded_full_text_coverage() { ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db.load_jsonl( r#"{"type":"Doc","data":{"slug":"t1","text":"graph","year":2022}} {"type":"Doc","data":{"slug":"t2","text":"graph graph graph graph","year":1990}}"#, diff --git a/crates/omnigraph/tests/integration/literal_filters.rs b/crates/omnigraph/tests/integration/literal_filters.rs index e493037db..41588b1db 100644 --- a/crates/omnigraph/tests/integration/literal_filters.rs +++ b/crates/omnigraph/tests/integration/literal_filters.rs @@ -579,7 +579,7 @@ async fn standalone_string_predicate_is_hoisted_into_scan() { async fn string_predicates_compose_with_search_and_negation() { let dir = tempfile::tempdir().unwrap(); let mut db = metric_db(&dir).await; - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let q = r#" query prefix_and_search() { match { diff --git a/crates/omnigraph/tests/integration/maintenance.rs b/crates/omnigraph/tests/integration/maintenance.rs index 15afbc9e8..c41a001f0 100644 --- a/crates/omnigraph/tests/integration/maintenance.rs +++ b/crates/omnigraph/tests/integration/maintenance.rs @@ -17,7 +17,7 @@ use lance_core::datatypes::BlobHandling; use omnigraph::IndexCoverage; use omnigraph::db::{ CleanupPolicyOptions, MergeOutcome, Omnigraph, ReadTarget, RepairAction, RepairClassification, - RepairOptions, SnapshotId, + SnapshotId, }; use omnigraph::error::{BranchStartRefusal, OmniError}; use omnigraph::loader::LoadMode; @@ -130,7 +130,7 @@ async fn optimize_on_empty_graph_returns_stats_per_table_with_no_changes() { let uri = dir.path().to_str().unwrap(); let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); assert_eq!(stats.len(), 5); for s in &stats { @@ -154,7 +154,7 @@ async fn optimize_after_load_then_again_is_idempotent() { let db = init_and_load(&dir).await; // First pass may compact (load wrote real fragments). - let _first = db.optimize().await.unwrap(); + let _first = db.optimize_as(None).await.unwrap(); let commits_before = db.list_commits(None).await.unwrap(); let head_before = commits_before @@ -165,7 +165,7 @@ async fn optimize_after_load_then_again_is_idempotent() { // Second pass should be a no-op: already-compacted graph produces no // fragments_removed / fragments_added. - let second = db.optimize().await.unwrap(); + let second = db.optimize_as(None).await.unwrap(); for s in &second { assert_eq!( s.fragments_removed, 0, @@ -229,7 +229,7 @@ async fn optimize_preserves_manifest_registry_fragments() { !cold_files.is_empty(), "the fixture must release more registry rows than the hot bound" ); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); let after = Dataset::open(&manifest_uri).await.unwrap(); let after_files: BTreeSet<_> = after .fragments() @@ -323,7 +323,7 @@ async fn optimize_clears_stale_auto_cleanup_and_preserves_versions() { .unwrap() .len(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let ds = Dataset::open(&manifest_uri).await.unwrap(); // (a) the stale auto_cleanup config was cleared (non-destructive by construction). @@ -363,10 +363,9 @@ async fn optimize_preserves_versions_under_stale_auto_cleanup_config_on_data_tab add_person_fragments(&db).await; // multiple fragments → will_compact // Simulate an upgraded graph: set an aggressive stored auto_cleanup config on - // the Person table. This is an out-of-band Lance commit (an `UpdateConfig` that - // advances HEAD past the manifest), so realign the manifest with a forced repair - // first — otherwise optimize skips the table as uncovered drift and never - // reaches the scrub. (Forced because UpdateConfig is not verified maintenance.) + // the Person table. This is an out-of-band Lance commit (an `UpdateConfig` + // above the table's last linear version); optimize stages on the pin, never + // the linear HEAD, so it still compacts the table. let (_, _, person_full) = person_pin_and_head(&db, &root).await; { let mut ds = Dataset::open(&person_full).await.unwrap(); @@ -377,12 +376,6 @@ async fn optimize_preserves_versions_under_stale_auto_cleanup_config_on_data_tab .await .unwrap(); } - db.repair(RepairOptions { - confirm: true, - force: true, - }) - .await - .unwrap(); let versions_before = Dataset::open(&person_full) .await @@ -393,7 +386,7 @@ async fn optimize_preserves_versions_under_stale_auto_cleanup_config_on_data_tab .len(); let rows_before = count_rows(&db, "node:Person").await; - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let ds = Dataset::open(&person_full).await.unwrap(); assert!( @@ -440,7 +433,7 @@ node Doc { ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); // A second load with NEW keys appends a fragment the existing BTREEs do not // cover (the existence gate skips re-building an index that already exists). @@ -462,7 +455,7 @@ node Doc { ); } - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); // Postcondition: optimize_indices folded the scalar tail in, but preserves // the existing FTS artifact instead of merging uncertified postings. @@ -487,7 +480,7 @@ node Doc { assert!(deferred[0].reason.contains("rebuild-full-text-indexes")); let commits_before = db.list_commits(None).await.unwrap().len(); for _ in 0..2 { - let repeated = db.optimize().await.unwrap(); + let repeated = db.optimize_as(None).await.unwrap(); let doc = repeated .iter() .find(|stat| stat.type_key == "node:Doc") @@ -506,7 +499,10 @@ node Doc { assert_eq!(recovery_sidecar_count(&dir), 0); } assert_eq!(db.list_commits(None).await.unwrap().len(), commits_before); - let rebuilt = db.rebuild_full_text_indices_on("main").await.unwrap(); + let rebuilt = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap(); assert_eq!(rebuilt.rebuilt_indexes.len(), 1); assert_eq!(rebuilt.rebuilt_indexes[0].property, "slug"); let snap = snapshot_main(&db).await.unwrap(); @@ -644,9 +640,10 @@ node Tag {\n slug: String @key\n}\n"; .clone(); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .expect("optimize must not crash on a graph with a Blob table"); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .expect("optimize must not crash on a graph with a Blob table"); assert_eq!( ( probes.compaction_blob_batch_calls(), @@ -778,9 +775,10 @@ async fn optimize_compacts_large_blob_rows_in_bounded_batches() { let commits_before = db.list_commits(None).await.unwrap().len(); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .unwrap(); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .unwrap(); assert_eq!( probes.compaction_blob_batch_calls(), 1, @@ -933,9 +931,10 @@ async fn optimize_sizes_a_blob_batch_by_the_sum_of_a_rows_blob_columns() { } let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .unwrap(); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .unwrap(); assert_eq!(probes.compaction_blob_batch_calls(), 1); assert_eq!( probes.compaction_blob_batch_rows(), @@ -1022,7 +1021,7 @@ async fn optimize_compacts_blob_fragments_with_deleted_rows() { "test precondition: both fragments carry a deletion vector" ); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); let doc = stats .iter() .find(|stat| stat.type_key == "node:Doc") @@ -1076,14 +1075,15 @@ async fn optimize_sizes_each_compaction_task_from_its_own_fragments() { }; db.load_jsonl(&line, mode).await.unwrap(); if load == 1 { - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); } } let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .unwrap(); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .unwrap(); assert_eq!( probes.compaction_blob_batch_calls(), 2, @@ -1159,9 +1159,10 @@ async fn optimize_does_not_size_a_blob_batch_by_external_references() { ); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .unwrap(); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .unwrap(); assert_eq!(probes.compaction_blob_batch_calls(), 1); assert_eq!( probes.compaction_blob_batch_rows(), @@ -1278,8 +1279,8 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { .await .unwrap(); // Settle index work, then prove Lance's planner leaves both tables alone. - db.optimize().await.unwrap(); - for stat in db.optimize().await.unwrap() { + db.optimize_as(None).await.unwrap(); + for stat in db.optimize_as(None).await.unwrap() { assert!(!stat.committed, "test precondition: {stat:?}"); } assert_eq!( @@ -1325,9 +1326,10 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { let commits_before = db.list_commits(None).await.unwrap().len(); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .unwrap(); + let stats = + omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize_as(None)) + .await + .unwrap(); let stat = |key: &str| stats.iter().find(|stat| stat.type_key == key).unwrap(); let doc = stat("node:Doc"); assert!( @@ -1375,7 +1377,7 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { doc_blob_values(&db, &["content"]).await == expected, "the remaining properties survive the rewrite" ); - for stat in db.optimize().await.unwrap() { + for stat in db.optimize_as(None).await.unwrap() { assert!( !stat.committed, "a second optimize has nothing left: {stat:?}" @@ -1399,8 +1401,8 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { db.branch_merge("retained", "main").await.unwrap().outcome, MergeOutcome::Merged ); - db.optimize().await.unwrap(); - let stats = db.cleanup(keep_one()).await.unwrap(); + db.optimize_as(None).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); for needle in &needles { assert!( @@ -1409,7 +1411,7 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { ); } db.branch_delete("retained").await.unwrap(); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); for needle in &needles { let found = files_containing(dir.path(), needle); @@ -1463,7 +1465,7 @@ async fn optimize_publishes_compaction_to_manifest_so_schema_apply_succeeds() { .get_fragments() .len(); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); let person = stats .iter() .find(|s| s.type_key == "node:Person") @@ -1523,7 +1525,7 @@ async fn optimize_compacts_the_pin_under_foreign_head_drift() { let (pin_before, head_before, _) = forge_person_compaction_drift(&db, &root).await; let rows_before = count_rows(&db, "node:Person").await; - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); let person = stats .iter() .find(|s| s.type_key == "node:Person") @@ -1541,10 +1543,11 @@ async fn optimize_compacts_the_pin_under_foreign_head_drift() { assert_eq!(count_rows(&db, "node:Person").await, rows_before); } -/// Repair preview names linear commits above the last linear version as -/// `foreign_drift` and publishes nothing. +/// The repair report names linear commits above the last linear version as +/// `foreign_drift` and publishes nothing, and writers stage on the pin +/// regardless: a strict schema apply still succeeds. #[tokio::test] -async fn repair_preview_reports_foreign_drift_without_adopting_it() { +async fn repair_report_names_foreign_drift_without_adopting_it() { let dir = tempfile::tempdir().unwrap(); let root = dir .path() @@ -1555,14 +1558,13 @@ async fn repair_preview_reports_foreign_drift_without_adopting_it() { let db = init_and_load(&dir).await; let (pin_before, head_before, _) = forge_person_compaction_drift(&db, &root).await; - let stats = db - .repair(RepairOptions { - confirm: false, - force: false, - }) - .await - .unwrap(); - assert_eq!(stats.graph_manifest_version, None); + let commits_before = db.list_commits(None).await.unwrap(); + let stats = db.repair_report_as(None).await.unwrap(); + assert_eq!( + db.list_commits(None).await.unwrap(), + commits_before, + "the repair report publishes nothing" + ); let person = stats .datasets .iter() @@ -1580,44 +1582,6 @@ async fn repair_preview_reports_foreign_drift_without_adopting_it() { let (pin_after, head_after, _) = person_pin_and_head(&db, &root).await; assert_eq!(pin_after, pin_before); assert_eq!(head_after, head_before); -} - -/// A confirmed repair never adopts foreign drift, and writers stage on the -/// pin regardless: the strict schema apply still succeeds. -#[tokio::test] -async fn repair_confirm_never_adopts_foreign_drift() { - let dir = tempfile::tempdir().unwrap(); - let root = dir - .path() - .to_str() - .unwrap() - .trim_end_matches('/') - .to_string(); - let db = init_and_load(&dir).await; - let (pin_before, head_before, _) = forge_person_compaction_drift(&db, &root).await; - - let stats = db - .repair(RepairOptions { - confirm: true, - force: false, - }) - .await - .unwrap(); - assert_eq!( - stats.graph_manifest_version, None, - "confirmed repair publishes nothing for foreign drift" - ); - let person = stats - .datasets - .iter() - .find(|s| s.type_key == "node:Person") - .expect("Person repair stat present"); - assert_eq!(person.classification, RepairClassification::ForeignDrift); - assert_eq!(person.action, RepairAction::NoOp); - - let (pin_after, head_after, _) = person_pin_and_head(&db, &root).await; - assert_eq!(pin_after, pin_before); - assert_eq!(head_after, head_before); let desired = TEST_SCHEMA.replace( " age: I32?\n}", @@ -1630,10 +1594,10 @@ async fn repair_confirm_never_adopts_foreign_drift() { assert!(result.applied); } -/// Force does not change the judgement: a foreign raw delete is reported as -/// `foreign_drift`, never adopted, and reads still resolve the pin. +/// A foreign raw delete is reported as `foreign_drift`, never adopted, and +/// reads still resolve the pin. #[tokio::test] -async fn repair_force_never_adopts_a_foreign_delete() { +async fn repair_report_never_adopts_a_foreign_delete() { let dir = tempfile::tempdir().unwrap(); let root = dir .path() @@ -1644,14 +1608,7 @@ async fn repair_force_never_adopts_a_foreign_delete() { let db = init_and_load(&dir).await; let (pin_before, head_before, _) = forge_person_delete_drift(&db, &root).await; - let stats = db - .repair(RepairOptions { - confirm: true, - force: true, - }) - .await - .unwrap(); - assert_eq!(stats.graph_manifest_version, None); + let stats = db.repair_report_as(None).await.unwrap(); let person = stats .datasets .iter() @@ -1717,17 +1674,7 @@ async fn write_on_a_never_written_table_succeeds_over_foreign_drift_and_repair_r "the write neither moves nor adopts the foreign HEAD" ); - let stats = db - .repair(RepairOptions { - confirm: true, - force: false, - }) - .await - .unwrap(); - assert_eq!( - stats.graph_manifest_version, None, - "repair publishes nothing for foreign drift" - ); + let stats = db.repair_report_as(None).await.unwrap(); let company = stats .datasets .iter() @@ -1775,7 +1722,7 @@ async fn full_text_rebuild_replaces_all_columns_and_segments_in_one_publication( db.load_jsonl(r#"{"type":"Person","data":{"name":"Alice","age":30,"biography":"organism university running"}}"#, LoadMode::Merge, ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); helpers::forge_linear_head_from_pin(&db, "main", "node:Person", 0).await; let person_uri = node_table_uri(&db, "Person").await; @@ -1919,7 +1866,10 @@ async fn full_text_rebuild_replaces_all_columns_and_segments_in_one_publication( assert_eq!(recovery_sidecar_count(&dir), 0); // A force rebuild remains a real operation after coverage is complete. - let repeated = db.rebuild_full_text_indices_on("main").await.unwrap(); + let repeated = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap(); assert_eq!(repeated.rebuilt_indexes, expected); assert_ne!(repeated.graph_commit_id, result.graph_commit_id); assert_eq!( @@ -1936,7 +1886,7 @@ async fn full_text_rebuild_on_a_branch_stages_on_the_inherited_location_without_ let db = init_and_load(&dir).await; db.branch_create("feature").await.unwrap(); let inherited = db.snapshot_of(ReadTarget::branch("feature")).await.unwrap(); - db.ensure_indices_on("feature").await.unwrap(); + db.ensure_indices_on_as("feature", None).await.unwrap(); assert_same_dataset_entry( inherited.dataset("node:Person").unwrap(), db.snapshot_of(ReadTarget::branch("feature")) @@ -1950,7 +1900,10 @@ async fn full_text_rebuild_on_a_branch_stages_on_the_inherited_location_without_ let main_commits_before = db.list_commits(None).await.unwrap(); let feature_commits_before = db.list_commits(Some("feature")).await.unwrap(); - let result = db.rebuild_full_text_indices_on("feature").await.unwrap(); + let result = db + .rebuild_full_text_indices_on_as("feature", None) + .await + .unwrap(); assert_eq!(result.branch, "feature"); assert_eq!(result.rebuilt_indexes.len(), 2); let feature = db.snapshot_of(ReadTarget::branch("feature")).await.unwrap(); @@ -2036,7 +1989,10 @@ async fn full_text_rebuild_reports_empty_builds_but_not_no_work() { .await .unwrap(); let before = snapshot_main(&db).await.unwrap(); - let result = db.rebuild_full_text_indices_on("main").await.unwrap(); + let result = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap(); assert_eq!(result.branch, "main"); assert_eq!(result.graph_commit_id, None); assert!(result.rebuilt_indexes.is_empty()); @@ -2051,7 +2007,10 @@ async fn full_text_rebuild_reports_empty_builds_but_not_no_work() { .await .unwrap(); let before = snapshot_main(&db).await.unwrap(); - let result = db.rebuild_full_text_indices_on("main").await.unwrap(); + let result = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap(); assert_eq!(result.rebuilt_indexes.len(), 2); assert!(result.graph_commit_id.is_some()); let after = snapshot_main(&db).await.unwrap(); @@ -2138,7 +2097,10 @@ async fn full_text_rebuild_refuses_unsupported_physical_inventory_before_effects .version, ); } - let error = db.rebuild_full_text_indices_on("main").await.unwrap_err(); + let error = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap_err(); assert!( error.to_string().contains("unsupported_internal_id_fts"), "{error}" @@ -2181,7 +2143,7 @@ async fn cleanup_without_any_policy_option_errors() { let db = init_and_load(&dir).await; let err = db - .cleanup(CleanupPolicyOptions::default()) + .cleanup_as(CleanupPolicyOptions::default(), None) .await .expect_err("cleanup with no policy options must error"); @@ -2211,10 +2173,13 @@ async fn cleanup_keep_one_preserves_head_and_table_remains_readable() { ); let _stats = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -2247,7 +2212,7 @@ async fn cleanup_keep_exceeding_history_preserves_every_available_version() { ); let pins = pins_in_history(&db, "main", "node:Person").await; - db.cleanup(keep_ten).await.unwrap(); + db.cleanup_as(keep_ten, None).await.unwrap(); // Every version a commit pins stays; only a chain's links below its // pinned tip (the first load's effect beneath its full-text declaration) @@ -2269,10 +2234,13 @@ async fn cleanup_older_than_zero_preserves_head() { // Lance must still preserve the head manifest, so the table is openable // afterwards and a subsequent load still works. let _stats = db - .cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::from_secs(0)), - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::from_secs(0)), + }, + None, + ) .await .unwrap(); @@ -2311,10 +2279,13 @@ async fn cleanup_preserves_main_version_pinned_by_live_lazy_branch() { "precondition: lazy-branch pin must fall outside keep=1 retention" ); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -2328,10 +2299,13 @@ async fn cleanup_preserves_main_version_pinned_by_live_lazy_branch() { // timestamp and version predicates with AND, so injecting the branch pin // as `before_version` keeps the inherited version even when every old // manifest satisfies the timestamp cutoff. - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::from_secs(0)), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::from_secs(0)), + }, + None, + ) .await .unwrap(); assert_eq!( @@ -2369,10 +2343,13 @@ async fn cleanup_retains_the_commit_a_branch_starts_at() { db.branch_create_from(at(&loaded), "at-load").await.unwrap(); let people_at_load = count_rows_branch(&db, "at-load", "node:Person").await; - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert_eq!( @@ -2428,10 +2405,13 @@ async fn cleanup_uses_oldest_pin_across_multiple_live_lazy_branches() { .await .unwrap(); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -2471,10 +2451,13 @@ async fn cleanup_fails_closed_when_live_lazy_branch_pin_is_unopenable() { let person_before = detached_versions(&person_uri).await; let stats = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let row = |type_key: &str| { @@ -2517,7 +2500,7 @@ async fn cleanup_reports_foreign_head_drift_and_keeps_it() { .expect("every registration records its last linear version"); let foreign: Vec = (last_linear + 1..=head_version).collect(); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let person = stats .iter() .find(|row| row.type_key == "node:Person") @@ -2557,13 +2540,16 @@ async fn cleanup_then_optimize_preserves_rows_and_table_remains_writable() { "fixture should seed both Person and Company rows" ); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); // Head is preserved through both ops. assert_eq!(count_rows(&db, "node:Person").await, people_before); @@ -2643,10 +2629,13 @@ async fn cleanup_reconciles_orphaned_branch_forks() { ); } - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -2695,10 +2684,13 @@ async fn cleanup_reconciles_orphaned_branch_forks() { ); // Idempotent: a second cleanup with the orphan already gone is a no-op. - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!( @@ -2774,10 +2766,13 @@ async fn cleanup_reconciles_live_branch_orphan_fork_and_keeps_the_branch_write() let main_people = count_rows(&db, "node:Person").await; let main_companies = count_rows(&db, "node:Company").await; - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -2877,10 +2872,13 @@ async fn cleanup_age_window_preserves_a_recent_snapshot_across_a_pointer_switch( .await .unwrap(); assert_eq!(before.num_rows(), base_count + 1); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), + }, + None, + ) .await .unwrap(); assert!( @@ -2898,10 +2896,13 @@ async fn cleanup_age_window_preserves_a_recent_snapshot_across_a_pointer_switch( .await .unwrap(); assert_eq!(after.num_rows(), base_count + 1); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: Some(Duration::ZERO), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: Some(Duration::ZERO), + }, + None, + ) .await .unwrap(); assert!( @@ -2931,10 +2932,13 @@ async fn cleanup_age_window_preserves_recent_ref_absent_descendant() { .set_times(std::fs::FileTimes::new().set_modified(std::time::SystemTime::UNIX_EPOCH)) .unwrap(); std::fs::write(&recent_file, b"recent interrupted residue").unwrap(); - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), + }, + None, + ) .await .unwrap(); assert!( @@ -2945,10 +2949,13 @@ async fn cleanup_age_window_preserves_recent_ref_absent_descendant() { recent_file.exists(), "a missing ref is not permission to discard a recent object" ); - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::ZERO), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::ZERO), + }, + None, + ) .await .unwrap(); assert!( @@ -3018,10 +3025,13 @@ async fn cleanup_reclaims_retirement_archive_after_the_age_window() { > std::time::SystemTime::UNIX_EPOCH, "retirement writes a recent archive even when the tree is old" ); - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), + }, + None, + ) .await .unwrap(); assert!( @@ -3032,10 +3042,13 @@ async fn cleanup_reclaims_retirement_archive_after_the_age_window() { tree.exists(), "the recent retirement metadata must protect the aged physical tree" ); - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::ZERO), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::ZERO), + }, + None, + ) .await .unwrap(); assert!( @@ -3094,10 +3107,13 @@ async fn cleanup_age_window_preserves_recent_native_mutation_and_aged_parent() { let changed = child.delete("name = 'Alice'").await.unwrap(); assert_eq!(changed.num_deleted_rows, 1); let latest = changed.new_dataset.version().version; - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::from_secs(30 * 24 * 60 * 60)), + }, + None, + ) .await .unwrap(); let branches = Dataset::open(&person_uri) @@ -3129,10 +3145,13 @@ async fn cleanup_age_window_preserves_recent_native_mutation_and_aged_parent() { retained.count_rows(None).await.unwrap(), count_rows(&db, "node:Person").await - 1 ); - db.cleanup(CleanupPolicyOptions { - keep_versions: None, - older_than: Some(Duration::ZERO), - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: None, + older_than: Some(Duration::ZERO), + }, + None, + ) .await .unwrap(); let branches = Dataset::open(&person_uri) @@ -3228,10 +3247,13 @@ async fn cleanup_preserves_the_pin_a_lazy_child_reads_after_its_parent_retires() .clone(); db.branch_delete("feature").await.unwrap(); db.branch_create("feature").await.unwrap(); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -3290,10 +3312,13 @@ async fn cleanup_reclaims_dead_incarnation_fork_of_live_branch() { ds.create_branch(&dead_native, base, None).await.unwrap(); } - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); @@ -3359,10 +3384,13 @@ async fn cleanup_reclaims_dead_incarnation_fork_of_live_branch() { db.apply_schema(TEST_SCHEMA).await.unwrap(); let replacement_uri = node_table_uri(&db, "Company").await; assert_ne!(replacement_uri, company_uri); - db.cleanup(CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!( @@ -3421,7 +3449,7 @@ async fn index_build_tolerates_null_vector_rows() { // Must not abort: the untrainable vector column is deferred, while id, // slug (FTS), and n (BTREE) are built together in one table transaction. let pending = db - .ensure_indices() + .ensure_indices_as(None) .await .expect("ensure_indices must not abort when a vector column has no trainable vectors yet"); assert_eq!(pending.len(), 1, "only the null vector index is pending"); @@ -3488,7 +3516,7 @@ async fn optimize_materializes_index_declared_but_unbuilt() { ); } - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); // Postcondition: optimize's reconciler materialized the declared index. let snap = snapshot_main(&db).await.unwrap(); @@ -3544,7 +3572,7 @@ async fn optimize_materializes_index_after_type_rename() { ); } - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let snap = snapshot_main(&db).await.unwrap(); let ds = snap.open_dataset("node:Item").await.unwrap(); @@ -3576,7 +3604,7 @@ async fn collector_retains_malformed_witnesses_after_retirement_history_is_gone( let identity = manifest.branches().get(&native).await.unwrap().identifier; assert_ne!(identity, BranchIdentifier::main()); db.branch_delete("retired").await.unwrap(); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!(!dir.path().join("__manifest/tree").join(&native).exists()); let identity = serde_json::to_string(&identity).unwrap(); @@ -3653,7 +3681,7 @@ async fn collector_retains_malformed_witnesses_after_retirement_history_is_gone( ); } } - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let remaining = detached_versions(&table_uri).await; for (label, version) in versions { @@ -3707,7 +3735,7 @@ async fn collector_sweeps_the_pin_that_only_a_pruned_manifest_version_names() { ); assert!(plan.sweep_paths.is_disjoint(&plan.marked_paths)); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); let row = stats .iter() .find(|row| row.type_key == "node:Person") @@ -3921,7 +3949,7 @@ async fn collector_repeat_run_discovers_what_an_earlier_pass_left() { async fn collector_finds_nothing_to_sweep_after_todays_cleanup() { let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); insert_person(&db, "main", "Eve").await; mutate_main( &db, @@ -3931,7 +3959,7 @@ async fn collector_finds_nothing_to_sweep_after_todays_cleanup() { ) .await .unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); db.branch_create("feature").await.unwrap(); insert_person(&db, "feature", "Frank").await; db.branch_merge("feature", "main").await.unwrap(); @@ -3941,7 +3969,7 @@ async fn collector_finds_nothing_to_sweep_after_todays_cleanup() { ) .await .unwrap(); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); for row in &stats { assert!(row.error.is_none(), "every table's version GC ran: {row:?}"); } @@ -3990,10 +4018,13 @@ async fn cleanup_on_a_detached_only_graph_counts_manifest_versions_for_keep() { let third = staged_since(&person_uri, &after_second).await; let stats = db - .cleanup(CleanupPolicyOptions { - keep_versions: Some(2), - older_than: None, - }) + .cleanup_as( + CleanupPolicyOptions { + keep_versions: Some(2), + older_than: None, + }, + None, + ) .await .unwrap(); let person = |stats: &[omnigraph::db::DatasetCleanupStats]| { @@ -4018,7 +4049,7 @@ async fn cleanup_on_a_detached_only_graph_counts_manifest_versions_for_keep() { assert!(!remaining.contains(&first), "{remaining:?}"); assert!(remaining.contains(&second) && remaining.contains(&third)); - let row = person(&db.cleanup(keep_one()).await.unwrap()); + let row = person(&db.cleanup_as(keep_one(), None).await.unwrap()); assert_eq!( row.manifests_removed, 1, "keep 1 prunes the publication that named the second pin: {row:?}" @@ -4027,7 +4058,7 @@ async fn cleanup_on_a_detached_only_graph_counts_manifest_versions_for_keep() { assert!(!remaining.contains(&second), "{remaining:?}"); assert!(remaining.contains(&third)); - let row = person(&db.cleanup(keep_one()).await.unwrap()); + let row = person(&db.cleanup_as(keep_one(), None).await.unwrap()); assert_eq!(row.manifests_removed, 0, "a rerun finds nothing: {row:?}"); let report = db.cleanup_plan(keep_one()).await.unwrap(); for plan in &report.tables { @@ -4222,7 +4253,7 @@ async fn collector_reclaims_aged_orphans_without_deleting_recent_blob_sidecars() .unwrap(); } let rows_before = count_rows(&db, "node:Person").await; - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); for relative in recent { assert!( @@ -4290,7 +4321,7 @@ async fn collector_reclaims_blob_sidecars_after_the_last_retained_pin() { db.load_jsonl(&row(¤t_payload), LoadMode::Overwrite) .await .unwrap(); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); for path in &old_sidecars { assert!( @@ -4311,7 +4342,7 @@ async fn collector_reclaims_blob_sidecars_after_the_last_retained_pin() { let report = db.cleanup_plan(keep_one()).await.unwrap(); let plan = main_plan(&report, "node:Doc"); assert!(plan.errors.is_empty(), "{:?}", plan.errors); - let stats = db.cleanup(keep_one()).await.unwrap(); + let stats = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(stats.iter().all(|row| row.error.is_none()), "{stats:?}"); for path in &old_sidecars { assert!( diff --git a/crates/omnigraph/tests/integration/merge_cleanup.rs b/crates/omnigraph/tests/integration/merge_cleanup.rs index 471842586..bad2124f8 100644 --- a/crates/omnigraph/tests/integration/merge_cleanup.rs +++ b/crates/omnigraph/tests/integration/merge_cleanup.rs @@ -36,7 +36,7 @@ async fn repeated_merge_keeps_its_selected_base_through_cleanup() { for age in 1..=3 { set_age(&db, "source", "p", age).await; set_age(&db, "target", "q", age).await; - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); let reopened = helpers::session(Omnigraph::open(uri).await.unwrap()); assert_eq!( @@ -106,7 +106,7 @@ async fn imported_merge_base_survives_owner_retirement_and_recreation() { .starts_with("zz-unrelated.") }) .expect("unrelated retirement archive exists before cleanup"); - let rows = db.cleanup(keep_one()).await.unwrap(); + let rows = db.cleanup_as(keep_one(), None).await.unwrap(); assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); assert!( !unrelated.exists(), diff --git a/crates/omnigraph/tests/integration/point_in_time.rs b/crates/omnigraph/tests/integration/point_in_time.rs index 6f088bd5c..4cb87dc97 100644 --- a/crates/omnigraph/tests/integration/point_in_time.rs +++ b/crates/omnigraph/tests/integration/point_in_time.rs @@ -1184,7 +1184,10 @@ async fn historical_read_of_reclaimed_version_is_typed() { .await .unwrap(); - let stats = db.cleanup(helpers::collector::keep_one()).await.unwrap(); + let stats = db + .cleanup_as(helpers::collector::keep_one(), None) + .await + .unwrap(); let person = stats .iter() .find(|row| row.type_key == "node:Person") diff --git a/crates/omnigraph/tests/integration/recovery.rs b/crates/omnigraph/tests/integration/recovery.rs index 0a3608558..f0cd513b5 100644 --- a/crates/omnigraph/tests/integration/recovery.rs +++ b/crates/omnigraph/tests/integration/recovery.rs @@ -105,12 +105,12 @@ async fn recovery_ensure_indices_steady_state_no_sidecar() { {"type":"Company","data":{"name":"acme"}} "#; db.load_jsonl(test_data, LoadMode::Append).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); drop(db); let db = Omnigraph::open(uri).await.unwrap(); let before = snapshot_main(&db).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let after = snapshot_main(&db).await.unwrap(); assert_eq!( after.graph_manifest_version(), @@ -153,7 +153,7 @@ async fn recovery_ensure_indices_handles_empty_tables() { let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); // Don't load any data — every table is empty. let before = snapshot_main(&db).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let after = snapshot_main(&db).await.unwrap(); assert_eq!( after.graph_manifest_version(), @@ -173,7 +173,7 @@ async fn recovery_ensure_indices_handles_empty_tables() { drop(db); let db = Omnigraph::open(uri).await.unwrap(); let before = snapshot_main(&db).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let after = snapshot_main(&db).await.unwrap(); assert_eq!( after.graph_manifest_version(), diff --git a/crates/omnigraph/tests/integration/repro_issue_563.rs b/crates/omnigraph/tests/integration/repro_issue_563.rs index 409ce09ac..733ca35c8 100644 --- a/crates/omnigraph/tests/integration/repro_issue_563.rs +++ b/crates/omnigraph/tests/integration/repro_issue_563.rs @@ -141,7 +141,7 @@ async fn ranked_read_with_join_returns_top_limit_issue_563() { db.load_jsonl(&edges, LoadMode::Append).await.unwrap(); } - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); // The ranked read without the join stays under the i32 offset ceiling. let no_join = query_main( diff --git a/crates/omnigraph/tests/integration/rrf_prefilter_gate.rs b/crates/omnigraph/tests/integration/rrf_prefilter_gate.rs index db0cdf997..121cef7f6 100644 --- a/crates/omnigraph/tests/integration/rrf_prefilter_gate.rs +++ b/crates/omnigraph/tests/integration/rrf_prefilter_gate.rs @@ -265,7 +265,7 @@ async fn init_gate_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(&gate_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -457,7 +457,7 @@ async fn init_stats_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(&stats_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -614,7 +614,7 @@ async fn natural_gate_prefilters_at_ratio_boundary() { db.load_jsonl(&rows.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; let text_params = params(&[("$q1", "needle"), ("$q2", "sharp")]); @@ -767,7 +767,7 @@ async fn empty_eligible_set_falls_back() { ) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; let text_params = params(&[("$q1", "needle"), ("$q2", "sharp")]); @@ -807,7 +807,7 @@ async fn natural_gate_prefilters_selective_fixture() { db.load_jsonl(&rows.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; let text_params = params(&[("$q1", "needle"), ("$q2", "sharp")]); @@ -880,7 +880,7 @@ async fn decoy_flood_winner_holds_under_both_plans() { db.load_jsonl(&rows.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; let text_params = params(&[("$q1", "alpha"), ("$q2", "beta")]); @@ -926,7 +926,7 @@ async fn natural_gate_falls_back_on_broad_fixture() { db.load_jsonl(&rows.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; let text_params = params(&[("$q1", "needle"), ("$q2", "sharp")]); diff --git a/crates/omnigraph/tests/integration/scalar_indexes.rs b/crates/omnigraph/tests/integration/scalar_indexes.rs index a87b5be4a..29443468b 100644 --- a/crates/omnigraph/tests/integration/scalar_indexes.rs +++ b/crates/omnigraph/tests/integration/scalar_indexes.rs @@ -42,7 +42,7 @@ async fn node_scalar_and_enum_index_columns_get_btree() { let uri = dir.path().to_str().unwrap(); let db = session(Omnigraph::init(uri, SCHEMA).await.unwrap()); db.load_jsonl(DATA, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let snap = snapshot_main(&db).await.unwrap(); let ds = snap.open_dataset("node:Item").await.unwrap(); @@ -74,7 +74,7 @@ async fn node_scalar_and_enum_index_columns_get_btree() { // Idempotence: a second run finds every declared index present and // publishes nothing (dual indexes must not re-plan as missing forever). let version_before = ds.published_dataset_version(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let snap2 = snapshot_main(&db).await.unwrap(); let ds2 = snap2.open_dataset("node:Item").await.unwrap(); assert_eq!( diff --git a/crates/omnigraph/tests/integration/traversal_adaptive.rs b/crates/omnigraph/tests/integration/traversal_adaptive.rs index dc3f70ce3..e393372ff 100644 --- a/crates/omnigraph/tests/integration/traversal_adaptive.rs +++ b/crates/omnigraph/tests/integration/traversal_adaptive.rs @@ -83,7 +83,7 @@ async fn clique_db(dir: &tempfile::TempDir) -> Session { .unwrap(); // The endpoint BTREEs must exist, or dispatch prices a degraded scan and // takes CSR outright — the switch under test would never arm. - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); db } @@ -189,7 +189,7 @@ query far($name: String) { jsonl.push('\n'); let db = session(Omnigraph::init(uri, TEST_SCHEMA).await.unwrap()); db.load_jsonl(&jsonl, LoadMode::Overwrite).await.unwrap(); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let p = params(&[("$name", "p0")]); let csr_db = with_traversal(&db, Traversal::Csr); @@ -322,7 +322,7 @@ async fn corrupt_artifact_falls_back_to_the_scan_build() { async fn stale_artifact_is_rejected_after_an_edge_write() { let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let artifact = dir.path().join("__graph_index/csr-current.bin"); assert!(artifact.is_file()); diff --git a/crates/omnigraph/tests/integration/warm_read_cost.rs b/crates/omnigraph/tests/integration/warm_read_cost.rs index d9e05e316..172c2d8b4 100644 --- a/crates/omnigraph/tests/integration/warm_read_cost.rs +++ b/crates/omnigraph/tests/integration/warm_read_cost.rs @@ -956,10 +956,13 @@ async fn assert_cached_borrower_survives_branch_delete( "{branch} payload changed" ); } - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); assert!( @@ -995,10 +998,13 @@ async fn assert_cached_borrower_survives_branch_delete( .unwrap(); let recreated_pin = helpers::pinned_version(writer, "feature", "node:Person").await; assert_ne!(recreated_pin, source_pin); - db.cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + db.cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); let pins = helpers::collector::detached_versions(&table_uri).await; diff --git a/crates/omnigraph/tests/integration/write_cost.rs b/crates/omnigraph/tests/integration/write_cost.rs index d33fd8053..7182c64e1 100644 --- a/crates/omnigraph/tests/integration/write_cost.rs +++ b/crates/omnigraph/tests/integration/write_cost.rs @@ -38,7 +38,7 @@ use crate::helpers::{ // `_graph_commits`/`_graph_commit_actors` tables are retired), so the manifest scan // also covers the lineage and actor rows the authenticated write path appends. // Without internal-table compaction these scans are O(fragments) and grow forever; -// step 2 brings the internal tables into `db.optimize()`, so after compaction the +// step 2 brings the internal tables into `db.optimize_as(None)`, so after compaction the // per-write scan is flat. The test runs the **authenticated (actorful) write path** // — every commit carries an actor — and compacts at each depth checkpoint before // measuring, pinning the production invariant "a periodically-compacted graph's @@ -62,7 +62,7 @@ async fn internal_table_scans_are_flat_in_history() { } // Step 2: compaction folds all three internal tables' O(depth) fragments back // to a small constant, so the following write's scan of them is flat. - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); let io = measure_insert_as(&db, &format!("lock_{d}"), ACTOR).await; current += 1; // the measured write advanced depth by one eprintln!( @@ -87,7 +87,7 @@ async fn compacted_graph_at_depth(depth: u64) -> (u64, tempfile::TempDir, helper let dir = tempfile::tempdir().unwrap(); let db = local_graph(&dir).await; commit_many(&db, depth as usize).await; - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); (depth, dir, db) } @@ -106,7 +106,7 @@ async fn ensure_indices_manifest_reads_are_flat_in_history() { for (depth, _dir, db) in [graphs.0, graphs.1] { let indexed_schema = helpers::TEST_SCHEMA.replace("age: I32?", "age: I32? @index"); db.apply_schema(&indexed_schema).await.unwrap(); - let (result, io) = measure(db.ensure_indices()).await; + let (result, io) = measure(db.ensure_indices_as(None)).await; result.unwrap(); eprintln!( "ensure_indices depth~{depth}: __manifest={} data={}", @@ -147,7 +147,7 @@ async fn ensure_indices_writes_no_control_object() { let before_write_text = counts.write_text(); let before_delete = counts.delete(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); assert_eq!( counts.write_text() - before_write_text, 0, @@ -274,7 +274,7 @@ async fn optimize_writes_no_control_object() { let before_write_text = counts.write_text(); let before_delete = counts.delete(); - let stats = db.optimize().await.unwrap(); + let stats = db.optimize_as(None).await.unwrap(); assert!( stats .iter() @@ -310,7 +310,7 @@ async fn optimize_manifest_reads_are_flat_in_history() { commit_many(&db, 3).await; let commits_before = db.list_commits(None).await.unwrap().len(); - let (result, io) = measure(db.optimize()).await; + let (result, io) = measure(db.optimize_as(None)).await; result.unwrap(); assert_eq!( db.list_commits(None).await.unwrap().len(), @@ -357,7 +357,7 @@ async fn internal_table_scans_are_flat_without_compaction() { commit_many_as(&db, (d - current) as usize, ACTOR).await; current = d; } - // NO `db.optimize()` here — that omission is the whole point. The flat gate + // NO `db.optimize_as(None)` here — that omission is the whole point. The flat gate // above compacts before measuring and so never exercises this served regime. let io = measure_insert_as(&db, &format!("served_{d}"), ACTOR).await; current += 1; // the measured write advanced depth by one diff --git a/crates/omnigraph/tests/policy_engine_chassis.rs b/crates/omnigraph/tests/policy_engine_chassis.rs index b0b806cc4..fa75a8553 100644 --- a/crates/omnigraph/tests/policy_engine_chassis.rs +++ b/crates/omnigraph/tests/policy_engine_chassis.rs @@ -8,7 +8,10 @@ //! PR #2 wired `apply_schema_as`. PR #3 fans the same `enforce()` call //! out to the remaining six writers — `mutate_as`, `load_as`, //! `ingest_as`, `branch_create_as` / `branch_create_from_as`, -//! `branch_delete_as`, `branch_merge_as`. Each writer pair below +//! `branch_delete_as`, `branch_merge_as`. The maintenance entry points — +//! `optimize_as`, `ensure_indices_as` / `ensure_indices_on_as`, +//! `rebuild_full_text_indices_on_as`, `repair_report_as` (`maintain`) and +//! `cleanup_as` (`reclaim`) — are covered the same way. Each writer pair below //! covers allow + deny via the engine-side gate; the allow case proves //! the enforce call is correctly scoped (i.e. doesn't reject a legit //! actor), the deny case proves it actually denies an unauthorized @@ -111,6 +114,31 @@ async fn init_with_policy_and_feature_branch( /// One JSONL record for `load_as` / `ingest_as` exercises. const ONE_PERSON_JSONL: &str = r#"{"type": "Person", "data": {"name": "Eve"}}"#; +/// Every directory and file under `root`, with file bytes: what a refused +/// call must leave as it found it. +fn physical_tree(root: &Path) -> std::collections::BTreeMap>> { + fn visit( + root: &Path, + path: &Path, + entries: &mut std::collections::BTreeMap>>, + ) { + for entry in fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let relative = path.strip_prefix(root).unwrap().to_path_buf(); + if entry.file_type().unwrap().is_dir() { + entries.insert(relative, None); + visit(root, &path, entries); + } else { + entries.insert(relative, Some(fs::read(path).unwrap())); + } + } + } + let mut entries = std::collections::BTreeMap::new(); + visit(root, root, &mut entries); + entries +} + fn assert_denied(result: Result, what: &str) { match result { Err(OmniError::Policy(msg)) => { @@ -801,51 +829,44 @@ async fn non_canonical_branch_names_cannot_reach_a_protected_branch() { } #[tokio::test] -async fn full_text_rebuild_enforces_selected_branch_before_effects_and_records_actor() { - fn physical_tree( - root: &Path, - ) -> std::collections::BTreeMap>> { - fn visit( - root: &Path, - path: &Path, - entries: &mut std::collections::BTreeMap>>, - ) { - for entry in fs::read_dir(path).unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - let relative = path.strip_prefix(root).unwrap().to_path_buf(); - if entry.file_type().unwrap().is_dir() { - entries.insert(relative, None); - visit(root, &path, entries); - } else { - entries.insert(relative, Some(fs::read(path).unwrap())); - } - } - } - let mut entries = std::collections::BTreeMap::new(); - visit(root, root, &mut entries); - entries - } - +async fn full_text_rebuild_enforces_change_or_maintain_on_the_selected_branch_and_records_actor() { let dir = tempfile::tempdir().unwrap(); let db = init_and_load(&dir).await; db.branch_create("feature").await.unwrap(); drop(db); - // This actor can change unprotected feature, but not protected main. - // A hardcoded main scope or a different maintenance action must fail. - let policy = POLICY_YAML.replace(" branch_scope: any", " branch_scope: unprotected"); + // `act-allowed` can change unprotected feature, `act-maintainer` can + // maintain it, and neither reaches protected main. A hardcoded main scope + // or an action other than `change` or `maintain` must fail. + let policy = format!( + "{} - id: maintainers\n allow:\n actors: {{ group: maintainers }}\n actions: [maintain]\n branch_scope: unprotected\n", + POLICY_YAML + .replace(" branch_scope: any", " branch_scope: unprotected") + .replace( + " readers: [act-denied]\n", + " readers: [act-denied]\n maintainers: [act-maintainer]\n", + ) + ); let db = Omnigraph::open(dir.path().to_str().unwrap()).await.unwrap(); let (db, _engine) = install_policy_source(db, dir.path(), &policy); let main_before = db.list_commits(None).await.unwrap(); let feature_before = db.list_commits(Some("feature")).await.unwrap(); let files_before = physical_tree(dir.path()); - for (branch, actor) in [("feature", "act-denied"), ("main", "act-allowed")] { - assert_denied( - db.rebuild_full_text_indices_on_as(branch, Some(actor)) - .await, - "rebuild_full_text_indices_on_as", + for (branch, actor) in [ + ("feature", "act-denied"), + ("main", "act-allowed"), + ("main", "act-maintainer"), + ] { + let result = db + .rebuild_full_text_indices_on_as(branch, Some(actor)) + .await; + // Either grant would admit the rebuild, so the refusal names both. + assert!( + matches!(&result, Err(OmniError::Policy(message)) + if message.contains("'maintain'") && message.contains("'change'")), + "{actor} on {branch}: {result:?}" ); + assert_denied(result, "rebuild_full_text_indices_on_as"); assert_eq!( physical_tree(dir.path()), files_before, @@ -853,7 +874,7 @@ async fn full_text_rebuild_enforces_selected_branch_before_effects_and_records_a ); } let error = db - .rebuild_full_text_indices_on("feature") + .rebuild_full_text_indices_on_as("feature", None) .await .unwrap_err(); assert!( @@ -863,25 +884,36 @@ async fn full_text_rebuild_enforces_selected_branch_before_effects_and_records_a assert_eq!( physical_tree(dir.path()), files_before, - "the no-actor wrapper must not bypass policy" + "a missing actor must not bypass policy" ); - let result = db - .rebuild_full_text_indices_on_as("feature", Some("act-allowed")) - .await - .expect("Change on the selected unprotected branch must permit a real rebuild"); - assert_eq!(result.branch, "feature"); - assert_eq!(result.rebuilt_indexes.len(), 2); - let feature_after = db.list_commits(Some("feature")).await.unwrap(); - assert_eq!(feature_after.len(), feature_before.len() + 1); - assert_eq!( - result.graph_commit_id.as_deref(), - Some(feature_after[0].graph_commit_id.as_str()) - ); - assert_eq!(feature_after[0].actor_id.as_deref(), Some("act-allowed")); + let mut parent = feature_before[0].graph_commit_id.clone(); + for (actor, grant) in [("act-allowed", "change"), ("act-maintainer", "maintain")] { + let result = db + .rebuild_full_text_indices_on_as("feature", Some(actor)) + .await + .unwrap_or_else(|error| { + panic!( + "{grant} on the selected unprotected branch must permit a real rebuild: {error}" + ) + }); + assert_eq!(result.branch, "feature"); + assert_eq!(result.rebuilt_indexes.len(), 2); + let feature_after = db.list_commits(Some("feature")).await.unwrap(); + assert_eq!( + result.graph_commit_id.as_deref(), + Some(feature_after[0].graph_commit_id.as_str()) + ); + assert_eq!(feature_after[0].actor_id.as_deref(), Some(actor)); + assert_eq!( + feature_after[0].parent_commit_id.as_deref(), + Some(parent.as_str()) + ); + parent = feature_after[0].graph_commit_id.clone(); + } assert_eq!( - feature_after[0].parent_commit_id.as_deref(), - Some(feature_before[0].graph_commit_id.as_str()) + db.list_commits(Some("feature")).await.unwrap().len(), + feature_before.len() + 2 ); let main_after = db.list_commits(None).await.unwrap(); assert_eq!(main_after.len(), main_before.len()); @@ -891,6 +923,176 @@ async fn full_text_rebuild_enforces_selected_branch_before_effects_and_records_a ); } +/// Maintenance grants, apart from data writes: `act-maintainer` may maintain +/// any branch and `act-branch-maintainer` only unprotected ones; +/// `act-reclaimer` may reclaim; `act-writer` holds `change` everywhere and no +/// maintenance grant; `act-denied` holds nothing. +const MAINTENANCE_POLICY_YAML: &str = r#" +version: 1 +groups: + maintainers: [act-maintainer] + branch-maintainers: [act-branch-maintainer] + reclaimers: [act-reclaimer] + writers: [act-writer] + readers: [act-denied] +protected_branches: [main] +rules: + - id: maintainers + allow: + actors: { group: maintainers } + actions: [maintain] + branch_scope: any + - id: branch-maintainers + allow: + actors: { group: branch-maintainers } + actions: [maintain] + branch_scope: unprotected + - id: reclaimers + allow: + actors: { group: reclaimers } + actions: [reclaim] + - id: writers + allow: + actors: { group: writers } + actions: [change] + branch_scope: any +"#; + +/// A graph with a `feature` branch and two extra Person fragments on main, so +/// optimize has a compaction to publish, under [`MAINTENANCE_POLICY_YAML`]. +async fn init_with_maintenance_policy(dir: &tempfile::TempDir) -> Session { + let db = init_and_load(dir).await; + db.branch_create("feature").await.unwrap(); + for (name, age) in [("Eve", 40), ("Frank", 41)] { + let params = mixed_params(&[("$name", name)], &[("$age", age)]); + db.mutate("main", MUTATION_QUERIES, "insert_person", ¶ms) + .await + .unwrap(); + } + drop(db); + let db = Omnigraph::open(dir.path().to_str().unwrap()).await.unwrap(); + install_policy_source(db, dir.path(), MAINTENANCE_POLICY_YAML).0 +} + +fn assert_no_actor(result: Result, what: &str) { + match result { + Err(OmniError::Policy(message)) if message.contains("no actor") => {} + other => panic!("{what}: expected the no-actor refusal, got {other:?}"), + } +} + +/// Optimize, index builds and the repair report require `maintain` on the +/// branch they maintain, before any effect: `change` does not imply it, a +/// grant on unprotected branches does not reach main, and a missing actor is +/// refused. An allowed optimize records its actor on the commit it publishes. +#[tokio::test] +async fn maintenance_entry_points_enforce_maintain_on_their_branch_before_effects() { + let dir = tempfile::tempdir().unwrap(); + let db = init_with_maintenance_policy(&dir).await; + let main_before = db.list_commits(None).await.unwrap(); + let files_before = physical_tree(dir.path()); + + for actor in [ + "act-denied", + "act-writer", + "act-reclaimer", + "act-branch-maintainer", + ] { + let actor = Some(actor); + assert_denied(db.optimize_as(actor).await, "optimize_as"); + assert_denied(db.ensure_indices_as(actor).await, "ensure_indices_as"); + assert_denied( + db.ensure_indices_on_as("main", actor).await, + "ensure_indices_on_as(main)", + ); + assert_denied(db.repair_report_as(actor).await, "repair_report_as"); + } + for actor in ["act-denied", "act-writer", "act-reclaimer"] { + assert_denied( + db.ensure_indices_on_as("feature", Some(actor)).await, + "ensure_indices_on_as(feature)", + ); + } + assert_no_actor(db.optimize_as(None).await, "optimize_as"); + assert_no_actor(db.ensure_indices_as(None).await, "ensure_indices_as"); + assert_no_actor( + db.ensure_indices_on_as("feature", None).await, + "ensure_indices_on_as", + ); + assert_no_actor(db.repair_report_as(None).await, "repair_report_as"); + assert_eq!( + physical_tree(dir.path()), + files_before, + "a refused maintenance call must not write index artifacts, compactions or publications" + ); + assert_eq!(db.list_commits(None).await.unwrap(), main_before); + + db.ensure_indices_on_as("feature", Some("act-branch-maintainer")) + .await + .expect("maintain on an unprotected branch admits its index builds"); + let report = db + .repair_report_as(Some("act-maintainer")) + .await + .expect("maintain on main admits the repair report"); + assert!(!report.datasets.is_empty()); + db.ensure_indices_as(Some("act-maintainer")) + .await + .expect("maintain on main admits index builds"); + + let head_before_optimize = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let stats = db + .optimize_as(Some("act-maintainer")) + .await + .expect("maintain on main admits optimize"); + assert!( + stats + .iter() + .any(|stat| stat.type_key == "node:Person" && stat.committed), + "{stats:?}" + ); + let main_after = db.list_commits(None).await.unwrap(); + assert_eq!( + main_after[0].parent_commit_id.as_deref(), + Some(head_before_optimize.as_str()) + ); + assert_eq!(main_after[0].actor_id.as_deref(), Some("act-maintainer")); +} + +/// Cleanup requires `reclaim`, which no other grant implies, before any +/// effect. +#[tokio::test] +async fn cleanup_as_enforces_reclaim_before_effects() { + let dir = tempfile::tempdir().unwrap(); + let db = init_with_maintenance_policy(&dir).await; + let keep_one = || omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }; + let files_before = physical_tree(dir.path()); + for actor in ["act-denied", "act-writer", "act-maintainer"] { + assert_denied(db.cleanup_as(keep_one(), Some(actor)).await, "cleanup_as"); + } + assert_no_actor(db.cleanup_as(keep_one(), None).await, "cleanup_as"); + assert_eq!( + physical_tree(dir.path()), + files_before, + "a refused cleanup must not delete anything" + ); + + let rows = db + .cleanup_as(keep_one(), Some("act-reclaimer")) + .await + .expect("reclaim admits cleanup"); + assert!(rows.iter().all(|row| row.error.is_none()), "{rows:?}"); + assert_ne!( + physical_tree(dir.path()), + files_before, + "the admitted cleanup reclaims the history keep 1 does not retain" + ); +} + #[tokio::test] async fn schema_settlement_authorizes_current_executor_and_preserves_original_authorship() { use omnigraph::db::{SchemaApplySettlement, SchemaNonPublicationProof}; diff --git a/crates/omnigraph/tests/schema_apply.rs b/crates/omnigraph/tests/schema_apply.rs index 9ac4ee162..692b44ddd 100644 --- a/crates/omnigraph/tests/schema_apply.rs +++ b/crates/omnigraph/tests/schema_apply.rs @@ -308,7 +308,7 @@ query insert_project($name: String) { ); let detached_before_indices = helpers::collector::detached_versions(&project_uri).await; stale_handle - .ensure_indices_on("source") + .ensure_indices_on_as("source", None) .await .expect("index planning must use the same token-bound post-apply catalog"); let project_after_indices = project_published( @@ -372,7 +372,7 @@ async fn stale_handle_branch_delete_gates_tables_added_by_schema_apply() { let index_handle = Arc::clone(&index_reconciler); let mut index_task = - tokio::spawn(async move { index_handle.ensure_indices_on("target").await }); + tokio::spawn(async move { index_handle.ensure_indices_on_as("target", None).await }); let index_blocked = tokio::time::timeout(std::time::Duration::from_millis(250), &mut index_task) .await @@ -1821,7 +1821,7 @@ node Tag { .collect::>() .join("\n"); db.load_jsonl(&rows, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let before = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); let before_snapshot = db.resolve_snapshot("main").await.unwrap(); @@ -1937,7 +1937,7 @@ node Tag { } // Reconciliation builds from the desired declarations, so nothing indexes // a dropped column again. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); for (table, dropped_id) in dropped_ids { assert!( coverage(&db, table) @@ -2124,7 +2124,7 @@ query strong($min: I32) { "an update carries the untouched Blob of an evolved row" ); - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); assert_eq!( read_managed_blob_bytes( &db, @@ -2479,7 +2479,7 @@ async fn apply_schema_pure_type_rename_preserves_identity_path_and_version() { ); let db = helpers::session(Omnigraph::init(uri, &initial).await.unwrap()); db.load_jsonl(&data, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let before_snapshot_id = db.resolve_snapshot("main").await.unwrap(); let before_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); let before_version = before_snapshot.graph_manifest_version(); @@ -2675,10 +2675,13 @@ edge WorksAt: Human -> Company /// prunes the `__manifest` version naming it. async fn reclaim_dropped_history(db: &Session) { let stats = db - .cleanup(omnigraph::db::CleanupPolicyOptions { - keep_versions: Some(1), - older_than: None, - }) + .cleanup_as( + omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }, + None, + ) .await .unwrap(); for row in &stats { @@ -3110,7 +3113,7 @@ async fn apply_schema_defers_vector_index_on_empty_table() { db.load_jsonl(r#"{"type":"Doc","data":{"slug":"d1","body":"hello","embedding":[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]}}"#, LoadMode::Merge, ) .await .expect("loading a Doc with an embedding must succeed"); - db.ensure_indices() + db.ensure_indices_as(None) .await .expect("the deferred vector index must build once the table has a trainable vector"); } @@ -3261,7 +3264,7 @@ async fn full_text_analyzer_survives_schema_evolution_issue_904() { assert_eq!(search("summary").await, ["d3"]); // The reconciler builds the postings over the declarations. - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); assert_eq!(segments("text").await, [Some(2)]); assert_eq!(search("text").await, ["d1", "d2"]); assert_eq!(search("summary").await, ["d3"]); diff --git a/crates/omnigraph/tests/search.rs b/crates/omnigraph/tests/search.rs index fd79b55e4..2715c2b0c 100644 --- a/crates/omnigraph/tests/search.rs +++ b/crates/omnigraph/tests/search.rs @@ -207,7 +207,7 @@ async fn init_search_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(SEARCH_DATA, LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -226,7 +226,7 @@ async fn init_mock_embedding_search_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(&mock_embedding_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -236,7 +236,7 @@ async fn init_model_recorded_search_db(dir: &tempfile::TempDir) -> Session { db.load_jsonl(&mock_embedding_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -561,7 +561,7 @@ query delete_middle() { "#; let db = session(Omnigraph::init(uri, schema).await.unwrap()); db.load_jsonl(&seed, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); helpers::forge_linear_head_from_pin(&db, "main", "node:Doc", 0).await; { use lance::index::DatasetIndexExt; @@ -1225,7 +1225,7 @@ query nearest_all($q: Vector(4)) { let uri = dir.path().to_str().unwrap(); let db = session(Omnigraph::init(uri, schema).await.unwrap()); db.load_jsonl(&rows, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = with_setting(&db, "ann_nprobes", "100000"); use omnigraph::instrumentation::{QueryIoProbes, with_query_io_probes}; @@ -1306,7 +1306,7 @@ query nearest_all($q: Vector(4)) { let uri = dir.path().to_str().unwrap(); let db = session(Omnigraph::init(uri, schema).await.unwrap()); db.load_jsonl(&rows, LoadMode::Overwrite).await.unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let probes = QueryIoProbes::default(); let result = with_query_io_probes(probes.clone(), async { @@ -1413,7 +1413,7 @@ async fn issue_567_line_docs(uri: &str, rows: usize, edge_every: Option) db.load_jsonl(&lines.join("\n"), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -2069,7 +2069,7 @@ async fn injected_embedding_config_is_used_instead_of_env() { db.load_jsonl(&mock_embedding_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let result = query_main( &db, @@ -2160,7 +2160,7 @@ async fn bm25_ordered_aggregate_counts_all_matches_not_the_capped_scan() { db.load_jsonl(&underfill_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let db = db; use omnigraph::instrumentation::{QueryIoProbes, with_query_io_probes}; @@ -2209,7 +2209,7 @@ async fn rrf_arms_scan_uncapped_in_one_pass() { db.load_jsonl(&underfill_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); use omnigraph::instrumentation::{QueryIoProbes, with_query_io_probes}; // This test pins the POSTFILTER plan's invariants (uncapped one-pass @@ -2281,7 +2281,7 @@ query uncapped_all($q: String) { db.load_jsonl(&underfill_seed_data(), LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); // Appended AFTER the index build: their fragment is uncovered, and their // term frequency (40 > the seed's max 20) puts them at the top of any // correct ranking. @@ -2426,7 +2426,7 @@ async fn mutation_with_deferred_index_coverage_remains_searchable() { // Ordinary optimize must preserve certified postings, not silently replace // them with an uncertified incremental fold. Both old and tail rows remain // searchable after data compaction and unrelated index maintenance. - db.optimize().await.unwrap(); + db.optimize_as(None).await.unwrap(); for (term, slug) in [("Quasar", "quasar-notes"), ("Learning", "ml-intro")] { let result = query_main(&db, SEARCH_QUERIES, "text_search", ¶ms(&[("$q", term)])) .await @@ -2587,7 +2587,10 @@ async fn uncertified_full_text_refuses_all_search_routes_but_not_ordinary_reads( Err(OmniError::FullTextIndexRebuildRequired { .. }) )); - let rebuilt = db.rebuild_full_text_indices_on("main").await.unwrap(); + let rebuilt = db + .rebuild_full_text_indices_on_as("main", None) + .await + .unwrap(); assert!(!rebuilt.rebuilt_indexes.is_empty()); assert!( query_main( @@ -2690,7 +2693,7 @@ node Doc { 0, "load publishes exact data effects and leaves physical indexes pending" ); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); let ds = snapshot_main(&db) .await @@ -2788,7 +2791,7 @@ async fn full_text_analyzer_holds_after_first_rows_and_overwrites_issue_904() { .unwrap(); assert_eq!(hits("main", "docs").await, ["d1", "d2"], "first rows"); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db.load("main", &rows("Doc", "d"), LoadMode::Overwrite) .await .unwrap(); @@ -2880,7 +2883,7 @@ async fn a_full_text_index_with_no_segment_refuses_the_call_issue_747() { .unwrap_err(), ); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); assert_eq!( first_column_sorted( &query_branch(&db, "main", ANALYZER_QUERIES, "docs", &deep) diff --git a/crates/omnigraph/tests/system_column_upgrade.rs b/crates/omnigraph/tests/system_column_upgrade.rs index cfaf0e354..ca5a2dc84 100644 --- a/crates/omnigraph/tests/system_column_upgrade.rs +++ b/crates/omnigraph/tests/system_column_upgrade.rs @@ -71,7 +71,7 @@ async fn legacy_graph_with_data(dir: &tempfile::TempDir) -> Session { db.load_jsonl(LEGACY_DATA, LoadMode::Overwrite) .await .unwrap(); - db.ensure_indices().await.unwrap(); + db.ensure_indices_as(None).await.unwrap(); db } @@ -226,7 +226,7 @@ async fn assert_upgraded(db: &Session, dir: &tempfile::TempDir, expected_export: .await .expect_err("the respelled @unique(@src, @dst) still refuses a duplicate endpoint pair"); assert_eq!(count_rows(db, "edge:WorksAt").await, 3); - db.ensure_indices() + db.ensure_indices_as(None) .await .expect("index reconciliation after the rename is a no-op by field id"); } @@ -407,7 +407,7 @@ async fn system_column_upgrade_history_survives_unrelated_reclaimed_tables() { main.retained.contains(&company_moved) && main.would_prune.contains(&version_before), "the run keeps the Company write's `__manifest` version and drops the pre-write one: {main:?}" ); - let stats = db.cleanup(keep).await.unwrap(); + let stats = db.cleanup_as(keep, None).await.unwrap(); let row = stats.iter().find(|row| row.type_key == first).unwrap(); assert!(row.error.is_none(), "{row:?}"); assert!( diff --git a/docs/dev/control-plane.md b/docs/dev/control-plane.md index 72a1959cb..821427a0d 100644 --- a/docs/dev/control-plane.md +++ b/docs/dev/control-plane.md @@ -608,6 +608,7 @@ server. Public actor construction cannot grant signed-token permissions. on `AuthenticatedActor` exposes only verified claims. Policy embedders must handle `PolicyAction::ConfigManage`, +`PolicyAction::Maintain`, `PolicyAction::Reclaim`, `PolicyResourceKind::Cluster`, and `PolicyEngineKind::Cluster`. These public enums are exhaustive, so an external exhaustive match must add the relevant arm when upgrading. Preserving existing constructors and direct storage-holder diff --git a/docs/dev/writes.md b/docs/dev/writes.md index 120fff36a..367c85cff 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -582,6 +582,15 @@ callers must pass one when a policy checker is installed. Actor attribution travels with the pre-minted graph lineage and is published with the same manifest CAS. +Maintenance has no actor-less public form. `optimize_as`, `ensure_indices_as`, +`ensure_indices_on_as` and `rebuild_full_text_indices_on_as` check `maintain` +on the branch they maintain, `main` for optimize; the full-text rebuild also +admits `change`, its grant before `maintain` existed (`Omnigraph::enforce_any`, +`PolicyChecker::check_any`). `cleanup_as` checks `reclaim` with no branch, and +the read-only `repair_report_as` checks `maintain` on `main`. +`forbidden_apis.rs::maintenance_entry_points_take_an_actor` keeps every +registered maintenance surface an `_as` form with an `actor` parameter. + Bearer authentication (`401 unauthorized`), the HTTP contract check (`400 api_contract_mismatch`) and policy (`403 forbidden`) refuse a served write only before its first durable effect. The server runs diff --git a/docs/user/operations/maintenance.md b/docs/user/operations/maintenance.md index 7bf20669a..c89fe5c01 100644 --- a/docs/user/operations/maintenance.md +++ b/docs/user/operations/maintenance.md @@ -21,6 +21,14 @@ accepts local-file direct targets only; cloud-direct maintenance is not exposed by this CLI. Engine/server cloud support does not change that client boundary. See [operator configuration](../cli/targets.md#transport-and-capability-boundaries). +Each command acts as the operator configuration's `operator.actor`; +`rebuild-full-text-indexes` also accepts `--as` in its place. A commit the +command publishes records that actor. Direct CLI access, including `--target`, +does not load the server's Cedar policy; storage permissions are its trust +boundary. An embedded host that installs a policy checker enforces the +[policy actions](policy.md#actions) before any effect: `maintain` for optimize, +index builds, the full-text rebuild and repair, and `reclaim` for cleanup. + ## Optimize ```bash @@ -118,10 +126,9 @@ Success means the selected branch's planned rebuild was published, or an explici no-op with an empty index list, null commit, and empty `warnings` array. The default branch is `main`. -`--as` supplies actor attribution for this command. Direct CLI access, including -`--target`, does not load the server's Cedar policy; storage permissions are its -trust boundary. An embedded host that installs a policy checker also enforces -the Change permission before rebuilding. +`--as` supplies actor attribution for this command. An embedded host that +installs a policy checker admits the rebuild under `maintain` or `change` on the +selected branch; a later release admits it under `maintain` only. Other branches and historical snapshots are not rewritten. Rebuild every live branch that needs full-text search; restoring an older snapshot may require diff --git a/docs/user/operations/policy.md b/docs/user/operations/policy.md index 2187d6e8d..b3d122450 100644 --- a/docs/user/operations/policy.md +++ b/docs/user/operations/policy.md @@ -21,6 +21,8 @@ Graph-scoped actions: | `branch_merge` | Branch merge | | `commit_publish` | Publication of an unpublished commit onto its recorded target branch | | `commit_drop` | Drop of an unpublished commit | +| `maintain` | Content-preserving maintenance: optimize, index builds, the full-text rebuild and the repair report | +| `reclaim` | Cleanup, which permanently removes table versions and files no retained commit needs | | `invoke_query` | Entry to a stored query | | `admin` | Reserved graph administration | @@ -42,6 +44,17 @@ The reserved graph `admin` action does not grant cluster management. A stored mutation requires both `invoke_query` and `change`. A stored read requires `invoke_query` and `read`. +`maintain` and `reclaim` are separate from `change`, so an automation account +can compact a graph without the right to write data, and can hold `maintain` +without the right to delete history. `maintain` is judged on the branch the +operation maintains (`main` for optimize and the repair report) and takes +`branch_scope`; `reclaim` covers every branch's history and takes no branch +scope. The full-text rebuild accepts `change` or `maintain` on its branch; a +later release accepts `maintain` only, so grant `maintain` to the accounts that +run it. The engine checks these actions whenever a policy is installed on the +graph handle; the direct maintenance commands load no policy (see +[maintenance](maintenance.md)). + ## Bind a policy ```yaml @@ -124,7 +137,7 @@ empty, has leading or trailing whitespace, or contains a control character fails policy validation, and a request naming such a branch is refused as a bad request before any rule is evaluated (see [branch names](../branching/index.md#branch-names)). Server actions and graph-wide -`invoke_query` rules do not take branch scopes. `commit_publish` and +`invoke_query` and `reclaim` rules do not take branch scopes. `commit_publish` and `commit_drop` take `target_branch_scope`: the scope is the branch the unpublished commit recorded as its target. Writing an unpublished commit needs no new action: a mutation `as commit` is a `change` on its branch and a @@ -211,7 +224,8 @@ authentication allows every graph action and reports `actor_id: null`. The MCP server target ([what you may do at a target](../cli/targets.md#what-you-may-do-at-a-target)). It reports policy actions, not actor views, so a view can still refuse a -`change` the policy allows. +`change` the policy allows. It does not report `maintain` or `reclaim`, since +no server route runs maintenance. ## Actor identity diff --git a/skills/omnigraph/references/server-policy.md b/skills/omnigraph/references/server-policy.md index 5eccf49d8..9486333b6 100644 --- a/skills/omnigraph/references/server-policy.md +++ b/skills/omnigraph/references/server-policy.md @@ -120,6 +120,8 @@ Graph-scoped actions are: | `schema_apply` | Schema changes: `cluster apply --target` needs it on each existing graph whose schema changes | | `branch_create`, `branch_delete`, `branch_merge` | Corresponding branch operation | | `commit_publish`, `commit_drop` | Publish or drop an unpublished commit; drop an unattached commit | +| `maintain` | Optimize, index builds, the full-text rebuild (which also accepts `change` for now) and the repair report | +| `reclaim` | Cleanup, which permanently removes unretained history | | `invoke_query` | Entry to a stored query | | `admin` | Reserved; no current public operation | @@ -165,12 +167,12 @@ rules: actions: [invoke_query] ``` -`branch_scope` is valid only with `read`, `export`, and `change`; +`branch_scope` is valid only with `read`, `export`, `change`, and `maintain`; `target_branch_scope` only with `branch_create`, `branch_delete`, `branch_merge`, `schema_apply`, `commit_publish`, and `commit_drop`. Any other pairing, or both scopes on one rule, is refused. Values are `any`, `protected`, or `unprotected`, where protected branches are the bundle's top-level `protected_branches: [main, …]` -list. `invoke_query`, `graph_list` and `config_manage` take no branch scope. +list. `invoke_query`, `reclaim`, `graph_list` and `config_manage` take no branch scope. ```bash omnigraph policy validate --target . --graph knowledge From bc0c45f0e9db9764cc8a50c904a9021f6046ee3d Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 26 Oct 2026 13:31:35 +0800 Subject: [PATCH 2/3] test(policy): match the refused load instead of unwrapping it --- crates/omnigraph-policy/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/omnigraph-policy/src/lib.rs b/crates/omnigraph-policy/src/lib.rs index 2b2383b7f..c0ea6200a 100644 --- a/crates/omnigraph-policy/src/lib.rs +++ b/crates/omnigraph-policy/src/lib.rs @@ -2141,7 +2141,9 @@ rules: "actions: [reclaim]", &format!("actions: [reclaim]\n {scope}: any"), ); - let error = PolicyEngine::load_graph_from_source(&scoped, "graph").unwrap_err(); + let Err(error) = PolicyEngine::load_graph_from_source(&scoped, "graph") else { + panic!("{scope} on reclaim must be refused"); + }; assert!( error.to_string().contains("unsupported action 'reclaim'"), "{scope}: {error}" From 9b45173c81defaab9002ef56e514f895d2e1585e Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 26 Oct 2026 13:43:09 +0800 Subject: [PATCH 3/3] fix(policy): gate the coordinator import the test-only publisher uses --- crates/omnigraph/src/db/graph_coordinator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/omnigraph/src/db/graph_coordinator.rs b/crates/omnigraph/src/db/graph_coordinator.rs index ae540613d..e5a0b74ca 100644 --- a/crates/omnigraph/src/db/graph_coordinator.rs +++ b/crates/omnigraph/src/db/graph_coordinator.rs @@ -12,8 +12,10 @@ use super::commit_graph::{ CommitGraph, FirstParentPage, GraphCommit, HistoryCache, Lineage, graph_commit_from_manifest_row, }; +#[cfg(any(test, feature = "failpoints"))] +use super::manifest::DatasetUpdate; use super::manifest::{ - BranchRecords, CapturedManifestProbe, CommitBuffer, DatasetUpdate, ExpectedTableVersions, + BranchRecords, CapturedManifestProbe, CommitBuffer, ExpectedTableVersions, GenesisManifestAttempt, GraphHeadExpectation, HistoryRecord, HistoryReleaseBytes, LineageIntent, ManifestChange, ManifestCoordinator, ManifestIncarnation, ManifestInitError, PublishPrecondition, SchemaContractRow, UnattachedCommit, UnattachedIntent,