Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions aube.usage.kdl
Original file line number Diff line number Diff line change
Expand Up @@ -3788,6 +3788,7 @@ It removes global virtual-store graph entries not referenced by any registered p
On reflink filesystems such as APFS or btrfs, link counts cannot prove project reachability, so content-store pruning relies on cached package indexes. Global virtual-store reachability comes from project links.
"""#
flag --dry-run help="Do not actually delete anything; report what would be pruned"
flag --json help="Emit the dry-run plan as one machine-readable JSON document (requires --dry-run)"
}
cmd status help="Verify the store against cached package indexes" effect=read {
long_help #"""
Expand Down
14 changes: 14 additions & 0 deletions crates/aube-codes/src/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ pub const ERR_AUBE_NO_HOME: &str = "ERR_AUBE_NO_HOME";
pub const ERR_AUBE_GIT_ERROR: &str = "ERR_AUBE_GIT_ERROR";
pub const ERR_AUBE_STORE_INDEX_SCAN_FAILED: &str = "ERR_AUBE_STORE_INDEX_SCAN_FAILED";
pub const ERR_AUBE_GVS_PRUNE_FAILED: &str = "ERR_AUBE_GVS_PRUNE_FAILED";
pub const ERR_AUBE_STORE_PRUNE_LOCK_FAILED: &str = "ERR_AUBE_STORE_PRUNE_LOCK_FAILED";
pub const ERR_AUBE_STORE_PRUNE_FAILED: &str = "ERR_AUBE_STORE_PRUNE_FAILED";
Comment on lines +65 to +66

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Document new prune errors

The two new store-prune diagnostics are registered and can be emitted, but docs/error-codes.md was not updated as required, leaving the public error-code reference incomplete.

Context Used: CLAUDE.md (source)

Knowledge Base Used: CLI Commands: Parsing, Dispatch, and Auto-Install

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Claude Code


// ── linker ──────────────────────────────────────────────────────────
pub const ERR_AUBE_LINK_FAILED: &str = "ERR_AUBE_LINK_FAILED";
Expand Down Expand Up @@ -305,6 +307,18 @@ pub const ALL: &[CodeMeta] = &[
description: "The global virtual store project registry or prune walk failed.",
exit_code: None,
},
CodeMeta {
name: ERR_AUBE_STORE_PRUNE_LOCK_FAILED,
category: category::TARBALL_STORE,
description: "Aube couldn't acquire the store-wide maintenance lock for pruning.",
exit_code: None,
},
CodeMeta {
name: ERR_AUBE_STORE_PRUNE_FAILED,
category: category::TARBALL_STORE,
description: "A planned content-store file couldn't be removed during pruning.",
exit_code: None,
},
// Registry / network
CodeMeta {
name: ERR_AUBE_PACKAGE_NOT_FOUND,
Expand Down
8 changes: 8 additions & 0 deletions crates/aube-codes/src/warnings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ pub const WARN_AUBE_DELTA_INVALIDATE_FAILED: &str = "WARN_AUBE_DELTA_INVALIDATE_
pub const WARN_AUBE_GVS_INCOMPATIBLE: &str = "WARN_AUBE_GVS_INCOMPATIBLE";
pub const WARN_AUBE_GVS_MODE_CHANGED: &str = "WARN_AUBE_GVS_MODE_CHANGED";
pub const WARN_AUBE_GVS_CROSS_VOLUME: &str = "WARN_AUBE_GVS_CROSS_VOLUME";
pub const WARN_AUBE_STORE_PRUNE_ENTRY_DISAPPEARED: &str = "WARN_AUBE_STORE_PRUNE_ENTRY_DISAPPEARED";

// ── settings / config validation ────────────────────────────────────
pub const WARN_AUBE_INVALID_CONCURRENCY: &str = "WARN_AUBE_INVALID_CONCURRENCY";
Expand Down Expand Up @@ -125,6 +126,7 @@ pub const WARN_AUBE_RUNTIME_MISE_FALLBACK: &str = "WARN_AUBE_RUNTIME_MISE_FALLBA
pub mod category {
pub const PNPMFILE_HOOKS: &str = "pnpmfile / hooks";
pub const INSTALL_LIFECYCLE: &str = "Install lifecycle";
pub const STORE: &str = "Store maintenance";
pub const SETTINGS_CONFIG: &str = "Settings / config validation";
pub const UPDATE_PRERELEASE: &str = "Update / prerelease";
pub const AUDIT_NPMRC: &str = "Audit / npmrc";
Expand Down Expand Up @@ -266,6 +268,12 @@ pub const ALL: &[CodeMeta] = &[
description: "`cacheDir` (global virtual store) and `storeDir` are on different volumes, so linking falls back to per-file copy.",
exit_code: None,
},
CodeMeta {
name: WARN_AUBE_STORE_PRUNE_ENTRY_DISAPPEARED,
category: category::STORE,
description: "A global virtual-store entry disappeared while a prune plan was being built; the preview skipped it.",
exit_code: None,
},
// Settings / config validation
CodeMeta {
name: WARN_AUBE_INVALID_CONCURRENCY,
Expand Down
3 changes: 3 additions & 0 deletions crates/aube-store/src/cas.rs
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ impl Store {
/// the directory already exists, but callers should still hoist the
/// call out of tight loops.
pub fn ensure_shards_exist(&self) -> Result<(), Error> {
self.prepare_for_write()?;
std::fs::create_dir_all(&self.root).map_err(|e| Error::Io(self.root.clone(), e))?;
// Windows Defender and Search both touch every file in the
// store on default installs. Setting this attribute makes
Expand Down Expand Up @@ -572,6 +573,7 @@ impl Store {
/// exist yet, the `create_new` open will fail with `NotFound`; we
/// fall back to the slow path for correctness.
pub fn import_bytes(&self, content: &[u8], executable: bool) -> Result<StoredFile, Error> {
self.prepare_for_write()?;
let hash_t0 = std::time::Instant::now();
let hex_hash = blake3_hex(content);
if aube_util::diag::enabled() {
Expand Down Expand Up @@ -733,6 +735,7 @@ impl Store {
executable: bool,
gate: Option<&Gate>,
) -> Result<StoredFile, Error> {
self.prepare_for_write()?;
let Some(gate) = gate else {
return self.import_bytes(content, executable);
};
Expand Down
6 changes: 5 additions & 1 deletion crates/aube-store/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,9 @@ impl Store {
let index: PackageIndex = sonic_rs::from_slice(&buf).ok()?;
if !index_files_match_metadata(&index, verify_files) {
trace!("cache stale: {name}@{version}");
let _ = xx::file::remove_file(&index_path);
if self.prepare_for_write().is_ok() {
let _ = xx::file::remove_file(&index_path);
}
return None;
}
trace!("cache hit: {name}@{version}");
Expand All @@ -232,6 +234,7 @@ impl Store {
version: &str,
integrity: Option<&str>,
) -> Result<bool, Error> {
self.prepare_for_write()?;
let Some(index_path) = self.index_path(name, version, integrity) else {
return Ok(false);
};
Expand All @@ -253,6 +256,7 @@ impl Store {
integrity: Option<&str>,
index: &PackageIndex,
) -> Result<(), Error> {
self.prepare_for_write()?;
let index_path = self.index_path(name, version, integrity).ok_or_else(|| {
Error::Tar(format!(
"refusing to cache: invalid coordinate {name:?}@{version:?} or integrity {integrity:?}"
Expand Down
146 changes: 139 additions & 7 deletions crates/aube-store/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,35 @@ use sha1::Sha1;
#[cfg(test)]
use sha2::{Digest as _, Sha256, Sha384, Sha512};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
#[cfg(target_os = "macos")]
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, OnceLock};

pub const CACHE_DIR_NAME: &str = "aube-cache";
pub const INDEX_SUBDIR: &str = "index";
pub const VIRTUAL_STORE_SUBDIR: &str = "virtual-store";
pub const PACKUMENT_CACHE_SUBDIR: &str = "packuments-v1";
pub const PACKUMENT_FULL_CACHE_SUBDIR: &str = "packuments-full-v1";
pub const MAINTENANCE_LOCK_FILE: &str = ".maintenance.lock";

#[derive(Default)]
struct MaintenanceState {
shared: Mutex<Option<std::fs::File>>,
}

/// Exclusive store-maintenance lease held by `aube store prune`.
///
/// Every CAS/index writer takes the corresponding shared lease through
/// [`Store::prepare_for_write`], so holding this guard freezes one complete
/// prune snapshot across the GVS, cached indexes, and CAS files.
pub struct StoreMaintenanceGuard(std::fs::File);

impl Drop for StoreMaintenanceGuard {
fn drop(&mut self) {
let _ = self.0.unlock();
}
}

/// The global content-addressable store, owned by aube.
///
Expand Down Expand Up @@ -83,6 +102,8 @@ pub struct Store {
/// `storeDir` volume so materialized packages can be hardlinked
/// out of the CAS.
virtual_store_dir: PathBuf,
maintenance: Arc<MaintenanceState>,
migration_done: Arc<OnceLock<()>>,
/// When set, `create_cas_file` writes directly to the final
/// content-addressed path on non-Linux platforms instead of the
/// tempfile-then-rename dance. Caller must guarantee no concurrent
Expand Down Expand Up @@ -125,14 +146,14 @@ impl Store {
/// user-facing store dir. The global virtual store lands under
/// `cache_dir` unless [`Store::with_virtual_store_dir`] moves it.
pub fn with_dirs(root: PathBuf, cache_dir: PathBuf) -> Self {
let store = Self {
Self {
root,
virtual_store_dir: cache_dir.join(VIRTUAL_STORE_SUBDIR),
cache_dir,
maintenance: Arc::new(MaintenanceState::default()),
migration_done: Arc::new(OnceLock::new()),
fast_path: Arc::new(AtomicBool::new(false)),
};
store.migrate_legacy_index_dir();
store
}
}

/// Point the global virtual store somewhere other than
Expand All @@ -153,6 +174,8 @@ impl Store {
root,
virtual_store_dir: cache_dir.join(VIRTUAL_STORE_SUBDIR),
cache_dir,
maintenance: Arc::new(MaintenanceState::default()),
migration_done: Arc::new(OnceLock::new()),
fast_path: Arc::new(AtomicBool::new(false)),
}
}
Expand Down Expand Up @@ -210,12 +233,18 @@ impl Store {
/// aube wrote cached package indexes before they were moved next
/// to the CAS files. Used only by [`migrate_legacy_index_dir`]; new
/// code should always go through [`index_dir`].
fn legacy_index_dir(&self) -> PathBuf {
pub fn legacy_index_dir(&self) -> PathBuf {
self.cache_dir.join(INDEX_SUBDIR)
}

/// Whether opening this store for writes would migrate the legacy index.
pub fn legacy_index_migration_needed(&self) -> bool {
self.legacy_index_dir().exists() && !self.index_dir().exists()
}

/// One-shot migration from the legacy XDG-cache index location to
/// the in-store `v1/index/` directory. Runs at `Store::open`-time.
/// the in-store `v1/index/` directory. Runs when the store first prepares
/// for a write, after its shared maintenance lease is acquired.
///
/// The legacy location was a footgun under Docker BuildKit cache
/// mounts: users would mount the CAS files dir, the indexes would
Expand Down Expand Up @@ -288,6 +317,81 @@ impl Store {
}
}

pub fn maintenance_lock_path(&self) -> PathBuf {
self.store_v1_dir().join(MAINTENANCE_LOCK_FILE)
}

fn open_maintenance_lock(&self) -> Result<std::fs::File, Error> {
let path = self.maintenance_lock_path();
let Some(parent) = path.parent() else {
return Err(Error::Io(
path,
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"store maintenance lock has no parent",
),
));
};
std::fs::create_dir_all(parent).map_err(|e| Error::Io(parent.to_path_buf(), e))?;
std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&path)
.map_err(|e| Error::Io(path, e))
}

/// Acquire the shared writer lease and perform any pending legacy-index
/// migration. The lease is retained by this `Store` and all of its clones.
pub fn prepare_for_write(&self) -> Result<(), Error> {
let mut shared = self.maintenance.shared.lock().map_err(|_| {
Error::Io(
self.maintenance_lock_path(),
std::io::Error::other("store maintenance lock state is poisoned"),
)
})?;
if shared.is_none() {
let file = self.open_maintenance_lock()?;
file.lock_shared()
.map_err(|e| Error::Io(self.maintenance_lock_path(), e))?;
*shared = Some(file);
}
drop(shared);
self.migration_done.get_or_init(|| {
self.migrate_legacy_index_dir();
});
Ok(())
}

/// Acquire an exclusive lease for a complete prune plan/apply operation.
pub fn lock_for_maintenance(&self) -> Result<StoreMaintenanceGuard, Error> {
let shared = self.maintenance.shared.lock().map_err(|_| {
Error::Io(
self.maintenance_lock_path(),
std::io::Error::other("store maintenance lock state is poisoned"),
)
})?;
if shared.is_some() {
return Err(Error::Io(
self.maintenance_lock_path(),
std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"this Store already holds a writer lease",
),
));
}
let file = self.open_maintenance_lock()?;
file.lock()
.map_err(|e| Error::Io(self.maintenance_lock_path(), e))?;
Ok(StoreMaintenanceGuard(file))
}

/// Apply the legacy-index migration while an exclusive maintenance lease
/// is held. Used by real prune after its candidate plan is complete.
pub fn migrate_legacy_index_for_maintenance(&self, _guard: &StoreMaintenanceGuard) {
self.migrate_legacy_index_dir();
}

/// Directory for the global virtual store (materialized packages).
/// `<cacheDir>/virtual-store/` unless `globalVirtualStoreDir`
/// moved it, so it follows `cacheDir` by default.
Expand Down Expand Up @@ -368,6 +472,8 @@ mod tests {
root,
virtual_store_dir: cache_dir.join(VIRTUAL_STORE_SUBDIR),
cache_dir,
maintenance: Arc::new(MaintenanceState::default()),
migration_done: Arc::new(OnceLock::new()),
fast_path: Arc::new(AtomicBool::new(false)),
}
}
Expand Down Expand Up @@ -453,6 +559,32 @@ mod tests {
);
}

#[test]
fn maintenance_lock_waits_for_writer_lease() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().join("store/v1/files");
let cache = tmp.path().join("cache");
let writer = Store::with_dirs(root.clone(), cache.clone());
writer.prepare_for_write().unwrap();

let maintenance = Store::with_dirs(root, cache);
let (tx, rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let guard = maintenance.lock_for_maintenance().unwrap();
tx.send(()).unwrap();
drop(guard);
});
assert!(
rx.recv_timeout(std::time::Duration::from_millis(50))
.is_err(),
"maintenance must wait while a writer lease is live"
);
drop(writer);
rx.recv_timeout(std::time::Duration::from_secs(2))
.expect("maintenance should proceed after the writer exits");
handle.join().unwrap();
}

#[test]
fn store_v1_dir_is_parent_of_files() {
let tmp = tempfile::tempdir().unwrap();
Expand Down
Loading
Loading