Improve support for sparse cache.

With the addition of a notes backed cache we will have the
possibility of avoiding a "cold start" by transferring just
the notes to another repo. To keep the amount of extra data
small the notes cache is kept sparse: Not all commits have
and entry and also in shards by sequence number, as old
entries are unlikely to be relevant.
The old traversal logic did not perform very well with sparse
cache. Especially "find_known" proved to be a bottleneck.
So the traversal is revised now to work better in the sparse
cache case.

Change: cache-sparse-shards

Author: christian-schilling

docs/src/reference/filters.md

@@ -121,6 +121,22 @@ tree.
 Normally Josh will keep all commits in the filtered history whose tree differs from any of it's
 parents.
 
+### Pin tree contents
+
+`:pin` filter prevents appearance of selected subtrees for a given revision.
+
+In practical terms, it means that file and folder updates are "held off", and revisions are "pinned".
+If a tree entry already existed in the parent revision, that version will be chosen.
+Otherwise, the tree entry will not appear in the filtered commit.
+
+The source of the parent revision is always the first commit parent.
+
+Note that this filter is only practical when used with `:hook` or `workspace.josh`,
+as it should apply per-revision only. Applying `:pin` for the whole history
+will result in the subtree being excluded from all revisions.
+
+Refer to `pin_filter_workspace.t` and `pin_filter_hook.t` for reference.
+
 Filter order matters
 --------------------
 

josh-core/src/cache.rs

@@ -9,9 +9,20 @@ use std::sync::{LazyLock, RwLock};
 pub(crate) const CACHE_VERSION: u64 = 24;
 
 pub trait CacheBackend: Send + Sync {
-    fn read(&self, filter: filter::Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>>;
-
-    fn write(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()>;
+    fn read(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>>;
+
+    fn write(
+        &self,
+        filter: filter::Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()>;
 }
 
 pub trait FilterHook {
@@ -323,6 +334,11 @@ impl Transaction {
     }
 
     pub fn insert(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid, store: bool) {
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).unwrap()
+        } else {
+            0
+        };
         let mut t2 = self.t2.borrow_mut();
         t2.commit_map
             .entry(filter.id())
@@ -334,14 +350,13 @@ impl Transaction {
         // the history length by a very large factor.
         if store || from.as_bytes()[0] == 0 {
             t2.cache
-                .write_all(filter, from, to)
+                .write_all(filter, from, to, sequence_number)
                 .expect("Failed to write cache");
         }
     }
 
     pub fn get_missing(&self) -> Vec<(filter::Filter, git2::Oid)> {
         let mut missing = self.t2.borrow().missing.clone();
-        missing.sort_by_key(|(f, i)| (filter::nesting(*f), *f, *i));
         missing.dedup();
         missing.retain(|(f, i)| !self.known(*f, *i));
         self.t2.borrow_mut().missing = missing.clone();
@@ -358,7 +373,9 @@ impl Transaction {
         } else {
             let mut t2 = self.t2.borrow_mut();
             t2.misses += 1;
-            t2.missing.push((filter, from));
+            if !t2.missing.contains(&(filter, from)) {
+                t2.missing.insert(0, (filter, from));
+            }
             None
         }
     }
@@ -367,6 +384,11 @@ impl Transaction {
         if filter == filter::nop() {
             return Some(from);
         }
+        let sequence_number = if filter != filter::sequence_number() {
+            compute_sequence_number(self, from).unwrap()
+        } else {
+            0
+        };
         let t2 = self.t2.borrow_mut();
         if let Some(m) = t2.commit_map.get(&filter.id()) {
             if let Some(oid) = m.get(&from).cloned() {
@@ -376,7 +398,7 @@ impl Transaction {
 
         let oid = t2
             .cache
-            .read_propagate(filter, from)
+            .read_propagate(filter, from, sequence_number)
             .expect("Failed to read from cache backend");
 
         let oid = if let Some(oid) = oid { Some(oid) } else { None };
@@ -385,6 +407,9 @@ impl Transaction {
             if oid == git2::Oid::zero() {
                 return Some(oid);
             }
+            if filter == filter::sequence_number() {
+                return Some(oid);
+            }
 
             if self.repo.odb().unwrap().exists(oid) {
                 // Only report an object as cached if it exists in the object database.
@@ -396,3 +421,66 @@ impl Transaction {
         None
     }
 }
+
+/// Encode a `u128` into a 20-byte git OID (SHA-1 sized).
+/// The high 4 bytes of the OID are zero; the low 16 bytes
+/// contain the big-endian integer.
+pub fn oid_from_u128(n: u128) -> git2::Oid {
+    let mut bytes = [0u8; 20];
+    // place the 16 integer bytes at the end (big-endian)
+    bytes[20 - 16..].copy_from_slice(&n.to_be_bytes());
+    // Safe: length is exactly 20
+    git2::Oid::from_bytes(&bytes).expect("20-byte OID construction cannot fail")
+}
+
+/// Decode a `u128` previously encoded by `oid_from_u128`.
+pub fn u128_from_oid(oid: git2::Oid) -> u128 {
+    let b = oid.as_bytes();
+    let mut n = [0u8; 16];
+    n.copy_from_slice(&b[20 - 16..]); // take the last 16 bytes
+    u128::from_be_bytes(n)
+}
+
+pub fn compute_sequence_number(
+    transaction: &cache::Transaction,
+    input: git2::Oid,
+) -> JoshResult<u128> {
+    if let Some(count) = transaction.get(filter::sequence_number(), input) {
+        return Ok(u128_from_oid(count));
+    }
+
+    let commit = transaction.repo().find_commit(input)?;
+    if let Some(p) = commit.parent_ids().next() {
+        if let Some(count) = transaction.get(filter::sequence_number(), p) {
+            let pc = u128_from_oid(count);
+            transaction.insert(
+                filter::sequence_number(),
+                input,
+                oid_from_u128(pc + 1),
+                true,
+            );
+            return compute_sequence_number(transaction, input);
+        }
+    }
+
+    let mut walk = transaction.repo().revwalk()?;
+    walk.set_sorting(git2::Sort::REVERSE | git2::Sort::TOPOLOGICAL)?;
+    walk.push(input)?;
+
+    for c in walk {
+        let commit = transaction.repo().find_commit(c?)?;
+        let pc = if let Some(p) = commit.parent_ids().next() {
+            compute_sequence_number(transaction, p)?
+        } else {
+            0
+        };
+
+        transaction.insert(
+            filter::sequence_number(),
+            commit.id(),
+            oid_from_u128(pc + 1),
+            true,
+        );
+    }
+    compute_sequence_number(transaction, input)
+}

josh-core/src/cache_notes.rs

@@ -1,5 +1,6 @@
 use crate::JoshResult;
 use crate::cache::{CACHE_VERSION, CacheBackend};
+use crate::filter;
 use crate::filter::Filter;
 
 pub struct NotesCacheBackend {
@@ -15,24 +16,43 @@ impl NotesCacheBackend {
     }
 }
 
-fn is_note_eligible(oid: git2::Oid) -> bool {
-    oid.as_bytes()[0] == 0
+fn is_note_eligible(repo: &git2::Repository, oid: git2::Oid, sequence_number: u128) -> bool {
+    let parent_count = if let Ok(c) = repo.find_commit(oid) {
+        c.parent_ids().count()
+    } else {
+        return false;
+    };
+
+    sequence_number % 100 == 0 || parent_count != 1
 }
 
-fn note_path(key: git2::Oid) -> String {
-    format!("refs/josh/{}/{}", CACHE_VERSION, key)
+fn note_path(key: git2::Oid, sequence_number: u128) -> String {
+    format!(
+        "refs/josh/{}/{}/{}",
+        CACHE_VERSION,
+        sequence_number / 10000,
+        key,
+    )
 }
 
 impl CacheBackend for NotesCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
+        if filter == filter::sequence_number() {
+            return Ok(None);
+        }
         let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
-
-        if !is_note_eligible(from) {
+        if !is_note_eligible(&repo, from, sequence_number) {
             return Ok(None);
         }
 
-        if let Ok(note) = repo.find_note(Some(&note_path(key)), from) {
+        let key = crate::filter::as_tree(&*repo, filter)?;
+
+        if let Ok(note) = repo.find_note(Some(&note_path(key, sequence_number)), from) {
             let message = note.message().unwrap_or("");
             let result = git2::Oid::from_str(message)?;
 
@@ -42,20 +62,29 @@ impl CacheBackend for NotesCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
-        let repo = self.repo.lock()?;
-        let key = crate::filter::as_tree(&repo, filter)?;
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        sequence_number: u128,
+    ) -> JoshResult<()> {
+        if filter == filter::sequence_number() {
+            return Ok(());
+        }
 
-        if !is_note_eligible(from) {
+        let repo = self.repo.lock()?;
+        if !is_note_eligible(&*repo, from, sequence_number) {
             return Ok(());
         }
 
+        let key = crate::filter::as_tree(&*repo, filter)?;
         let signature = crate::cache::josh_commit_signature()?;
 
         repo.note(
             &signature,
             &signature,
-            Some(&note_path(key)),
+            Some(&note_path(key, sequence_number)),
             from,
             &to.to_string(),
             true,

josh-core/src/cache_sled.rs

@@ -80,7 +80,12 @@ fn insert_sled_tree(filter: Filter) -> sled::Tree {
 }
 
 impl CacheBackend for SledCacheBackend {
-    fn read(&self, filter: Filter, from: git2::Oid) -> JoshResult<Option<git2::Oid>> {
+    fn read(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<Option<git2::Oid>> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())
@@ -94,7 +99,13 @@ impl CacheBackend for SledCacheBackend {
         }
     }
 
-    fn write(&self, filter: Filter, from: git2::Oid, to: git2::Oid) -> JoshResult<()> {
+    fn write(
+        &self,
+        filter: Filter,
+        from: git2::Oid,
+        to: git2::Oid,
+        _sequence_number: u128,
+    ) -> JoshResult<()> {
         let mut trees = self.trees.lock()?;
         let tree = trees
             .entry(filter.id())

josh-core/src/cache_stack.rs

@@ -33,9 +33,10 @@ impl CacheStack {
         filter: filter::Filter,
         from: git2::Oid,
         to: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<()> {
         for backend in &self.backends {
-            backend.write(filter, from, to)?;
+            backend.write(filter, from, to, sequence_number)?;
         }
 
         Ok(())
@@ -51,16 +52,19 @@ impl CacheStack {
         &self,
         filter: filter::Filter,
         from: git2::Oid,
+        sequence_number: u128,
     ) -> JoshResult<Option<git2::Oid>> {
         let values = self
             .backends
             .iter()
             .enumerate()
-            .find_map(|(index, backend)| match backend.read(filter, from) {
-                Ok(None) => None,
-                Ok(Some(oid)) => Some(Ok((index, oid))),
-                Err(e) => Some(Err(e)),
-            });
+            .find_map(
+                |(index, backend)| match backend.read(filter, from, sequence_number) {
+                    Ok(None) => None,
+                    Ok(Some(oid)) => Some(Ok((index, oid))),
+                    Err(e) => Some(Err(e)),
+                },
+            );
 
         let (index, oid) = match values {
             // None of the backends had the value
@@ -74,7 +78,7 @@ impl CacheStack {
         self.backends
             .iter()
             .take(index)
-            .try_for_each(|backend| backend.write(filter, from, oid))?;
+            .try_for_each(|backend| backend.write(filter, from, oid, sequence_number))?;
 
         Ok(Some(oid))
     }