Add way to get the info to open an entry independently of the store

rklaehn · rklaehn · commit 78c06ec3de22 · 2025-03-11T19:50:15.000+02:00
diff --git a/src/store/bao_file.rs b/src/store/bao_file.rs
@@ -189,16 +189,7 @@ impl FileStorage {
     }
 
     fn current_size(&self) -> io::Result<u64> {
-        let len = self.sizes.metadata()?.len();
-        if len < 8 {
-            Ok(0)
-        } else {
-            // todo: use the last full u64 in case the sizes file is not a multiple of 8
-            // bytes. Not sure how that would happen, but we should handle it.
-            let mut buf = [0u8; 8];
-            self.sizes.read_exact_at(len - 8, &mut buf)?;
-            Ok(u64::from_le_bytes(buf))
-        }
+        read_current_size(&self.sizes)
     }
 
     fn write_batch(&mut self, size: u64, batch: &[BaoContentItem]) -> io::Result<()> {
@@ -470,6 +461,18 @@ impl AsyncSliceReader for OutboardReader {
     }
 }
 
+pub fn read_current_size(sizes: &File) -> io::Result<u64> {
+    let len = sizes.metadata()?.len();
+    if len < 8 {
+        Ok(0)
+    } else {
+        let len = len & !7;
+        let mut buf = [0u8; 8];
+        sizes.read_exact_at(len - 8, &mut buf)?;
+        Ok(u64::from_le_bytes(buf))
+    }
+}
+
 enum HandleChange {
     None,
     MemToFile,
diff --git a/src/store/fs.rs b/src/store/fs.rs
@@ -98,9 +98,9 @@ use tables::{ReadOnlyTables, ReadableTables, Tables};
 
 use self::{tables::DeleteSet, test_support::EntryData, util::PeekableFlumeReceiver};
 use super::{
-    bao_file::{BaoFileConfig, BaoFileHandle, BaoFileHandleWeak, CreateCb},
-    temp_name, BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, EntryStatus, ExportMode,
-    ExportProgressCb, ImportMode, ImportProgress, Map, ReadableStore, TempCounterMap,
+    bao_file::{read_current_size, BaoFileConfig, BaoFileHandle, BaoFileHandleWeak, CreateCb},
+    temp_name, BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, EntryPathOrData, EntryStatus,
+    ExportMode, ExportProgressCb, ImportMode, ImportProgress, Map, ReadableStore, TempCounterMap,
 };
 use crate::{
     store::{
@@ -532,6 +532,10 @@ pub(crate) enum ActorMessage {
         hash: Hash,
         tx: oneshot::Sender<ActorResult<EntryStatus>>,
     },
+    EntryPathOrData {
+        hash: Hash,
+        tx: oneshot::Sender<ActorResult<Option<EntryPathOrData>>>,
+    },
     #[cfg(test)]
     /// Query method: get the full entry state for a hash, both in memory and in redb.
     /// This is everything we got about the entry, including the actual inline outboard and data.
@@ -664,6 +668,7 @@ impl ActorMessage {
             | Self::Tags { .. }
             | Self::GcStart { .. }
             | Self::GetFullEntryState { .. }
+            | Self::EntryPathOrData { .. }
             | Self::Dump => MessageCategory::ReadOnly,
             Self::Import { .. }
             | Self::Export { .. }
@@ -870,6 +875,14 @@ impl StoreInner {
         Ok(tags)
     }
 
+    async fn entry_path_or_data(&self, hash: Hash) -> OuterResult<Option<EntryPathOrData>> {
+        let (tx, rx) = oneshot::channel();
+        self.tx
+            .send(ActorMessage::EntryPathOrData { hash, tx })
+            .await?;
+        Ok(rx.await??)
+    }
+
     async fn set_tag(&self, tag: Tag, value: Option<HashAndFormat>) -> OuterResult<()> {
         let (tx, rx) = oneshot::channel();
         self.tx
@@ -1371,6 +1384,10 @@ impl super::Store for Store {
         .await??)
     }
 
+    async fn entry_path_or_data(&self, hash: Hash) -> io::Result<Option<EntryPathOrData>> {
+        Ok(self.0.entry_path_or_data(hash).await?)
+    }
+
     async fn set_tag(&self, name: Tag, hash: Option<HashAndFormat>) -> io::Result<()> {
         Ok(self.0.set_tag(name, hash).await?)
     }
@@ -2266,6 +2283,65 @@ impl ActorState {
         Ok(())
     }
 
+    fn entry_path_or_data(
+        &mut self,
+        tables: &impl ReadableTables,
+        hash: Hash,
+    ) -> ActorResult<Option<EntryPathOrData>> {
+        let data_path = || self.options.path.owned_data_path(&hash);
+        let outboard_path = || self.options.path.owned_outboard_path(&hash);
+        let sizes_path = || self.options.path.owned_sizes_path(&hash);
+        Ok(match tables.blobs().get(hash)? {
+            Some(guard) => match guard.value() {
+                EntryState::Complete {
+                    data_location,
+                    outboard_location,
+                } => {
+                    let data = match data_location {
+                        DataLocation::External(paths, size) => {
+                            let path = paths.first().ok_or_else(|| {
+                                ActorError::Inconsistent("external data missing".to_owned())
+                            })?;
+                            MemOrFile::File((path.clone(), size))
+                        }
+                        DataLocation::Owned(size) => MemOrFile::File((data_path(), size)),
+                        DataLocation::Inline(_) => {
+                            let data = tables.inline_data().get(hash)?.ok_or_else(|| {
+                                ActorError::Inconsistent("inline data missing".to_owned())
+                            })?;
+                            MemOrFile::Mem(data.value().to_vec().into())
+                        }
+                    };
+                    let outboard = match outboard_location {
+                        OutboardLocation::Owned => MemOrFile::File(outboard_path()),
+                        OutboardLocation::Inline(_) => MemOrFile::Mem(
+                            tables
+                                .inline_outboard()
+                                .get(hash)?
+                                .ok_or_else(|| {
+                                    ActorError::Inconsistent("inline outboard missing".to_owned())
+                                })?
+                                .value()
+                                .to_vec()
+                                .into(),
+                        ),
+                        OutboardLocation::NotNeeded => MemOrFile::Mem(Bytes::new()),
+                    };
+                    Some(EntryPathOrData { data, outboard })
+                }
+                EntryState::Partial { .. } => {
+                    let sizes = std::fs::File::open(sizes_path())?;
+                    let size = read_current_size(&sizes)?;
+                    Some(EntryPathOrData {
+                        data: MemOrFile::File((data_path(), size)),
+                        outboard: MemOrFile::File(outboard_path()),
+                    })
+                }
+            },
+            None => None,
+        })
+    }
+
     fn handle_toplevel(&mut self, db: &redb::Database, msg: ActorMessage) -> ActorResult<()> {
         match msg {
             ActorMessage::UpdateInlineOptions {
@@ -2339,6 +2415,10 @@ impl ActorState {
                 let res = self.get_full_entry_state(tables, hash);
                 tx.send(res).ok();
             }
+            ActorMessage::EntryPathOrData { hash, tx } => {
+                let res = self.entry_path_or_data(tables, hash);
+                tx.send(res).ok();
+            }
             x => return Ok(Err(x)),
         }
         Ok(Ok(()))
diff --git a/src/store/mem.rs b/src/store/mem.rs
@@ -19,16 +19,16 @@ use futures_lite::{Stream, StreamExt};
 use iroh_io::AsyncSliceReader;
 
 use super::{
-    temp_name, BaoBatchWriter, ConsistencyCheckProgress, ExportMode, ExportProgressCb, ImportMode,
-    ImportProgress, Map, TempCounterMap,
+    temp_name, BaoBatchWriter, ConsistencyCheckProgress, EntryPathOrData, ExportMode,
+    ExportProgressCb, ImportMode, ImportProgress, Map, TempCounterMap,
 };
 use crate::{
     store::{
         mutable_mem_storage::MutableMemStorage, BaoBlobSize, MapEntry, MapEntryMut, ReadableStore,
     },
     util::{
         progress::{BoxedProgressSender, IdGenerator, IgnoreProgressSender, ProgressSender},
-        TagCounter, TagDrop,
+        MemOrFile, TagCounter, TagDrop,
     },
     BlobFormat, Hash, HashAndFormat, Tag, TempTag, IROH_BLOCK_SIZE,
 };
@@ -246,6 +246,22 @@ impl super::Store for Store {
         Ok(())
     }
 
+    async fn entry_path_or_data(&self, hash: Hash) -> io::Result<Option<EntryPathOrData>> {
+        let state = self.read_lock();
+        match state.entries.get(&hash) {
+            Some(entry) => {
+                let inner = entry.inner.data.read().unwrap();
+                let data = inner.data.to_vec().into();
+                let outboard = inner.outboard.to_vec().into();
+                Ok(Some(EntryPathOrData {
+                    data: MemOrFile::Mem(data),
+                    outboard: MemOrFile::Mem(outboard),
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
     async fn shutdown(&self) {}
 
     async fn sync(&self) -> io::Result<()> {
diff --git a/src/store/readonly_mem.rs b/src/store/readonly_mem.rs
@@ -18,15 +18,18 @@ use futures_lite::Stream;
 use iroh_io::AsyncSliceReader;
 use tokio::io::AsyncWriteExt;
 
-use super::{BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, DbIter, ExportProgressCb};
+use super::{
+    BaoBatchWriter, BaoBlobSize, ConsistencyCheckProgress, DbIter, EntryPathOrData,
+    ExportProgressCb,
+};
 use crate::{
     store::{
         EntryStatus, ExportMode, ImportMode, ImportProgress, Map, MapEntry, MapEntryMut,
         ReadableStore,
     },
     util::{
         progress::{BoxedProgressSender, IdGenerator, ProgressSender},
-        Tag,
+        MemOrFile, Tag,
     },
     BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE,
 };
@@ -321,6 +324,24 @@ impl super::Store for Store {
         Err(io::Error::new(io::ErrorKind::Other, "not implemented"))
     }
 
+    fn entry_path_or_data(
+        &self,
+        hash: Hash,
+    ) -> impl Future<Output = io::Result<Option<super::EntryPathOrData>>> + Send {
+        let res = match self.0.get(&hash) {
+            Some((outboard, data)) => {
+                let outboard = outboard.data.clone();
+                let data = data.clone();
+                Ok(Some(EntryPathOrData {
+                    outboard: MemOrFile::Mem(outboard),
+                    data: MemOrFile::Mem(data),
+                }))
+            }
+            None => Ok(None),
+        };
+        futures_lite::future::ready(res)
+    }
+
     fn temp_tag(&self, inner: HashAndFormat) -> TempTag {
         TempTag::new(inner, None)
     }
diff --git a/src/store/traits.rs b/src/store/traits.rs
@@ -20,7 +20,7 @@ use crate::{
     util::{
         local_pool::{self, LocalPool},
         progress::{BoxedProgressSender, IdGenerator, ProgressSender},
-        Tag,
+        MemOrFile, Tag,
     },
     BlobFormat, Hash, HashAndFormat, TempTag, IROH_BLOCK_SIZE,
 };
@@ -42,6 +42,15 @@ pub enum EntryStatus {
     NotFound,
 }
 
+/// Get the path or data for an entry
+#[derive(Debug)]
+pub struct EntryPathOrData {
+    /// The path to the data file or the inline data
+    pub data: MemOrFile<Bytes, (PathBuf, u64)>,
+    /// The path to the outboard file, or the inline outboard
+    pub outboard: MemOrFile<Bytes, PathBuf>,
+}
+
 /// The size of a bao file
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)]
 pub enum BaoBlobSize {
@@ -384,6 +393,12 @@ pub trait Store: ReadableStore + MapMut + std::fmt::Debug {
     ) -> impl Future<Output = io::Result<()>> + Send {
         validate_impl(self, repair, tx)
     }
+
+    /// Get the info needed to open an entry independently of the store.
+    fn entry_path_or_data(
+        &self,
+        hash: Hash,
+    ) -> impl Future<Output = io::Result<Option<EntryPathOrData>>> + Send;
 }
 
 async fn validate_impl(
