WIP

Author: rklaehn

Cargo.toml

@@ -95,7 +95,7 @@ futures-util = "0.3.30"
 testdir = "0.9.1"
 
 [features]
-default = ["fs-store", "net_protocol"]
+default = ["fs-store", "net_protocol", "formats-collection"]
 downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"]
 net_protocol = ["downloader", "dep:futures-util"]
 fs-store = ["dep:reflink-copy", "redb", "dep:tempfile"]
@@ -112,6 +112,8 @@ rpc = [
     "dep:walkdir",
     "downloader",
 ]
+formats = []
+formats-collection = ["formats"]
 
 example-iroh = [
   "dep:clap",

examples/local-swarm-discovery.rs

@@ -140,7 +140,10 @@ mod progress {
         ProgressStyle,
     };
     use iroh_blobs::{
-        fetch::{db::DownloadProgress, progress::BlobProgress, Stats},
+        fetch::{
+            progress::{BlobProgress, DownloadProgress},
+            Stats,
+        },
         Hash,
     };
 

src/cli.rs

@@ -19,7 +19,10 @@ use iroh::{NodeAddr, PublicKey, RelayUrl};
 use tokio::io::AsyncWriteExt;
 
 use crate::{
-    fetch::{db::DownloadProgress, progress::BlobProgress, Stats},
+    fetch::{
+        progress::{BlobProgress, DownloadProgress},
+        Stats,
+    },
     net_protocol::DownloadMode,
     provider::AddProgress,
     rpc::client::blobs::{

src/downloader.rs

@@ -55,7 +55,7 @@ use tokio_util::{either::Either, sync::CancellationToken, time::delay_queue};
 use tracing::{debug, error, error_span, trace, warn, Instrument};
 
 use crate::{
-    fetch::{db::DownloadProgress, Stats},
+    fetch::{progress::DownloadProgress, Stats},
     metrics::Metrics,
     store::Store,
     util::{local_pool::LocalPoolHandle, progress::ProgressSender},

src/downloader/get.rs

@@ -7,8 +7,8 @@ use iroh::endpoint;
 
 use super::{progress::BroadcastProgressSender, DownloadKind, FailureAction, GetStartFut, Getter};
 use crate::{
-    fetch::{db::fetch_to_db_in_steps, Error},
-    store::Store,
+    fetch::Error,
+    store::{fetch_to_db_in_steps, FetchState, FetchStateNeedsConn, Store},
 };
 
 impl From<Error> for FailureAction {
@@ -34,7 +34,7 @@ pub(crate) struct IoGetter<S: Store> {
 
 impl<S: Store> Getter for IoGetter<S> {
     type Connection = endpoint::Connection;
-    type NeedsConn = crate::fetch::db::FetchStateNeedsConn;
+    type NeedsConn = FetchStateNeedsConn;
 
     fn get(
         &mut self,
@@ -45,10 +45,8 @@ impl<S: Store> Getter for IoGetter<S> {
         async move {
             match fetch_to_db_in_steps(store, kind.hash_and_format(), progress_sender).await {
                 Err(err) => Err(err.into()),
-                Ok(crate::fetch::db::FetchState::Complete(stats)) => {
-                    Ok(super::GetOutput::Complete(stats))
-                }
-                Ok(crate::fetch::db::FetchState::NeedsConn(needs_conn)) => {
+                Ok(FetchState::Complete(stats)) => Ok(super::GetOutput::Complete(stats)),
+                Ok(FetchState::NeedsConn(needs_conn)) => {
                     Ok(super::GetOutput::NeedsConn(needs_conn))
                 }
             }
@@ -57,7 +55,7 @@ impl<S: Store> Getter for IoGetter<S> {
     }
 }
 
-impl super::NeedsConn<endpoint::Connection> for crate::fetch::db::FetchStateNeedsConn {
+impl super::NeedsConn<endpoint::Connection> for FetchStateNeedsConn {
     fn proceed(self, conn: endpoint::Connection) -> super::GetProceedFut {
         async move {
             let res = self.proceed(conn).await;

src/downloader/progress.rs

@@ -11,7 +11,7 @@ use parking_lot::Mutex;
 
 use super::DownloadKind;
 use crate::{
-    fetch::{db::DownloadProgress, progress::TransferState},
+    fetch::progress::{DownloadProgress, TransferState},
     util::progress::{AsyncChannelProgressSender, IdGenerator, ProgressSendError, ProgressSender},
 };
 

src/downloader/test.rs

@@ -10,10 +10,7 @@ use iroh::SecretKey;
 
 use super::*;
 use crate::{
-    fetch::{
-        db::BlobId,
-        progress::{BlobProgress, TransferState},
-    },
+    fetch::progress::{BlobId, BlobProgress, TransferState},
     util::{
         local_pool::LocalPool,
         progress::{AsyncChannelProgressSender, IdGenerator},

src/fetch.rs

@@ -31,7 +31,6 @@ use crate::{
 
 mod error;
 pub use error::Error;
-pub mod db;
 pub mod progress;
 pub mod request;
 

src/fetch/progress.rs

@@ -5,7 +5,7 @@ use std::{collections::HashMap, num::NonZeroU64};
 use serde::{Deserialize, Serialize};
 use tracing::warn;
 
-use super::db::{BlobId, DownloadProgress};
+use super::Stats;
 use crate::{protocol::RangeSpec, store::BaoBlobSize, Hash};
 
 /// The identifier for progress events.
@@ -183,3 +183,90 @@ impl TransferState {
         }
     }
 }
+
+/// Progress updates for the get operation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum DownloadProgress {
+    /// Initial state if subscribing to a running or queued transfer.
+    InitialState(TransferState),
+    /// Data was found locally.
+    FoundLocal {
+        /// child offset
+        child: BlobId,
+        /// The hash of the entry.
+        hash: Hash,
+        /// The size of the entry in bytes.
+        size: BaoBlobSize,
+        /// The ranges that are available locally.
+        valid_ranges: RangeSpec,
+    },
+    /// A new connection was established.
+    Connected,
+    /// An item was found with hash `hash`, from now on referred to via `id`.
+    Found {
+        /// A new unique progress id for this entry.
+        id: u64,
+        /// Identifier for this blob within this download.
+        ///
+        /// Will always be [`BlobId::Root`] unless a hashseq is downloaded, in which case this
+        /// allows to identify the children by their offset in the hashseq.
+        child: BlobId,
+        /// The hash of the entry.
+        hash: Hash,
+        /// The size of the entry in bytes.
+        size: u64,
+    },
+    /// An item was found with hash `hash`, from now on referred to via `id`.
+    FoundHashSeq {
+        /// The name of the entry.
+        hash: Hash,
+        /// Number of children in the collection, if known.
+        children: u64,
+    },
+    /// We got progress ingesting item `id`.
+    Progress {
+        /// The unique id of the entry.
+        id: u64,
+        /// The offset of the progress, in bytes.
+        offset: u64,
+    },
+    /// We are done with `id`.
+    Done {
+        /// The unique id of the entry.
+        id: u64,
+    },
+    /// All operations finished.
+    ///
+    /// This will be the last message in the stream.
+    AllDone(Stats),
+    /// We got an error and need to abort.
+    ///
+    /// This will be the last message in the stream.
+    Abort(serde_error::Error),
+}
+
+/// The id of a blob in a transfer
+#[derive(
+    Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, std::hash::Hash, Serialize, Deserialize,
+)]
+pub enum BlobId {
+    /// The root blob (child id 0)
+    Root,
+    /// A child blob (child id > 0)
+    Child(NonZeroU64),
+}
+
+impl BlobId {
+    pub(crate) fn from_offset(id: u64) -> Self {
+        NonZeroU64::new(id).map(Self::Child).unwrap_or(Self::Root)
+    }
+}
+
+impl From<BlobId> for u64 {
+    fn from(value: BlobId) -> Self {
+        match value {
+            BlobId::Root => 0,
+            BlobId::Child(id) => id.into(),
+        }
+    }
+}

src/format.rs

@@ -13,4 +13,5 @@
 //! n-1 items, where n is the number of blobs in the HashSeq.
 //!
 //! [postcard]: https://docs.rs/postcard/latest/postcard/
+#[cfg(feature = "formats-collection")]
 pub mod collection;