0.4.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lfspull"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"
 authors = ["Volume Graphics GmbH"]
@@ -11,9 +11,9 @@ description = "A simple git lfs file pulling implementation in pure rust. Can on
 
 [dependencies]
 clap = { version = "4.1", features = ["derive", "env"] }
-thiserror = "1"
-reqwest = { version="0.11" , features = ["json", "stream"] }
-http = "0.2"
+thiserror = "2"
+reqwest = { version="0.12" , features = ["json", "stream"] }
+http = "1.3"
 serde = {version ="1.0", features=['derive']}
 serde_json = "1.0"
 bytes = "1.4"
@@ -30,7 +30,7 @@ futures-util = "0.3.30"
 tempfile = "3.12"
 
 [dev-dependencies]
-cucumber = "0.19.1"
+cucumber = "0.21"
 tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
 uuid = { version = "1.2", features = ["serde", "v4"] }
 

README.md

@@ -32,6 +32,11 @@ Please see our docs.rs for example code and the gherkin tests for how to check t
 
 ## Changelog
 
+### 0.4.0
+
+- upgrade a few dependencies
+- add retry attempt when failing fetching from git
+
 ### 0.3.1
 
 - fix bug when trying to rename temp file to cache file, but cache file is already created and locked by other parallel job

src/lib.rs

@@ -81,6 +81,9 @@ pub mod prelude {
         /// something failed while creating tempfile
         #[error("TempFile error: {0}")]
         TempFile(String),
+        /// all download attempts have failed
+        #[error("Maximum download attempts reached")]
+        ReachedMaxDownloadAttempt,
     }
 }
 pub use prelude::FilePullMode;

src/main.rs

@@ -22,6 +22,10 @@ struct Args {
     #[clap(short = 'b', long)]
     random_bytes: Option<usize>,
 
+    ///max number of retry attempt when http request fails
+    #[clap(short, long, default_value_t = 3)]
+    max_retry: u32,
+
     /// Print debug information
     #[clap(short, long)]
     verbose: bool,
@@ -48,14 +52,19 @@ pub async fn main() -> Result<(), LFSError> {
     let access_token = args.access_token.as_deref();
     if let Some(file) = args.file_to_pull {
         info!("Single file mode: {}", file.to_string_lossy());
-        let result = lfspull::pull_file(file, access_token, args.random_bytes).await?;
+        let result =
+            lfspull::pull_file(file, access_token, args.max_retry, args.random_bytes).await?;
         info!("Result: {}", result);
     }
     if let Some(recurse_pattern) = args.recurse_pattern {
         info!("Glob-recurse mode: {}", &recurse_pattern);
-        let results =
-            lfspull::glob_recurse_pull_directory(&recurse_pattern, access_token, args.random_bytes)
-                .await?;
+        let results = lfspull::glob_recurse_pull_directory(
+            &recurse_pattern,
+            access_token,
+            args.max_retry,
+            args.random_bytes,
+        )
+        .await?;
         info!("Pulling finished! Listing files and sources: ");
 
         results.into_iter().enumerate().for_each(|(id, (n, r))| {

src/repo_tools/mod.rs

@@ -31,7 +31,7 @@ async fn get_real_repo_root<P: AsRef<Path>>(repo_path: P) -> Result<PathBuf, LFS
     let git_path = repo_path.as_ref().join(".git");
     let real_git_path = if repo_path.as_ref().join(".git").is_file() {
         //worktree case
-        let worktree_file_contents = fat_io_wrap_tokio(git_path, tokio::fs::read_to_string).await?;
+        let worktree_file_contents = fat_io_wrap_tokio(git_path, fs::read_to_string).await?;
         let worktree_path = worktree_file_contents
             .split(':')
             .find(|c| c.contains(".git"))
@@ -99,8 +99,10 @@ async fn get_file_cached<P: AsRef<Path>>(
     repo_root: P,
     metadata: &primitives::MetaData,
     access_token: Option<&str>,
+    max_retry: u32,
     randomizer_bytes: Option<usize>,
 ) -> Result<(PathBuf, FilePullMode), LFSError> {
+    debug!("version: {}", &metadata.version);
     let cache_dir = get_cache_dir(&repo_root, metadata).await?;
     debug!("cache dir {:?}", &cache_dir);
     let cache_file = cache_dir.join(&metadata.oid);
@@ -118,8 +120,14 @@ async fn get_file_cached<P: AsRef<Path>>(
                 )
             })?;
 
-        let temp_file =
-            primitives::download_file(metadata, &repo_url, access_token, randomizer_bytes).await?;
+        let temp_file = primitives::download_file(
+            metadata,
+            &repo_url,
+            access_token,
+            max_retry,
+            randomizer_bytes,
+        )
+        .await?;
         if cache_file.exists() {
             info!(
                 "cache file {:?} is already written from other process",
@@ -159,6 +167,7 @@ async fn get_file_cached<P: AsRef<Path>>(
 pub async fn pull_file<P: AsRef<Path>>(
     lfs_file: P,
     access_token: Option<&str>,
+    max_retry: u32,
     randomizer_bytes: Option<usize>,
 ) -> Result<FilePullMode, LFSError> {
     info!("Pulling file {}", lfs_file.as_ref().to_string_lossy());
@@ -176,8 +185,14 @@ pub async fn pull_file<P: AsRef<Path>>(
     let repo_root = get_repo_root(&lfs_file).await.map_err(|e| {
         LFSError::DirectoryTraversalError(format!("Could not find git repo root: {:?}", e))
     })?;
-    let (file_name_cached, origin) =
-        get_file_cached(&repo_root, &metadata, access_token, randomizer_bytes).await?;
+    let (file_name_cached, origin) = get_file_cached(
+        &repo_root,
+        &metadata,
+        access_token,
+        max_retry,
+        randomizer_bytes,
+    )
+    .await?;
     info!(
         "Found file (Origin: {:?}), linking to {}",
         origin,
@@ -212,26 +227,29 @@ fn glob_recurse(wildcard_pattern: &str) -> Result<Vec<PathBuf>, LFSError> {
 ///
 /// * `access_token` - the token for Bearer-Auth via HTTPS
 ///
+/// * `max retry` - max number of retry attempt when http request fails
+///
 /// * `randomizer bytes` - bytes used to create a randomized named temp file
 ///
 /// # Examples
 ///
 /// Load all .jpg files from all subdirectories
 /// ```no_run
-/// let result = lfspull::glob_recurse_pull_directory("dir/to/pull/**/*.jpg", Some("secret-token"), Some(5));
+/// let result = lfspull::glob_recurse_pull_directory("dir/to/pull/**/*.jpg", Some("secret-token"), 3, Some(5));
 /// ```
 ///
 pub async fn glob_recurse_pull_directory(
     wildcard_pattern: &str,
     access_token: Option<&str>,
+    max_retry: u32,
     randomizer_bytes: Option<usize>,
 ) -> Result<Vec<(String, FilePullMode)>, LFSError> {
     let mut result_vec = Vec::new();
     let files = glob_recurse(wildcard_pattern)?;
     for path in files {
         result_vec.push((
             path.to_string_lossy().to_string(),
-            pull_file(&path, access_token, randomizer_bytes).await?,
+            pull_file(&path, access_token, max_retry, randomizer_bytes).await?,
         ));
     }
 

src/repo_tools/primitives.rs

@@ -10,9 +10,11 @@ use std::convert::TryInto;
 use std::io::Write;
 use std::path::Path;
 use std::path::PathBuf;
+use std::time::Duration;
 use tempfile::NamedTempFile;
 use tokio::fs;
 use tokio::io::AsyncReadExt;
+use tokio::time::sleep;
 use tracing::{debug, error, info};
 use url::Url;
 use vg_errortools::{fat_io_wrap_tokio, FatIOError};
@@ -119,7 +121,7 @@ fn url_with_auth(url: &str, access_token: Option<&str>) -> Result<Url, LFSError>
     Ok(url)
 }
 
-pub async fn download_file(
+pub async fn handle_download(
     meta_data: &MetaData,
     repo_remote_url: &str,
     access_token: Option<&str>,
@@ -148,7 +150,7 @@ pub async fn download_file(
         .await?;
     if !response.status().is_success() {
         let status = response.status();
-        println!(
+        error!(
             "Failed to request git lfs actions with status code {} and body {}",
             status,
             response.text().await?,
@@ -237,6 +239,32 @@ pub async fn download_file(
     }
 }
 
+pub async fn download_file(
+    meta_data: &MetaData,
+    repo_remote_url: &str,
+    access_token: Option<&str>,
+    max_retry: u32,
+    randomizer_bytes: Option<usize>,
+) -> Result<NamedTempFile, LFSError> {
+    for attempt in 1..=max_retry {
+        debug!("Download attempt {attempt}");
+        match handle_download(meta_data, repo_remote_url, access_token, randomizer_bytes).await {
+            Ok(tempfile) => {
+                return Ok(tempfile);
+            }
+            Err(e) => {
+                if matches!(e, LFSError::AccessDenied) {
+                    return Err(e);
+                }
+                error!("Download error: {e}. Attempting another download: {attempt}");
+                sleep(Duration::from_secs(1)).await;
+            }
+        }
+    }
+
+    Err(LFSError::ReachedMaxDownloadAttempt)
+}
+
 pub async fn is_lfs_node_file<P: AsRef<Path>>(path: P) -> Result<bool, LFSError> {
     if path.as_ref().is_dir() {
         return Ok(false);
@@ -314,7 +342,7 @@ size 226848"#;
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     async fn try_pull_from_demo_repo() {
         let parsed = parse_lfs_string(LFS_TEST_DATA).expect("Could not parse demo-string!");
-        let temp_file = download_file(&parsed, URL, None, None)
+        let temp_file = download_file(&parsed, URL, None, 3, None)
             .await
             .expect("could not download file");
         let temp_size = temp_file

tests/lfspull.rs

@@ -55,7 +55,7 @@ async fn pull_file_step(world: &mut LFSWorld) {
         .clone()
         .join(TEST_LFS_FILE_NAME);
     world.pull_result = Some(
-        lfspull::pull_file(file_path, None, Some(5))
+        lfspull::pull_file(file_path, None, 3, Some(5))
             .await
             .expect("Could not pull file"),
     );
@@ -65,7 +65,7 @@ async fn pull_file_step(world: &mut LFSWorld) {
 async fn pull_directory(world: &mut LFSWorld) {
     let fake_repo = world.current_fake_repo.as_ref().unwrap().to_string_lossy();
     let pattern = format!("{}/**/*", fake_repo);
-    let recurse_pull = lfspull::glob_recurse_pull_directory(&pattern, None, Some(5))
+    let recurse_pull = lfspull::glob_recurse_pull_directory(&pattern, None, 3, Some(5))
         .await
         .expect("Could not pull directory")
         .into_iter()