feat(progress): add JSON backup progress output

config/README.md

@@ -90,6 +90,7 @@ If you want to contribute your own configuration, please
 | log-level-dependencies | Logging level for dependencies. Possible values: "off", "error", "warn", "info", "debug", "trace". | "warn"             |                          | RUSTIC_LOG_LEVEL_DEPENDENCIES                    | --log-level-dependencies |
 | log-file               | Path to the log file.                                                                              | No log file        | "/log/rustic.log"        | RUSTIC_LOG_FILE                                  | --log-file               |
 | no-progress            | If true, disables progress indicators.                                                             | false              |                          | RUSTIC_NO_PROGRESS                               | --no-progress            |
+| json-progress          | If true, writes progress as newline-delimited JSON.                                                | false              |                          | RUSTIC_JSON_PROGRESS                             | --json-progress          |
 | progress-interval      | The interval at which progress indicators are shown.                                               | "100ms"            | "1m"                     | RUSTIC_PROGRESS_INTERVAL                         | --progress-interval      |
 | group-by               | Group snapshots by any combination of host,label,paths,tags e.g. for "latest"                      | "host,label,paths" |                          | RUSTIC_GROUP_BY                                  | --group-by, -g           |
 | check-index            | If true, check the index and read pack headers if index information is missing.                    | false              |                          | RUSTIC_CHECK_INDEX                               | --check-index            |

config/full.toml

@@ -17,6 +17,7 @@ log-level-dryrun = "info" # any of "off", "error", "warn", "info", "debug", "tra
 log-level-dependencies = "warn" # any of "off", "error", "warn", "info", "debug", "trace"; default: "warn"
 log-file = "/path/to/rustic.log" # Default: not set
 no-progress = false
+json-progress = false
 progress-interval = "100ms"
 group-by = "host,label,paths"
 check-index = false

src/commands/backup.rs

@@ -27,7 +27,8 @@ use serde_with::serde_as;
 
 use rustic_core::{
     BackupOptions, CommandInput, ConfigOptions, KeyOptions, LocalSourceFilterOptions,
-    LocalSourceSaveOptions, ParentOptions, PathList, SnapshotOptions, repofile::SnapshotFile,
+    LocalSourceSaveOptions, ParentOptions, PathList, SnapshotOptions,
+    repofile::{SnapshotFile, SnapshotId, SnapshotSummary},
 };
 
 /// `backup` subcommand
@@ -377,7 +378,9 @@ impl BackupCmd {
             Ok(repo.backup(&backup_opts, &source, self.snap_opts.to_snapshot()?)?)
         })?;
 
-        if self.json {
+        if config.global.progress_options.json_progress {
+            write_json_progress_summary(&snap)?;
+        } else if self.json {
             let mut stdout = std::io::stdout();
             serde_json::to_writer_pretty(&mut stdout, &snap)?;
         } else if self.long {
@@ -431,6 +434,56 @@ impl BackupCmd {
     }
 }
 
+#[derive(Serialize)]
+struct JsonProgressSummary<'a> {
+    message_type: &'static str,
+    files_new: u64,
+    files_changed: u64,
+    files_unmodified: u64,
+    dirs_new: u64,
+    dirs_changed: u64,
+    dirs_unmodified: u64,
+    data_blobs: u64,
+    tree_blobs: u64,
+    data_added: u64,
+    data_added_packed: u64,
+    total_files_processed: u64,
+    total_bytes_processed: u64,
+    total_duration: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    snapshot_id: Option<&'a SnapshotId>,
+}
+
+impl<'a> JsonProgressSummary<'a> {
+    fn new(summary: &'a SnapshotSummary, snapshot_id: &'a SnapshotId) -> Self {
+        Self {
+            message_type: "summary",
+            files_new: summary.files_new,
+            files_changed: summary.files_changed,
+            files_unmodified: summary.files_unmodified,
+            dirs_new: summary.dirs_new,
+            dirs_changed: summary.dirs_changed,
+            dirs_unmodified: summary.dirs_unmodified,
+            data_blobs: summary.data_blobs,
+            tree_blobs: summary.tree_blobs,
+            data_added: summary.data_added,
+            data_added_packed: summary.data_added_packed,
+            total_files_processed: summary.total_files_processed,
+            total_bytes_processed: summary.total_bytes_processed,
+            total_duration: summary.total_duration,
+            snapshot_id: (*snapshot_id != SnapshotId::default()).then_some(snapshot_id),
+        }
+    }
+}
+
+fn write_json_progress_summary(snap: &SnapshotFile) -> Result<()> {
+    let summary = snap.summary.as_ref().unwrap();
+    let mut stdout = std::io::stdout();
+    serde_json::to_writer(&mut stdout, &JsonProgressSummary::new(summary, &snap.id))?;
+    println!();
+    Ok(())
+}
+
 #[cfg(not(any(feature = "prometheus", feature = "opentelemetry")))]
 fn publish_metrics(
     snap: &SnapshotFile,

src/config/progress_options.rs

@@ -1,6 +1,6 @@
 //! Progress Bar Config
 
-use std::{fmt::Write, time::Duration};
+use std::{fmt::Write, io::Write as _, time::Duration};
 
 use std::io::IsTerminal;
 use std::sync::{Arc, Mutex, OnceLock};
@@ -49,6 +49,16 @@ pub struct ProgressOptions {
     #[merge(strategy=conflate::bool::overwrite_false)]
     pub no_progress: bool,
 
+    /// Write progress as newline-delimited JSON
+    #[clap(
+        long,
+        global = true,
+        env = "RUSTIC_JSON_PROGRESS",
+        conflicts_with = "no_progress"
+    )]
+    #[merge(strategy=conflate::bool::overwrite_false)]
+    pub json_progress: bool,
+
     /// Interval to update progress bars (default: 100ms)
     #[clap(
         long,
@@ -89,14 +99,22 @@ impl ProgressOptions {
             return Progress::hidden();
         }
 
+        let interval = self.log_interval();
+        if self.json_progress {
+            return if interval > Duration::ZERO && matches!(kind, ProgressType::Bytes) {
+                Progress::new(JsonProgress::new(prefix, interval, kind))
+            } else {
+                Progress::hidden()
+            };
+        }
+
         if std::io::stderr().is_terminal() {
             Progress::new(InteractiveProgress::new(
                 prefix,
                 kind,
                 self.interactive_interval(),
             ))
         } else {
-            let interval = self.log_interval();
             if interval > Duration::ZERO {
                 Progress::new(NonInteractiveProgress::new(prefix, interval, kind))
             } else {
@@ -200,6 +218,28 @@ struct NonInteractiveState {
     last_log: Instant,
 }
 
+impl NonInteractiveState {
+    fn progress_text(&self, kind: ProgressType) -> String {
+        let format_value = |value| match kind {
+            ProgressType::Bytes => ByteSize(value).to_string(),
+            ProgressType::Counter | ProgressType::Spinner => value.to_string(),
+        };
+
+        self.length.map_or_else(
+            || format_value(self.position),
+            |len| format!("{} / {}", format_value(self.position), format_value(len)),
+        )
+    }
+
+    fn should_log(&self, interval: Duration) -> bool {
+        self.last_log.elapsed() >= interval
+    }
+
+    fn mark_logged(&mut self) {
+        self.last_log = Instant::now();
+    }
+}
+
 /// Periodic logger for non-interactive environments (i.e. systemd)
 /// Implemented thread-safe and decouples logging logic from indicatif
 #[derive(Clone, Debug)]
@@ -234,17 +274,7 @@ impl NonInteractiveProgress {
     }
 
     fn log_progress(&self, state: &NonInteractiveState) {
-        let progress = state.length.map_or_else(
-            || self.format_value(state.position),
-            |len| {
-                format!(
-                    "{} / {}",
-                    self.format_value(state.position),
-                    self.format_value(len)
-                )
-            },
-        );
-        info!("{}: {}", state.prefix, progress);
+        info!("{}: {}", state.prefix, state.progress_text(self.kind));
     }
 }
 
@@ -269,9 +299,9 @@ impl RusticProgress for NonInteractiveProgress {
         if let Ok(mut state) = self.state.lock() {
             state.position += inc;
 
-            if state.last_log.elapsed() >= self.interval {
+            if state.should_log(self.interval) {
                 self.log_progress(&state);
-                state.last_log = Instant::now();
+                state.mark_logged();
             }
         }
     }
@@ -289,3 +319,112 @@ impl RusticProgress for NonInteractiveProgress {
         );
     }
 }
+
+// ================ JSON ================
+
+/// Periodic JSON lines progress for machine-readable consumers
+#[derive(Clone, Debug)]
+pub struct JsonProgress {
+    state: Arc<Mutex<NonInteractiveState>>,
+    start: Instant,
+    interval: Duration,
+    kind: ProgressType,
+}
+
+#[derive(Serialize)]
+struct JsonProgressStatus {
+    message_type: &'static str,
+    seconds_elapsed: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    seconds_remaining: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    percent_done: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    total_bytes: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    bytes_done: Option<u64>,
+}
+
+impl JsonProgress {
+    fn new(prefix: &str, interval: Duration, kind: ProgressType) -> Self {
+        let now = Instant::now();
+        Self {
+            state: Arc::new(Mutex::new(NonInteractiveState {
+                prefix: prefix.to_string(),
+                position: 0,
+                length: None,
+                last_log: now,
+            })),
+            start: now,
+            interval,
+            kind,
+        }
+    }
+
+    fn log_progress(&self, state: &NonInteractiveState) {
+        let is_bytes = matches!(self.kind, ProgressType::Bytes);
+        let elapsed = self.start.elapsed().as_secs();
+        let percent_done = state
+            .length
+            .filter(|len| *len > 0)
+            .map(|len| (state.position as f64 / len as f64).min(1.0));
+        let seconds_remaining = match (state.position, state.length) {
+            (position, Some(len)) if position > 0 && len > position => {
+                Some(elapsed.saturating_mul(len - position) / position)
+            }
+            _ => None,
+        };
+
+        let status = JsonProgressStatus {
+            message_type: "status",
+            seconds_elapsed: elapsed,
+            seconds_remaining,
+            percent_done,
+            total_bytes: is_bytes.then_some(state.length).flatten(),
+            bytes_done: is_bytes.then_some(state.position),
+        };
+
+        let mut stdout = std::io::stdout().lock();
+        _ = serde_json::to_writer(&mut stdout, &status);
+        _ = writeln!(stdout);
+    }
+}
+
+impl RusticProgress for JsonProgress {
+    fn is_hidden(&self) -> bool {
+        false
+    }
+
+    fn set_length(&self, len: u64) {
+        if let Ok(mut state) = self.state.lock() {
+            state.length = Some(len);
+            self.log_progress(&state);
+            state.mark_logged();
+        }
+    }
+
+    fn set_title(&self, title: &str) {
+        if let Ok(mut state) = self.state.lock() {
+            state.prefix = title.to_string();
+        }
+    }
+
+    fn inc(&self, inc: u64) {
+        if let Ok(mut state) = self.state.lock() {
+            state.position += inc;
+
+            if state.should_log(self.interval) {
+                self.log_progress(&state);
+                state.mark_logged();
+            }
+        }
+    }
+
+    fn finish(&self) {
+        let Ok(state) = self.state.lock() else {
+            return;
+        };
+
+        self.log_progress(&state);
+    }
+}

src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap

@@ -9,6 +9,7 @@ dry-run = false
 dry-run-warmup = false
 check-index = false
 no-progress = false
+json-progress = false
 show-time-offset = false
 
 [global.hooks]

src/snapshots/rustic_rs__config__tests__default_config_passes.snap

@@ -19,6 +19,7 @@ RusticConfig {
         },
         progress_options: ProgressOptions {
             no_progress: false,
+            json_progress: false,
             progress_interval: None,
         },
         hooks: Hooks {

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap

@@ -9,6 +9,7 @@ dry-run = false
 dry-run-warmup = false
 check-index = false
 no-progress = false
+json-progress = false
 show-time-offset = false
 
 [global.hooks]

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap

@@ -19,6 +19,7 @@ RusticConfig {
         },
         progress_options: ProgressOptions {
             no_progress: false,
+            json_progress: false,
             progress_interval: None,
         },
         hooks: Hooks {

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap

@@ -9,6 +9,7 @@ dry-run = false
 dry-run-warmup = false
 check-index = false
 no-progress = false
+json-progress = false
 show-time-offset = false
 
 [global.hooks]

tests/snapshots/show_config__show_config_passes.snap

@@ -9,6 +9,7 @@ dry-run = false
 dry-run-warmup = false
 check-index = false
 no-progress = false
+json-progress = false
 show-time-offset = false
 
 [global.hooks]