pu-orfe · pubino · Oct 19, 2025 · Oct 18, 2025 · Oct 18, 2025 · Oct 19, 2025
diff --git a/README.md b/README.md
@@ -40,6 +40,7 @@ Optional keys:
 - `TD_NOTIFY` – comma-separated email recipients.
 - `TD_DRY_RUN` – `1` or `true` for `rsync --dry-run`.
 - `TD_SLURM_RUNTIME` – job runtime (`HH:MM:SS`, defaults to `01:00:00`).
+- `TD_RSYNC_OPTS` – additional `rsync` flags (e.g., `--exclude .venv --delete`).
 
 ## Test Mode
 
@@ -67,6 +68,7 @@ Below is a representative profile and command sequence for a project that mirror
 	TD_NOTIFY=user@example.edu
 	TD_DRY_RUN=1
 	TD_SLURM_RUNTIME=02:00:00
+	TD_RSYNC_OPTS=--exclude .venv --delete
 	```
 
 2. Export any optional runtime variables, then execute the scheduler:
@@ -76,4 +78,15 @@ Below is a representative profile and command sequence for a project that mirror
 	td-sync/bin/td-sync
 	```
 
-3. Tiny Data Sync will submit a Slurm job that performs an `rsync --dry-run`, writes a log under `logs/`, emails a short summary to the specified recipient, and requeues itself to run again after seven days. Switching `TD_DRY_RUN` to `0` promotes the synchronization from a preview to a real transfer.
+3. Tiny Data Sync will submit a Slurm job that performs an `rsync --dry-run`, writes a log under `logs/`, emails a short summary to the specified recipient, and requeues itself to run again after seven days. Switching `TD_DRY_RUN` to `0` promotes the synchronization from a preview to a real transfer. Any custom `TD_RSYNC_OPTS` (such as excluding `.venv`) are appended to the `rsync` invocation.
+
+### Passing Complex rsync Filters
+
+When `TD_RSYNC_OPTS` contains multiple arguments, quote values exactly as you would on the command line. For example:
+
+```env
+TD_RSYNC_OPTS="--filter='- .venv' --filter='- __pycache__'"
+```
+
+Tiny Data Sync evaluates these options inside the generated job script, so each quoted segment is preserved and forwarded to `rsync` without additional escaping.
+
diff --git a/bin/td-sync b/bin/td-sync
@@ -136,7 +136,7 @@ purge_old_logs() {
 load_profile() {
   local profile_path="$1"
 
-  unset TD_SRC TD_DEST TD_SUBMIT_INTERVAL TD_SLURM_ACCOUNT TD_NOTIFY TD_DRY_RUN TD_SLURM_RUNTIME
+  unset TD_SRC TD_DEST TD_SUBMIT_INTERVAL TD_SLURM_ACCOUNT TD_NOTIFY TD_DRY_RUN TD_SLURM_RUNTIME TD_RSYNC_OPTS
 
   # shellcheck disable=SC1090
   source "$profile_path"
@@ -206,6 +206,7 @@ write_job_script() {
   local log_base_escaped
   local job_name_escaped
   local self_path_escaped
+  local td_rsync_opts_escaped
 
   printf -v mailer_escaped '%q' "$mailer_cmd"
   printf -v td_src_escaped '%q' "${TD_SRC:-}"
@@ -218,6 +219,7 @@ write_job_script() {
   printf -v log_base_escaped '%q' "$profile_label"
   printf -v job_name_escaped '%q' "td-sync-$sanitized_name"
   printf -v self_path_escaped '%q' "$job_script_path"
+  printf -v td_rsync_opts_escaped '%q' "${TD_RSYNC_OPTS:-}"
 
   {
     printf '#!/usr/bin/env bash\n'
@@ -233,7 +235,8 @@ write_job_script() {
   printf 'LOG_BASENAME=%s\n' "$log_base_escaped"
     printf 'MAILER_CMD=%s\n' "$mailer_escaped"
     printf 'JOB_NAME=%s\n' "$job_name_escaped"
-    printf 'SELF_PATH=%s\n\n' "$self_path_escaped"
+    printf 'SELF_PATH=%s\n' "$self_path_escaped"
+    printf 'TD_RSYNC_OPTS=%s\n\n' "$td_rsync_opts_escaped"
     cat <<'EOF'
 mkdir -p "${TD_LOG_DIR}"
 
@@ -246,8 +249,15 @@ rsync_opts=("-av")
 if [[ "$TD_DRY_RUN" == "1" ]]; then
   rsync_opts+=("--dry-run")
 fi
+if [[ -n "$TD_RSYNC_OPTS" ]]; then
+  extra_opts=()
+  eval "extra_opts=($TD_RSYNC_OPTS)"
+  rsync_opts+=("${extra_opts[@]}")
+  unset extra_opts
+fi
 rsync_opts+=("$TD_SRC" "$TD_DEST")
 
+set +e
 {
   printf 'Tiny Data Sync job: %s\n' "$JOB_NAME"
   printf 'Started: %s\n' "$timestamp"
@@ -257,16 +267,39 @@ rsync_opts+=("$TD_SRC" "$TD_DEST")
   rsync "${rsync_opts[@]}"
 } &>"$log_file"
 rsync_status=$?
+set -e
+
+requeue_note="Requeue not requested"
 
 if [[ $rsync_status -ne 0 ]]; then
   printf 'rsync exited with status %s\n' "$rsync_status" >>"$log_file"
+  requeue_note="Job not requeued; rsync exited with status $rsync_status"
+fi
+
+if [[ -n "$TD_SUBMIT_INTERVAL" ]]; then
+  if [[ $rsync_status -eq 0 ]]; then
+    if sbatch --job-name="$JOB_NAME" --time="$TD_SLURM_RUNTIME" --account="$TD_SLURM_ACCOUNT" --begin="now+${TD_SUBMIT_INTERVAL}" "$SELF_PATH"; then
+      requeue_note="Next run scheduled after $TD_SUBMIT_INTERVAL"
+    else
+      printf 'Unable to requeue job with interval %s\n' "$TD_SUBMIT_INTERVAL" >>"$log_file"
+      requeue_note="Failed to schedule next run; check logs"
+    fi
+  else
+    requeue_note="Job not requeued; rsync exited with status $rsync_status"
+  fi
+else
+  requeue_note="Requeue interval not set"
 fi
 
 {
   printf 'Tiny Data Sync job: %s\n' "$JOB_NAME"
   printf 'Started: %s\n' "$timestamp"
   printf 'Dry run: %s\n' "$TD_DRY_RUN"
   printf 'Rsync exit status: %s\n' "$rsync_status"
+  if [[ -n "$TD_RSYNC_OPTS" ]]; then
+    printf 'Additional rsync options: %s\n' "$TD_RSYNC_OPTS"
+  fi
+  printf 'Requeue status: %s\n' "$requeue_note"
   printf 'Log file: %s\n\n' "$log_file"
   printf 'Last 200 lines of log:\n'
   tail -n 200 "$log_file"
@@ -289,11 +322,8 @@ if [[ -n "$TD_NOTIFY" && -n "$MAILER_CMD" ]]; then
   done
 fi
 
-if [[ -n "$TD_SUBMIT_INTERVAL" ]]; then
-  if ! sbatch --job-name="$JOB_NAME" --time="$TD_SLURM_RUNTIME" --account="$TD_SLURM_ACCOUNT" --begin="now+${TD_SUBMIT_INTERVAL}" "$SELF_PATH"; then
-    printf 'Unable to requeue job with interval %s\n' "$TD_SUBMIT_INTERVAL" >>"$log_file"
-    exit 1
-  fi
+if [[ $rsync_status -ne 0 ]]; then
+  exit "$rsync_status"
 fi
 EOF
   } >"$job_script_path"

diff --git a/tests/td_sync.bats b/tests/td_sync.bats
@@ -11,6 +11,7 @@ setup() {
   unset TD_CONFIG_DIR
   unset TD_LOG_DIR
   unset TD_LOG_RETENTION_DAYS
+  unset TD_RSYNC_OPTS
   mkdir -p "$HOME"
   rm -f "$TD_MOCK_OUTPUT"
 }
@@ -59,6 +60,7 @@ TD_SLURM_ACCOUNT=research
 TD_NOTIFY=ops@example.com
 TD_DRY_RUN=1
 TD_SLURM_RUNTIME=02:30:00
+TD_RSYNC_OPTS="--exclude .venv --delete"
 EOF
   export TD_CONFIG_DIR="$config_dir"
   export TD_LOG_DIR="$log_dir"
@@ -68,6 +70,7 @@ EOF
   job_script_contents="$(cat "$PROJECT_ROOT/jobs/sample.job.sh")"
   [[ "$job_script_contents" == *"TD_NOTIFY=ops@example.com"* ]]
   [[ "$job_script_contents" == *"TD_DRY_RUN=1"* ]]
+  [[ "$job_script_contents" == *"TD_RSYNC_OPTS=--exclude\\ .venv\\ --delete"* ]]
   mock_log="$(cat "$TD_MOCK_OUTPUT")"
   [[ "$mock_log" == *"--account=research"* ]]
   [[ "$mock_log" == *"--time=02:30:00"* ]]