From 7fd544ef5bf9f169d0417325ed8cd28fdb56a325 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 12 Nov 2025 20:22:40 +0100 Subject: [PATCH 01/12] replace cronjob with systemd timer --- pyinfra_borgbackup/__init__.py | 41 ++++++++++++++++++++------ pyinfra_borgbackup/borgbackup.cron | 4 --- pyinfra_borgbackup/borgbackup.service | 9 ++++++ pyinfra_borgbackup/borgbackup.timer.j2 | 8 +++++ 4 files changed, 49 insertions(+), 13 deletions(-) delete mode 100644 pyinfra_borgbackup/borgbackup.cron create mode 100644 pyinfra_borgbackup/borgbackup.service create mode 100644 pyinfra_borgbackup/borgbackup.timer.j2 diff --git a/pyinfra_borgbackup/__init__.py b/pyinfra_borgbackup/__init__.py index 4c55ca8..5b55240 100644 --- a/pyinfra_borgbackup/__init__.py +++ b/pyinfra_borgbackup/__init__.py @@ -2,7 +2,7 @@ import random from io import StringIO -from pyinfra.operations import apt, files, server +from pyinfra.operations import apt, files, server, systemd def deploy_borgbackup( @@ -90,13 +90,36 @@ def deploy_borgbackup( ], ) - files.template( - name="Create cron job for backup script", - src=importlib.resources.files(__package__).joinpath("borgbackup.cron"), - dest="/etc/cron.d/borgbackup", - user="root", - group="root", + files.file( + name="Remove old backup cronjob, replaced by systemd timer", + path="/etc/cron.d/borgbackup", + present=False, + ) + + reconcile_service_file = files.put( + src=importlib.resources.files(__package__).joinpath("borgbackup.service"), + dest="/etc/systemd/system/borgbackup.service", mode="644", - minute=str(random.randint(0, 59)), - hour=str(random.randint(0, 4)), + ) + systemd.service( + name="Setup borgbackup service", + service="borgbackup.service", + running=False, + enabled=False, + daemon_reload=reconcile_service_file.changed, + ) + + reconcile_timer_file = files.put( + src=importlib.resources.files(__package__).joinpath("borgbackup.timer.j2"), + dest="/etc/systemd/system/borgbackup.timer", + mode="644", + minute = str(random.randint(0, 59)), + hour = str(random.randint(0, 4)), + ) + systemd.service( + name="Setup borgbackup timer", + service="borgbackup.timer", + running=True, + enabled=True, + daemon_reload=reconcile_timer_file.changed, ) diff --git a/pyinfra_borgbackup/borgbackup.cron b/pyinfra_borgbackup/borgbackup.cron deleted file mode 100644 index e10719a..0000000 --- a/pyinfra_borgbackup/borgbackup.cron +++ /dev/null @@ -1,4 +0,0 @@ -SHELL=/bin/bash -PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -MAILTO=root -{{ minute }} {{ hour }} * * * root /root/backup.sh diff --git a/pyinfra_borgbackup/borgbackup.service b/pyinfra_borgbackup/borgbackup.service new file mode 100644 index 0000000..c788e0b --- /dev/null +++ b/pyinfra_borgbackup/borgbackup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Run nightly backups +After=network.target + +[Service] +Type=oneshot +ExecStart=/root/backup.sh + + diff --git a/pyinfra_borgbackup/borgbackup.timer.j2 b/pyinfra_borgbackup/borgbackup.timer.j2 new file mode 100644 index 0000000..9619463 --- /dev/null +++ b/pyinfra_borgbackup/borgbackup.timer.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Run nightly backups + +[Timer] +OnCalendar=*-*-* {{ hour }}:{{ minute }}:00 + +[Install] +WantedBy=timers.target From 4426b917a4d7b3df52f9fa2a14ca98a26a58c4b7 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 12 Nov 2025 20:24:07 +0100 Subject: [PATCH 02/12] timer is a template actually --- pyinfra_borgbackup/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinfra_borgbackup/__init__.py b/pyinfra_borgbackup/__init__.py index 5b55240..a43b178 100644 --- a/pyinfra_borgbackup/__init__.py +++ b/pyinfra_borgbackup/__init__.py @@ -109,7 +109,7 @@ def deploy_borgbackup( daemon_reload=reconcile_service_file.changed, ) - reconcile_timer_file = files.put( + reconcile_timer_file = files.template( src=importlib.resources.files(__package__).joinpath("borgbackup.timer.j2"), dest="/etc/systemd/system/borgbackup.timer", mode="644", From 2e4cfa97caaee898ab63e8a0b2d555ee1c7addcb Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 17 Nov 2025 09:38:24 +0100 Subject: [PATCH 03/12] fail properly --- pyinfra_borgbackup/backup.sh.j2 | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index 88b9806..3d90390 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -21,9 +21,6 @@ echo "$DEST1" > /dev/null export BORG_RSH='ssh -F /root/.ssh/config -o "StrictHostKeyChecking=no"' -# don't fail just because the pod is not running -set +e - # check if variable exists: https://stackoverflow.com/a/11369388 if [ "$SKIP_CHECK" = "true" ]; then echo "skipping borg check." @@ -49,33 +46,18 @@ borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ bo --exclude /mnt \ --exclude /tmp \ --exclude /media \ - --exclude /var/lib/lxcfs && FINISHED=true & -if [ -n "${DEST2-}" ]; then - # If there is a second backup destination, create a backup there. - # Note: not yet supported by pyinfra deploy. - borg create --stats --compression lzma ${DEST2}::'backup{now:%Y-%m-%d-%H}' {{ borg_args }} \ - --exclude *.cache/ \ - --exclude /dev \ - --exclude /proc \ - --exclude /sys \ - --exclude /var/run \ - --exclude /run \ - --exclude /lost+found \ - --exclude /mnt \ - --exclude /tmp \ - --exclude /media \ - --exclude /var/lib/lxcfs && FINISHED=true & -fi -wait + --exclude /var/lib/lxcfs +exit_code=$? {% if prometheus_file is defined %} -if [ "$FINISHED" = "true" ]; then +if [ $exit_code = 0 ]; then echo "borgbackup_last_completed $(date +%s)" > {{ prometheus_file }} +else + echo "borg failed with exit code $exit_code" fi {%- endif %} start_services -set -e borg prune --keep-daily=7 --keep-weekly=4 ${DEST1} borg compact ${DEST1} if [ -n "${DEST2-}" ]; then From 55fc7bf0d4d106a81914efce5d8c887f58b40c48 Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 17 Nov 2025 10:51:54 +0100 Subject: [PATCH 04/12] fix backup.sh exit code --- pyinfra_borgbackup/backup.sh.j2 | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index 3d90390..edbb256 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -2,7 +2,9 @@ function start_services { # start the services/containers which write data again + rv=$? if [ -f /root/backup-pre.py ]; then python3 /root/backup-pre.py start; fi + exit $(( $rv + $? )) } trap "start_services" EXIT @@ -47,14 +49,9 @@ borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ bo --exclude /tmp \ --exclude /media \ --exclude /var/lib/lxcfs -exit_code=$? {% if prometheus_file is defined %} -if [ $exit_code = 0 ]; then - echo "borgbackup_last_completed $(date +%s)" > {{ prometheus_file }} -else - echo "borg failed with exit code $exit_code" -fi +echo "borgbackup_last_completed $(date +%s)" > {{ prometheus_file }} {%- endif %} start_services From ab9b196e64de7a4b60b4ecd89365fdf88bf41f8c Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 17 Nov 2025 11:26:22 +0100 Subject: [PATCH 05/12] fix backup-pre.py exit code handling --- pyinfra_borgbackup/backup-pre.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyinfra_borgbackup/backup-pre.py b/pyinfra_borgbackup/backup-pre.py index 104f115..29f9900 100644 --- a/pyinfra_borgbackup/backup-pre.py +++ b/pyinfra_borgbackup/backup-pre.py @@ -3,6 +3,12 @@ import argparse import os + +def cmd(command: str) -> int: + """Run a command and return its exit code. """ + return os.waitstatus_to_exitcode(os.system(command)) + + services = { "isolationbot": "isolationbot.service", "root": "docker.service", @@ -19,9 +25,9 @@ for user in services: if user == "root": - returncode = os.system(f"systemctl {args.command} {services[user]}") + returncode = cmd(f"systemctl {args.command} {services[user]}") else: - returncode = os.system( + returncode = cmd( f"su -l {user} -c 'systemctl --user {args.command} {services[user]}'" ) if returncode != 0: From 895083d2e60ad082819cf85fb41e2f5b60072d7f Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 17 Nov 2025 13:28:22 +0100 Subject: [PATCH 06/12] exclude /var/log by default --- pyinfra_borgbackup/backup.sh.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index edbb256..abd414c 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -43,6 +43,7 @@ borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ bo --exclude /proc \ --exclude /sys \ --exclude /var/run \ + --exclude /var/log \ --exclude /run \ --exclude /lost+found \ --exclude /mnt \ From 31b805d92183f240aae5a832347dd674c1f7e1b3 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 18 Nov 2025 09:39:25 +0100 Subject: [PATCH 07/12] don't fail if borg exits with rc 1 (warning) --- pyinfra_borgbackup/backup.sh.j2 | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index abd414c..65c21c7 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -23,11 +23,21 @@ echo "$DEST1" > /dev/null export BORG_RSH='ssh -F /root/.ssh/config -o "StrictHostKeyChecking=no"' +function run_borg () { + set +e # don't fail if borg quits with exit code 1 (warning) + borg $@ + borg_rc=$? + if [[ $borg_rc -gt 1 ]]; then + exit $borg_rc + fi + set -e +} + # check if variable exists: https://stackoverflow.com/a/11369388 if [ "$SKIP_CHECK" = "true" ]; then echo "skipping borg check." else - borg check ${DEST1} & + run_borg check ${DEST1} & if [ -n "${DEST2-}" ]; then borg check ${DEST2} & fi @@ -37,7 +47,7 @@ wait # stop services/containers which write data if [ -f /root/backup-pre.py ]; then python3 /root/backup-pre.py stop; fi -borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ borg_args }} \ +run_borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ borg_args }} \ --exclude *.cache/ \ --exclude /dev \ --exclude /proc \ @@ -56,8 +66,8 @@ echo "borgbackup_last_completed $(date +%s)" > {{ prometheus_file }} {%- endif %} start_services -borg prune --keep-daily=7 --keep-weekly=4 ${DEST1} -borg compact ${DEST1} +run_borg prune --keep-daily=7 --keep-weekly=4 ${DEST1} +run_borg compact ${DEST1} if [ -n "${DEST2-}" ]; then borg prune --keep-daily=7 --keep-weekly=4 ${DEST2} borg compact ${DEST2} From d5bc42916c137921a20c5c02ce40b9af91a0d451 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 18 Nov 2025 09:40:43 +0100 Subject: [PATCH 08/12] drop DEST2, only one backup host supported --- pyinfra_borgbackup/backup.sh.j2 | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index 65c21c7..c23cae4 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -37,12 +37,8 @@ function run_borg () { if [ "$SKIP_CHECK" = "true" ]; then echo "skipping borg check." else - run_borg check ${DEST1} & - if [ -n "${DEST2-}" ]; then - borg check ${DEST2} & - fi + run_borg check ${DEST1} fi -wait # stop services/containers which write data if [ -f /root/backup-pre.py ]; then python3 /root/backup-pre.py stop; fi @@ -68,8 +64,4 @@ start_services run_borg prune --keep-daily=7 --keep-weekly=4 ${DEST1} run_borg compact ${DEST1} -if [ -n "${DEST2-}" ]; then - borg prune --keep-daily=7 --keep-weekly=4 ${DEST2} - borg compact ${DEST2} -fi From 171c21cc347a294f2e224e444ed02fa858586e7c Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 18 Nov 2025 09:42:57 +0100 Subject: [PATCH 09/12] reconcile_service -> backup_service --- pyinfra_borgbackup/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyinfra_borgbackup/__init__.py b/pyinfra_borgbackup/__init__.py index a43b178..957a340 100644 --- a/pyinfra_borgbackup/__init__.py +++ b/pyinfra_borgbackup/__init__.py @@ -96,7 +96,7 @@ def deploy_borgbackup( present=False, ) - reconcile_service_file = files.put( + backup_service_file = files.put( src=importlib.resources.files(__package__).joinpath("borgbackup.service"), dest="/etc/systemd/system/borgbackup.service", mode="644", @@ -106,10 +106,10 @@ def deploy_borgbackup( service="borgbackup.service", running=False, enabled=False, - daemon_reload=reconcile_service_file.changed, + daemon_reload=backup_service_file.changed, ) - reconcile_timer_file = files.template( + backup_timer_file = files.template( src=importlib.resources.files(__package__).joinpath("borgbackup.timer.j2"), dest="/etc/systemd/system/borgbackup.timer", mode="644", @@ -121,5 +121,5 @@ def deploy_borgbackup( service="borgbackup.timer", running=True, enabled=True, - daemon_reload=reconcile_timer_file.changed, + daemon_reload=backup_timer_file.changed, ) From 2bfc3b93511e89b1ae799b4ccea46a20566a7512 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 18 Nov 2025 09:46:15 +0100 Subject: [PATCH 10/12] introduce ruff formatting --- .github/workflows/ci.yaml | 20 ++++++++++++++++++++ pyinfra_borgbackup/__init__.py | 8 +++----- pyinfra_borgbackup/backup-pre.py | 6 ++---- 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..42e4c9c --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,20 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + python: + name: Lint Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install black + run: pip install ruff + - name: Check style + run: ruff format --quiet --line-length 120 --diff */*.py + - name: Lint + run: ruff check */*.py diff --git a/pyinfra_borgbackup/__init__.py b/pyinfra_borgbackup/__init__.py index 957a340..72033ec 100644 --- a/pyinfra_borgbackup/__init__.py +++ b/pyinfra_borgbackup/__init__.py @@ -56,9 +56,7 @@ def deploy_borgbackup( if borg_repo.startswith("hetzner-backup:"): files.put( name="create SSH config", - src=importlib.resources.files(__package__) - .joinpath("dot_ssh", "config") - .open("rb"), + src=importlib.resources.files(__package__).joinpath("dot_ssh", "config").open("rb"), dest="/root/.ssh/config", user="root", group="root", @@ -113,8 +111,8 @@ def deploy_borgbackup( src=importlib.resources.files(__package__).joinpath("borgbackup.timer.j2"), dest="/etc/systemd/system/borgbackup.timer", mode="644", - minute = str(random.randint(0, 59)), - hour = str(random.randint(0, 4)), + minute=str(random.randint(0, 59)), + hour=str(random.randint(0, 4)), ) systemd.service( name="Setup borgbackup timer", diff --git a/pyinfra_borgbackup/backup-pre.py b/pyinfra_borgbackup/backup-pre.py index 29f9900..f5ef5d0 100644 --- a/pyinfra_borgbackup/backup-pre.py +++ b/pyinfra_borgbackup/backup-pre.py @@ -5,7 +5,7 @@ def cmd(command: str) -> int: - """Run a command and return its exit code. """ + """Run a command and return its exit code.""" return os.waitstatus_to_exitcode(os.system(command)) @@ -27,8 +27,6 @@ def cmd(command: str) -> int: if user == "root": returncode = cmd(f"systemctl {args.command} {services[user]}") else: - returncode = cmd( - f"su -l {user} -c 'systemctl --user {args.command} {services[user]}'" - ) + returncode = cmd(f"su -l {user} -c 'systemctl --user {args.command} {services[user]}'") if returncode != 0: print(f"WARNING: Failed to {args.command} {services[user]} as {user} user") From 3ef45e14ce29b97b3a1f1a974d2bcab0a0bafec5 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 7 Dec 2025 09:31:41 +0100 Subject: [PATCH 11/12] use BORG_REPO implicitly, instead of DEST1 --- pyinfra_borgbackup/__init__.py | 2 +- pyinfra_borgbackup/backup.sh.j2 | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyinfra_borgbackup/__init__.py b/pyinfra_borgbackup/__init__.py index 72033ec..c8cb042 100644 --- a/pyinfra_borgbackup/__init__.py +++ b/pyinfra_borgbackup/__init__.py @@ -28,7 +28,7 @@ def deploy_borgbackup( secrets = [ f"BORG_PASSPHRASE={passphrase}", - f"DEST1={borg_repo}", + f"BORG_REPO={borg_repo}", "SKIP_CHECK=true" if skip_check else "SKIP_CHECK=false", ] env = "\n".join(secrets) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index c23cae4..63dd954 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -18,7 +18,7 @@ export $(xargs < /root/backup.env) # check if these variables have been set echo "$BORG_PASSPHRASE" > /dev/null -echo "$DEST1" > /dev/null +echo "$BORG_REPO" > /dev/null : "${SKIP_CHECK:='false'}" export BORG_RSH='ssh -F /root/.ssh/config -o "StrictHostKeyChecking=no"' @@ -37,13 +37,13 @@ function run_borg () { if [ "$SKIP_CHECK" = "true" ]; then echo "skipping borg check." else - run_borg check ${DEST1} + run_borg check fi # stop services/containers which write data if [ -f /root/backup-pre.py ]; then python3 /root/backup-pre.py stop; fi -run_borg create --stats --compression lzma ${DEST1}::'backup{now:%Y-%m-%d-%H}' {{ borg_args }} \ +run_borg create --stats --compression lzma ::'backup{now:%Y-%m-%d-%H}' {{ borg_args }} \ --exclude *.cache/ \ --exclude /dev \ --exclude /proc \ @@ -62,6 +62,6 @@ echo "borgbackup_last_completed $(date +%s)" > {{ prometheus_file }} {%- endif %} start_services -run_borg prune --keep-daily=7 --keep-weekly=4 ${DEST1} -run_borg compact ${DEST1} +run_borg prune --keep-daily=7 --keep-weekly=4 +run_borg compact From 98d3615baff5f0799a5c3ef57b722650ce6118bc Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 9 Dec 2025 11:38:58 +0100 Subject: [PATCH 12/12] don't print secrets to log --- pyinfra_borgbackup/backup.sh.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyinfra_borgbackup/backup.sh.j2 b/pyinfra_borgbackup/backup.sh.j2 index 63dd954..d2298b7 100644 --- a/pyinfra_borgbackup/backup.sh.j2 +++ b/pyinfra_borgbackup/backup.sh.j2 @@ -9,7 +9,7 @@ function start_services { trap "start_services" EXIT -set -exuo pipefail +set -euo pipefail # setup env DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -21,6 +21,8 @@ echo "$BORG_PASSPHRASE" > /dev/null echo "$BORG_REPO" > /dev/null : "${SKIP_CHECK:='false'}" +set -x + export BORG_RSH='ssh -F /root/.ssh/config -o "StrictHostKeyChecking=no"' function run_borg () {