From 6df27e0f709b4ab3e58983bd1d18720442714311 Mon Sep 17 00:00:00 2001
From: Guillaume Subiron <maethor@subiron.org>
Date: Thu, 13 Feb 2025 08:52:08 +0100
Subject: [PATCH 1/5] replace borg source/ignore_missing by filter_warnings,
and allow to disable warnings on file changed during backup
---
examples/example.borg | 29 +++++++++++++++--------
handlers/borg.in | 54 +++++++++++++++++++++++++++++++------------
test/borg.bats | 4 ++--
3 files changed, 61 insertions(+), 26 deletions(-)
diff --git a/examples/example.borg b/examples/example.borg
index 8f536cb..4fced66 100644
--- a/examples/example.borg
+++ b/examples/example.borg
@@ -124,21 +124,32 @@ exclude = /var/lib/mysql
## Default:
# prune_options =
-## by default borg emits a warning when a source file or directory
-## vanishes during the backup operations
-## set to yes to ignore such warnings
+## Path to the directory that will hold borg's cache files. By default this is
+## empty, which will let borg use its default path of "~/.cache/borg".
##
-## Example:
-## ignore_missing = yes
+## Default:
+# cache_directory =
+
+## by default borg emits various warnings that are impossible to check on large
+## infrastructures.
+## - when some files/repositories included in borg create does not exists
+## - when some files have changed during the backup (happens a lot on log files)
+## This option allows to disable these warning.
##
## Default:
-# ignore_missing =
+# filter_warnings = yes
-## Path to the directory that will hold borg's cache files. By default this is
-## empty, which will let borg use its default path of "~/.cache/borg".
+## when filter_warning == yes, allows to choose to disable warning if
+## file changed during backup
##
## Default:
-# cache_directory =
+# warning_if_file_changed_during_backup = yes
+
+## when warning_if_file_changed_during_backup == yes, allows to ignore some
+## paths or filenames.
+##
+## Default:
+# file_changes_to_ignore = /
######################################################
## destination section
diff --git a/handlers/borg.in b/handlers/borg.in
index d31d844..aaf658e 100644
--- a/handlers/borg.in
+++ b/handlers/borg.in
@@ -37,7 +37,9 @@ getconf prune yes
getconf keep 30d
getconf prune_options
getconf cache_directory
-getconf ignore_missing
+getconf filter_warnings yes
+getconf warning_if_file_changed_during_backup yes
+getconf file_changes_to_ignore /
setsection dest
getconf user
@@ -180,24 +182,46 @@ execstr="${execstr} ${excludes} $execstr_repository::$execstr_archive ${includes
debug "executing borg create"
debug "$nice $execstr"
-if [ $test = 0 ]; then
- output=`$nice su -c "$execstr" 2>&1`
+if [ "$test" = 0 ]; then
+ output=$($nice su -c "$execstr" 2>&1)
ret=$?
- if [ $ret = 0 ]; then
- debug $output
- info "Successfully finished backing up source."
- elif [ $ret = 1 ]; then
- warnmsg=$(echo "$output" | @SED@ -n '1,/^-\+$/{x;p;d;}; x' | @SED@ '/^$/d')
- if [ "$ignore_missing" = "yes" ] && ! echo "$warnmsg" | grep -qv '\[Errno 2\] No such file or directory:'; then
- debug $output
- info "Backing up source finished with missing file warnings."
+ if [ $ret = 0 ]; then # borg ok
+ debug "$output"
+ info "Successfully finished backing up source"
+ elif [ $ret = 1 ]; then # borg warning
+ # Borg can return 1 for warnings that are impossible to manually check on large infrastructures :
+ # - when some files/repositories included in borg create does not exist
+ # - when some files have changed during the backup (happens a lot on log files)
+ # So we need to filter the output line by line to print these warnings as debug.
+
+ # Warnings are always printed by borg before a long "---…" line.
+ # So we break the output in two along this line.
+ warning_output=$(echo "$output" | sed '/----------------------------/Q') # Before -----…
+ debug_output=$(echo "$output" | sed -n '/----------------------------/,$p') # After -----…
+
+ if [ -n "$warning_output" ] && [ "$filter_warnings" == "yes" ]; then
+ if [ "$warning_if_file_changed_during_backup" == "yes" ]; then
+ file_changes_to_ignore="($file_changes_to_ignore"'|/var/lib/apt/.*|/var/backups/atop/.*|/var/lib/postfix/.*|.*prometheus.*/wal/.*|.*/pg_wal/.*|.*/upload(s)?/.*|.*/(page_)?cache/.*|/var/mail/.*|.*/log(s)?/.*|.*/var/log/.*|.*(log\.json|\.log|\.err|\.wal|\.rrd(4j)?|\.wsp|\.part|\.seq|\.out|\.aof|\.rdb))'
+ else
+ file_changes_to_ignore=""
+ fi
+ non_warning_regex="(Attempting to access a previously unknown unencrypted repository|The repository at location.*was previously located at|Do you want to continue?|No such file or directory|$file_changes_to_ignore: file changed while we backed it up|Using a pure-python msgpack! This will result in lower performance.|/var/backups/drbd/.*scandir.)"
+ echo "$warning_output" | while IFS= read -r line; do
+ if echo "$line" | grep -Eq "$non_warning_regex"; then
+ debug "$line"
+ else
+ warning "$line"
+ fi
+ done
+ debug "$debug_output"
+ info "Successfully finished backing up source"
else
- warning $output
+ warning "$output"
warning "Backing up source finished with warnings."
fi
- else
- error $output
- fatal "Failed backing up source."
+ else # borg error
+ error "$output"
+ fatal "Failed backuping up source. Borg returned exit code ${ret}."
fi
fi
diff --git a/test/borg.bats b/test/borg.bats
index f04874b..e16cf02 100644
--- a/test/borg.bats
+++ b/test/borg.bats
@@ -218,10 +218,10 @@ finish_borg() {
greplog 'Debug: export BORG_CACHE_DIR="/var/cache/borg"$'
}
-@test "check config parameter source/ignore_missing" {
+@test "check config parameter source/filter_warnings" {
delconfig source include
setconfig_repeat source include "$BN_SRCDIR/boot" "$BN_SRCDIR/etc" "$BN_SRCDIR/lib" "$BN_SRCDIR/var" "$BN_SRCDIR/foo"
- setconfig source ignore_missing yes
+ setconfig source filter_warnings yes
setconfig dest archive testarchive
setconfig dest encryption none
setconfig dest compression zstd,16
--
GitLab
From 0bce7106095e50a5ef7ff4d2cb3c8d8c035c94ea Mon Sep 17 00:00:00 2001
From: Guillaume Subiron <maethor@subiron.org>
Date: Thu, 13 Feb 2025 08:53:24 +0100
Subject: [PATCH 2/5] shellcheck borg handler
---
handlers/borg.in | 35 ++++++++++++++++++++---------------
1 file changed, 20 insertions(+), 15 deletions(-)
diff --git a/handlers/borg.in b/handlers/borg.in
index aaf658e..fa309c7 100644
--- a/handlers/borg.in
+++ b/handlers/borg.in
@@ -1,3 +1,6 @@
+#!/bin/bash
+# shellcheck shell=bash
+# shellcheck disable=SC2154
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:
#
@@ -48,7 +51,7 @@ getconf port
getconf directory
# strip trailing /
directory=${directory%/}
-getconf archive {now:%Y-%m-%dT%H:%M:%S}
+getconf archive "{now:%Y-%m-%dT%H:%M:%S}"
getconf compression lz4
getconf encryption none
getconf passphrase
@@ -106,21 +109,23 @@ fi
# check the connection at the source and destination
[ -n "$test" ] || test=0
+# shellcheck disable=SC2235
if [ "$host" != "localhost" ] && ([ "$testconnect" = "yes" ] || [ "${test}" -eq 1 ]); then
- debug "ssh $sshoptions -o PasswordAuthentication=no ${host}${port:+ -p ${port}} -l $user 'echo -n 1'"
- local ret=`ssh $sshoptions -o PasswordAuthentication=no ${host}${port:+ -p ${port}} -l $user 'echo -n 1'`
+ teststr="ssh $sshoptions -o PasswordAuthentication=no ${host}${port:+ -p ${port}} -l $user 'echo -n 1'"
+ debug "$teststr"
+ ret=$(su -c "$teststr")
if [ "$ret" = 1 ]; then
debug "Connected to $host as $user successfully"
else
teststr="borg list $options --show-rc -v $execstr_repository"
debug "$teststr"
- output=`su -c "$teststr" 2>&1`
+ output=$(su -c "$teststr" 2>&1)
if echo "$output" | grep "terminating with success status" \
|| echo "$output" | grep "^\S\+ is not a valid repository." \
|| echo "$output" | grep "^Repository \S\+ does not exist."; then
debug "Connected to $host as $user successfully (forced command)"
else
- error $output
+ error "$output"
fatal "Can't connect to $host as $user."
fi
fi
@@ -132,13 +137,13 @@ if [ "$init" == "yes" ]; then
initstr="borg init $options --encryption=$encryption $execstr_repository"
debug "executing borg init"
debug "$initstr"
- if [ $test = 0 ]; then
- output="`su -c "$initstr" 2>&1`"
+ if [ "$test" = 0 ]; then
+ output="$(su -c "$initstr" 2>&1)"
if [ $? = 2 ]; then
- debug $output
+ debug "$output"
info "Repository was already initialized"
else
- warning $output
+ warning "$output"
warning "Repository has been initialized"
fi
fi
@@ -168,11 +173,11 @@ IFS=$SAVEIFS
set +o noglob
-if [ ! -z $bwlimit ]; then
+if [ -n "$bwlimit" ]; then
execstr="${execstr} --remote-ratelimit=${bwlimit}"
fi
-if [ ! -z "$create_options" ]; then
+if [ -n "$create_options" ]; then
execstr="${execstr} ${create_options}"
fi
@@ -235,14 +240,14 @@ if [ "$prune" == "yes" ]; then
prunestr="borg prune $options $prune_options $execstr_repository"
debug "executing borg prune"
debug "$prunestr"
- if [ $test = 0 ]; then
- output="`su -c "$prunestr" 2>&1`"
+ if [ "$test" = 0 ]; then
+ output="$(su -c "$prunestr" 2>&1)"
ret=$?
if [ $ret = 0 ]; then
- debug $output
+ debug "$output"
info "Removing old backups succeeded."
elif [ $ret = 1 ]; then
- warning $output
+ warning "$output"
warning "Removing old backups finished with warnings."
else
error $output
--
GitLab
From 1d7e00fac63bdca523aef9cfa5901b2076c4db1b Mon Sep 17 00:00:00 2001
From: Guillaume Subiron <maethor@subiron.org>
Date: Thu, 13 Feb 2025 08:55:41 +0100
Subject: [PATCH 3/5] borg: allow to export borg list and borg info for
monitoring
---
examples/example.borg | 16 +++++++++++++++
handlers/borg.in | 47 +++++++++++++++++++++++++++++++++++++++++--
2 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/examples/example.borg b/examples/example.borg
index 4fced66..9544a94 100644
--- a/examples/example.borg
+++ b/examples/example.borg
@@ -46,6 +46,22 @@
## Default:
# bwlimit = 0
+## export "borg info last_archive" to a given file
+## this is usefull for monitoring without using borg
+##
+## Example:
+# borginfo = /var/backups/borginfo.json
+## Default:
+# borginfo =
+
+## export "borg list repository" to a given file
+## this is usefull for monitoring without using borg
+##
+## Example
+# borglist = /var/backups/borglist.json
+## Default:
+# borglist =
+
######################################################
## source section
## (where the files to be backed up are coming from)
diff --git a/handlers/borg.in b/handlers/borg.in
index fa309c7..0fc72b6 100644
--- a/handlers/borg.in
+++ b/handlers/borg.in
@@ -31,6 +31,9 @@ getconf nicelevel 0
getconf ionicelevel
getconf bwlimit
+getconf borginfo
+getconf borglist
+
setsection source
getconf init yes
getconf include
@@ -250,8 +253,48 @@ if [ "$prune" == "yes" ]; then
warning "$output"
warning "Removing old backups finished with warnings."
else
- error $output
- fatal "Failed removing old backups."
+ info "$output"
+ warning "Failed removing old backups. Borg returned exit code ${ret}."
+ fi
+ fi
+fi
+
+### WRITE STATS FILES ###
+
+if [ -n "$borginfo" ]; then
+ mkdir -p "$(dirname "$borginfo")"
+
+ infostr="borg info $execstr_repository --last 1 --json > $borginfo"
+
+ debug "$infostr"
+ if [ "$test" = 0 ]; then
+ output=$(su -c "$infostr" 2>&1)
+ ret=$?
+ if [ $ret = 0 ]; then
+ debug "$output"
+ info "Successfully writing borg info to $borginfo"
+ else
+ info "$output"
+ error "Failed to write borg info to $borginfo"
+ fi
+ fi
+fi
+
+if [ -n "$borglist" ]; then
+ mkdir -p "$(dirname "$borglist")"
+
+ infostr="borg list $execstr_repository --json > $borglist"
+
+ debug "$infostr"
+ if [ "$test" = 0 ]; then
+ output=$(su -c "$infostr" 2>&1)
+ ret=$?
+ if [ $ret = 0 ]; then
+ debug "$output"
+ info "Successfully writing borg list to $borglist"
+ else
+ info "$output"
+ error "Failed to write borg list to $borglist"
fi
fi
fi
--
GitLab
From 1cc50d051ae9eec39134243977716e50f4c6a442 Mon Sep 17 00:00:00 2001
From: Guillaume Subiron <maethor@subiron.org>
Date: Thu, 13 Feb 2025 08:58:50 +0100
Subject: [PATCH 4/5] borg: add keephourly, keepdaily, keepweekly and
keepmonthly variables
---
examples/example.borg | 10 ++++++++++
handlers/borg.in | 16 ++++++++++++++++
2 files changed, 26 insertions(+)
diff --git a/examples/example.borg b/examples/example.borg
index 9544a94..e7425ec 100644
--- a/examples/example.borg
+++ b/examples/example.borg
@@ -130,6 +130,16 @@ exclude = /var/lib/mysql
## Default:
# keep = 30d
+## define hourly, daily, weekly and monthly retention for the "borg prune" operation.
+##
+## theses options will be ignored if set to 0
+##
+## Default:
+## keephourly = 0
+## keepdaily = 0
+## keepweekly = 0
+## keepmonthly = 0
+
## define extra command-line options for the "borg prune" operation.
##
## Example:
diff --git a/handlers/borg.in b/handlers/borg.in
index 0fc72b6..06bde87 100644
--- a/handlers/borg.in
+++ b/handlers/borg.in
@@ -41,6 +41,10 @@ getconf exclude
getconf create_options
getconf prune yes
getconf keep 30d
+getconf keephourly 0
+getconf keepdaily 0
+getconf keepweekly 0
+getconf keepmonthly 0
getconf prune_options
getconf cache_directory
getconf filter_warnings yes
@@ -240,6 +244,18 @@ if [ "$prune" == "yes" ]; then
if [ ! "$keep" == "0" ]; then
prune_options="${prune_options} --keep-within=${keep}"
fi
+ if [ ! "$keephourly" == "0" ]; then
+ prune_options="${prune_options} --keep-hourly=${keephourly}"
+ fi
+ if [ ! "$keepdaily" == "0" ]; then
+ prune_options="${prune_options} --keep-daily=${keepdaily}"
+ fi
+ if [ ! "$keepweekly" == "0" ]; then
+ prune_options="${prune_options} --keep-weekly=${keepweekly}"
+ fi
+ if [ ! "$keepmonthly" == "0" ]; then
+ prune_options="${prune_options} --keep-monthly=${keepmonthly}"
+ fi
prunestr="borg prune $options $prune_options $execstr_repository"
debug "executing borg prune"
debug "$prunestr"
--
GitLab
From b37edd26ab246a73f79d0dc40651ff84f62dc7c0 Mon Sep 17 00:00:00 2001
From: Guillaume Subiron <maethor@subiron.org>
Date: Thu, 13 Feb 2025 08:59:33 +0100
Subject: [PATCH 5/5] borg: run borg compact after borg prune when borg version
> 1.2
---
handlers/borg.in | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/handlers/borg.in b/handlers/borg.in
index 06bde87..d4de31e 100644
--- a/handlers/borg.in
+++ b/handlers/borg.in
@@ -265,6 +265,21 @@ if [ "$prune" == "yes" ]; then
if [ $ret = 0 ]; then
debug "$output"
info "Removing old backups succeeded."
+
+ if [[ "$(borg --version)" > "borg 1.2" ]] ; then
+ compactstr="borg compact $execstr_repository"
+
+ debug "$compactstr"
+ output="$(su -c "$compactstr" 2>&1)"
+ ret=$?
+ if [ $ret = 0 ]; then
+ debug "$output"
+ info "Compacting borg repository succeeded."
+ else
+ info "$output"
+ warning "Compacting borg repository failed. Borg returned exit code ${ret}."
+ fi
+ fi
elif [ $ret = 1 ]; then
warning "$output"
warning "Removing old backups finished with warnings."
--
GitLab