diff --git a/AUTHORS b/AUTHORS
index 1319688617ea3d7fb18036b304adf404d433dbba..4e7299bed3469e8c014694159c29312b8fc8ffb7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -43,3 +43,4 @@ aihtdikh -- Allow 'when = XXX' with spaces in .sh files.
Chris Lamb <lamby@debian.org> -- rdiff.helper bugfix
Yuval Kogman <nothingmuch@woobling.org> -- RackSpace's CloudFiles support for duplicity
exobuzz - mysql bugfixes
+Glennie Vignarajah <glennie@glennie.fr> -- mysql bugfix
diff --git a/ChangeLog b/ChangeLog
index 1e20e003a6589bb4ff88b4555c25c35121a444b3..faf06dd56ad79ba22ae4474cd5ccf51efcd2f5bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,12 @@
version 1.0 -- UNRELEASED
+ handler changes
+ mysql:
+ . Use --skip-events when backing up the performance_schema database.
+ (Closes: #673572)
+ backupninja changes
+ . Make it clear what lockfile could not be acquired, if any.
+
+version 1.0-rc1 -- May 15, 2012
handler changes
dup:
. Make the .dup generated by ninjahelper more consistent with
@@ -10,6 +18,8 @@ version 1.0 -- UNRELEASED
(Closes: #657201)
. Report failure output at error loglevel so that it is emailed
(Closes: #536858)
+ maildir:
+ . Remove 'loadlimit' parameter - it is not used anywhere.
mysql:
. Don't attempt to dump performance_schema database (Redmine#3741).
pgsql:
@@ -22,15 +32,29 @@ version 1.0 -- UNRELEASED
. Fix metadata rotation.
. Allow disabling rotation or setting 2 days as minimum for backup
increments in rsync short format (Redmine#2107).
- . Use fatal instead of custom "exit 1" (Redmine#3721).
. Abort on rsync error (Redmine#3692).
. Cleanup orphaned metadata (Redmine#3727).
- . Use the backup start time and not the time the backup was finished
- (Closes: #654192)
+ . Use the backup start time and not the time the backup was finished.
+ (Closes: #654192).
+ . Use 'debug', 'fatal' and 'warning' functions instead of regular echo
+ and exit (Redmine#3840, Redmine#3721).
+ . Quoting $starttime (Redmine#3868).
+ . Validate created date on long_rotation to avoid too many arguments
+ at comparison (Redmine#3868).
+ . Quoting $exclude and $excludes and avoiding a for loop on $exclude
+ to not expand wildcards in beforehand (Redmine#3882).
+ . Quote excludes (Redmine#3882).
+ . Changing remaining 'exit' to 'fatal' at rsync handler (Redmine#3721).
+ . Removing duplicated locking support (Redmine#3838).
+ . Documenting rotation parameters at example.rsync (Redmine#3891).
+ . Ensure that a non-zero rsync exit status is caught (Redmine#3892).
build system changes
. Workaround automake sanity check that would prevent us from
installing lib/* into lib/backupninja/. Where else are be supposed
to install such files anyway?
+ . Have "make dist" ship handlers/*.in instead of make results.
+ . Have "make dist" ship the FAQ.
+ . Install handlers as pkgdata_DATA, instead of their .in files.
documentation changes
. Document what features available to .sh jobs (Redmine #1558).
diff --git a/Makefile.am b/Makefile.am
index 3c50ad9f422ee4089efc99f5f519e2ef4c390ccd..6e8d1709609b1f44f1ae6b05ad6549d04bad591b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,7 @@
# vi: noexpandtab softtabstop=0
## Process this file with automake to produce Makefile.in
-EXTRA_DIST = README COPYING AUTHORS INSTALL NEWS ChangeLog \
+EXTRA_DIST = FAQ README COPYING AUTHORS INSTALL NEWS ChangeLog \
backupninja.spec backupninja.spec.in autogen.sh
SUBDIRS = etc examples handlers lib man src
diff --git a/configure.in b/configure.in
index 2d3068794c9f639fb9c867b2abe21647921d9ba5..3cddf61609cb394ffbe293324672ebf3ed45fc28 100644
--- a/configure.in
+++ b/configure.in
@@ -3,7 +3,7 @@
# The maintainer mode is causing me grief with newest versions of autotools
#AM_MAINTAINER_MODE
-AC_INIT([backupninja],[0.9.10],[backupninja@lists.riseup.net])
+AC_INIT([backupninja],[1.0-rc1],[backupninja@lists.riseup.net])
AC_CONFIG_SRCDIR([src/backupninja.in])
AM_INIT_AUTOMAKE
diff --git a/examples/example.rsync b/examples/example.rsync
index a2795db63f32a4d52f3416cc664f47f28526cf46..80365ae6805bc212759179cdb667b7161e1645cf 100644
--- a/examples/example.rsync
+++ b/examples/example.rsync
@@ -36,9 +36,33 @@ backupdir = myserver
#tmp = /tmp
# specify backup storage format: short, long or mirror (i.e, no rotations)
+#
+# In the short format, incremental backups are rotated every day the handler
+# runs an by a finite number of times (backup.0, backup.1, backup.1, etc), so
+# if you want to have incremental backups for longer periods (like months) you
+# have to configure rotations for 30 or more using the "days" parameter at the
+# [general] section in the handler config.
+#
+# The short format is better described here:
+# http://www.mikerubel.org/computers/rsync_snapshots/#Incremental
+#
+# The long format is inspired by the maildir handler and allows keeping backups
+# of longer periods (weeks and months) using less rotations as it stores
+# the increments in folders like daily.1, weekly.1, monthly.1 and has three
+# rotation parameters:
+#
+# keepdaily = number of daily backup increments
+# keepweekly = number of weekly backup increments
+# keepmonthly = number of monthly backup increments
+#
format = short
# for short storage format, specify the number of backup increments (min = 2, set to 1 or less to disable)
+#
+# Note that setting days = 0 is almost the same as using format = mirror except
+# that with the days config your backup gets a .0 suffix at the destination
+# folder, making it easier to turn it later to an incremental backup.
+#
days = 7
# for long storage format, specify the number of daily backup increments
@@ -50,11 +74,6 @@ days = 7
# for long storage format, specify the number of monthly backup increments
#keepmonthly = 1
-# use this if you need a lockfile to be kept during backup execution
-# this is an useful feature in case you have some tasks that should
-# know if the backup is running or not
-#lockfile =
-
# rsync command nice level
#nicelevel = 0
diff --git a/handlers/Makefile.am b/handlers/Makefile.am
index 54155e89214f87e224216c2a8c1e7e48e4c1878a..bad53bbe857b9ffb5d833033e9b5b1ca50a7e9ff 100644
--- a/handlers/Makefile.am
+++ b/handlers/Makefile.am
@@ -3,16 +3,20 @@ HANDLERS = dup dup.helper maildir makecd \
makecd.helper mysql mysql.helper pgsql pgsql.helper rdiff \
rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper
+DIST_HANDLERS = dup.in dup.helper.in maildir.in makecd.in \
+ makecd.helper.in mysql.in mysql.helper.in pgsql.in pgsql.helper.in rdiff.in \
+ rdiff.helper.in rsync.in sh.in svn.in sys.in sys.helper.in trac.in tar.in tar.helper.in wget
+
CLEANFILES = $(HANDLERS)
-EXTRA_DIST = Makefile.am $(HANDLERS)
+EXTRA_DIST = Makefile.am $(DIST_HANDLERS)
edit = sed \
-e "s,@BASH\@,$(BASH),g" \
-e "s,@AWK\@,$(AWK),g" \
-e "s,@SED\@,$(SED),g"
-dist_pkgdata_DATA = $(HANDLERS)
+pkgdata_DATA = $(HANDLERS)
dup: $(srcdir)/dup.in
rm -f dup
diff --git a/handlers/mysql.in b/handlers/mysql.in
index 65deebbc62ac840a6c10c15e263daf37141768a7..6ade49e88e882a495dcb0f02de0f77b750d20ee4 100644
--- a/handlers/mysql.in
+++ b/handlers/mysql.in
@@ -256,10 +256,15 @@ then
for db in $databases
do
DUMP_BASE="$MYSQLDUMP $defaultsfile $sqldumpoptions"
- if [ "$db" = "information_schema" ] || [ "$db" = "performance_schema" ]
- then
- DUMP_BASE="${DUMP_BASE} --skip-lock-tables"
- fi
+
+ case "$db" in
+ information_schema)
+ DUMP_BASE="${DUMP_BASE} --skip-lock-tables"
+ ;;
+ performance_schema)
+ DUMP_BASE="${DUMP_BASE} --skip-lock-tables --skip-events"
+ ;;
+ esac
# Dumping structure and data
DUMP="$DUMP_BASE $ignore $db"
diff --git a/handlers/rsync.in b/handlers/rsync.in
index 884a885f96fa6c20ea9d90a27744efa19a8e8dd4..386255ea865b990fc8e0919b7ed160e37e0533ba 100644
--- a/handlers/rsync.in
+++ b/handlers/rsync.in
@@ -39,7 +39,6 @@
# keepdaily = for long storage format, specify the number of daily backup increments
# keepweekly = for long storage format, specify the number of weekly backup increments
# keepmonthly = for long storage format, specify the number of monthly backup increments
-# lockfile = lockfile to be kept during backup execution
# nicelevel = rsync command nice level
# enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
# tmp = temp folder
@@ -129,7 +128,6 @@ function eval_config {
getconf keepdaily 5
getconf keepweekly 3
getconf keepmonthly 1
- getconf lockfile
getconf nicelevel 0
getconf enable_mv_timestamp_bug no
getconf tmp /tmp
@@ -277,9 +275,7 @@ function eval_config {
mv=move_files
fi
- for path in $exclude; do
- excludes="$excludes --exclude=$path"
- done
+ excludes=`echo "$exclude" | @SED@ -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"`
}
@@ -391,8 +387,8 @@ function rotate_long {
local metadata
if [ ! -d "$backuproot" ]; then
- echo "Debug: skipping rotate of $backuproot as it doesn't exist."
- exit
+ warning "Skipping rotate of $backuproot as it doesn't exist."
+ return
fi
for rottype in daily weekly monthly; do
@@ -405,12 +401,12 @@ function rotate_long {
echo "Debug: $dir.1 does not exist, skipping."
continue 1
elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then
- echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
+ warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
continue 1
fi
# Rotate the current list of backups, if we can.
- oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+ oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
[ "$oldest" == "" ] && oldest=0
for (( i=$oldest; i > 0; i-- )); do
if [ -d $dir.$i ]; then
@@ -421,11 +417,16 @@ function rotate_long {
else
created=0
fi
+ # Validate created date
+ if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then
+ warning "Invalid metadata $created. Skipping rotation."
+ break
+ fi
cutoff_time=$(( now - (seconds*(i-1)) ))
if [ ! $created -gt $cutoff_time ]; then
next=$(( i + 1 ))
if [ ! -d $dir.$next ]; then
- echo "Debug: $rottype.$i --> $rottype.$next"
+ debug "$rottype.$i --> $rottype.$next"
$nice mv $dir.$i $dir.$next
mkdir -p $metadata.$next
date +%c%n%s > $metadata.$next/rotated
@@ -433,10 +434,10 @@ function rotate_long {
$nice mv $metadata.$i/created $metadata.$next
fi
else
- echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
+ debug "skipping rotation of $dir.$i because $dir.$next already exists."
fi
else
- echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
+ debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
fi
fi
done
@@ -444,7 +445,7 @@ function rotate_long {
max=$((keepdaily+1))
if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
- echo "Debug: daily.$max --> weekly.1"
+ debug "daily.$max --> weekly.1"
$nice mv $backuproot/daily.$max $backuproot/weekly.1
mkdir -p $backuproot/metadata/weekly.1
date +%c%n%s > $backuproot/metadata/weekly.1/rotated
@@ -455,7 +456,7 @@ function rotate_long {
max=$((keepweekly+1))
if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
- echo "Debug: weekly.$max --> monthly.1"
+ debug "weekly.$max --> monthly.1"
$nice mv $backuproot/weekly.$max $backuproot/monthly.1
mkdir -p $backuproot/metadata/monthly.1
date +%c%n%s > $backuproot/metadata/monthly.1/rotated
@@ -467,16 +468,16 @@ function rotate_long {
for rottype in daily weekly monthly; do
max=$((keep${rottype}+1))
dir="$backuproot/$rottype"
- oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+ oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
[ "$oldest" == "" ] && oldest=0
# if we've rotated the last backup off the stack, remove it.
for (( i=$oldest; i >= $max; i-- )); do
if [ -d $dir.$i ]; then
if [ -d $backuproot/rotate.tmp ]; then
- echo "Debug: removing rotate.tmp"
+ debug "removing rotate.tmp"
$nice rm -rf $backuproot/rotate.tmp
fi
- echo "Debug: moving $rottype.$i to rotate.tmp"
+ debug "moving $rottype.$i to rotate.tmp"
$nice mv $dir.$i $backuproot/rotate.tmp
fi
done
@@ -509,7 +510,7 @@ function rotate_long_remote {
now=\`date +%s\`
if [ ! -d "$backuproot" ]; then
- echo "Debug: skipping rotate of $backuproot as it doesn't exist."
+ echo "Fatal: skipping rotate of $backuproot as it doesn't exist."
exit
fi
@@ -528,7 +529,7 @@ function rotate_long_remote {
fi
# Rotate the current list of backups, if we can.
- oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
+ oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
[ "\$oldest" == "" ] && oldest=0
for (( i=\$oldest; i > 0; i-- )); do
if [ -d \$dir.\$i ]; then
@@ -539,6 +540,11 @@ function rotate_long_remote {
else
created=0
fi
+ # Validate created date
+ if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then
+ echo "Warning: Invalid metadata \$created. Skipping rotation."
+ break
+ fi
cutoff_time=\$(( now - (seconds*(i-1)) ))
if [ ! \$created -gt \$cutoff_time ]; then
next=\$(( i + 1 ))
@@ -585,7 +591,7 @@ function rotate_long_remote {
for rottype in daily weekly monthly; do
max=\$((keep\${rottype}+1))
dir="$backuproot/\$rottype"
- oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
+ oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
[ "\$oldest" == "" ] && oldest=0
# if we've rotated the last backup off the stack, remove it.
for (( i=\$oldest; i >= \$max; i-- )); do
@@ -830,7 +836,7 @@ function update_metadata {
# finished, otherwise daily rotations might not take place.
# If we used backup end time, in the next handler run
# we might not have $now - $created >= 24:00
- echo $starttime > $metadata/created
+ echo "$starttime" > $metadata/created
$touch $backupdir/$SECTION/$suffix
else
folder="`echo $dest_path | cut -d : -f 2`"
@@ -844,7 +850,7 @@ function update_metadata {
# finished, otherwise daily rotations might not take place.
# If we used backup end time, in the next handler run
# we might not have $now - $created >= 24:00
- echo $starttime > $metadata/created
+ echo "$starttime" > $metadata/created
##### END REMOTE SCRIPT #######
EOF
) | (while read a; do passthru $a; done)
@@ -875,45 +881,6 @@ function test_connect {
}
-function set_lockfile {
-
- if [ ! -z "$lockfile" ]; then
- mkdir -p `dirname $lockfile`
- if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then
- trap 'unset_lockfile' INT TERM EXIT
- else
- fatal "Could not create lockfile $lockfile, exiting"
- fi
- fi
-
-}
-
-function unset_lockfile {
-
- if [ ! -z "$lockfile" ]; then
- $rm -f $lockfile || warning "Could not remove lockfile $lockfile"
- fi
-
-}
-
-function check_lockfile {
-
- local pid process
-
- if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then
- pid="`cat $lockfile`"
- process="`ps --no-headers -o comm $pid`"
- if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then
- info "Another backup is running for $lockfile, skipping run"
- exit
- else
- info "Found old lockfile $lockfile, removing it"
- unset_lockfile
- fi
- fi
-
-}
-
function set_filelist {
filelist_flag=""
@@ -1100,18 +1067,38 @@ function end_mux {
}
+function set_pipefail {
+
+ # Save initial pipefail status for later restoration
+ if echo "$SHELLOPTS" | grep -q ":pipefail"; then
+ pipefail="-o"
+ else
+ pipefail="+o"
+ fi
+
+ # Ensure that a non-zero rsync exit status is caught by our handler
+ set -o pipefail
+
+}
+
+function restore_pipefail {
+
+ if [ ! -z "$pipefail" ]; then
+ set $pipefail pipefail
+ fi
+
+}
+
# the backup procedure
eval_config
-check_lockfile
-set_lockfile
set_rsync_options
start_mux
stop_services
mount_rw
starttime="`date +%c%n%s`"
-echo "Starting backup at `echo $starttime | head -n 1`" >> $log
+echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log
for SECTION in $include; do
@@ -1122,13 +1109,15 @@ for SECTION in $include; do
set_dest
info "Syncing $SECTION on $dest_path..."
- debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path
- $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log
+ debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path
+ set_pipefail
+ $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log
if [ "$?" != "0" ]; then
fatal "Rsync error when trying to transfer $SECTION"
fi
+ restore_pipefail
update_metadata
done
@@ -1136,7 +1125,6 @@ done
mount_ro
run_fsck
start_services
-unset_lockfile
end_mux
echo "Finnishing backup at `date`" >> $log
diff --git a/src/backupninja.in b/src/backupninja.in
index 040082844ee0dbbc17888c1d23496af8d130a11f..0f74906d843b147009a4d20c44ec0601fc5c2bad 100755
--- a/src/backupninja.in
+++ b/src/backupninja.in
@@ -349,7 +349,7 @@ function process_action() {
else
# a backup is probably ongoing already, so display an error message
- debug "failed to acquire lock"
+ debug "failed to acquire lock $lockfile"
echo "Fatal: Could not acquire lock $lockfile. A backup is probably already running for $file." >>$bufferfile
fi
} 200> $lockfile