diff options
| -rw-r--r-- | AUTHORS | 1 | ||||
| -rw-r--r-- | ChangeLog | 30 | ||||
| -rw-r--r-- | Makefile.am | 2 | ||||
| -rw-r--r-- | configure.in | 2 | ||||
| -rw-r--r-- | examples/example.rsync | 29 | ||||
| -rw-r--r-- | handlers/Makefile.am | 8 | ||||
| -rw-r--r-- | handlers/mysql.in | 13 | ||||
| -rw-r--r-- | handlers/rsync.in | 122 | ||||
| -rwxr-xr-x | src/backupninja.in | 2 |
9 files changed, 125 insertions, 84 deletions
@@ -43,3 +43,4 @@ aihtdikh -- Allow 'when = XXX' with spaces in .sh files. Chris Lamb <lamby@debian.org> -- rdiff.helper bugfix Yuval Kogman <nothingmuch@woobling.org> -- RackSpace's CloudFiles support for duplicity exobuzz - mysql bugfixes +Glennie Vignarajah <glennie@glennie.fr> -- mysql bugfix @@ -1,5 +1,13 @@ version 1.0 -- UNRELEASED handler changes + mysql: + . Use --skip-events when backing up the performance_schema database. + (Closes: #673572) + backupninja changes + . Make it clear what lockfile could not be acquired, if any. + +version 1.0-rc1 -- May 15, 2012 + handler changes dup: . Make the .dup generated by ninjahelper more consistent with example.dup. @@ -10,6 +18,8 @@ version 1.0 -- UNRELEASED (Closes: #657201) . Report failure output at error loglevel so that it is emailed (Closes: #536858) + maildir: + . Remove 'loadlimit' parameter - it is not used anywhere. mysql: . Don't attempt to dump performance_schema database (Redmine#3741). pgsql: @@ -22,15 +32,29 @@ version 1.0 -- UNRELEASED . Fix metadata rotation. . Allow disabling rotation or setting 2 days as minimum for backup increments in rsync short format (Redmine#2107). - . Use fatal instead of custom "exit 1" (Redmine#3721). . Abort on rsync error (Redmine#3692). . Cleanup orphaned metadata (Redmine#3727). - . Use the backup start time and not the time the backup was finished - (Closes: #654192) + . Use the backup start time and not the time the backup was finished. + (Closes: #654192). + . Use 'debug', 'fatal' and 'warning' functions instead of regular echo + and exit (Redmine#3840, Redmine#3721). + . Quoting $starttime (Redmine#3868). + . Validate created date on long_rotation to avoid too many arguments + at comparison (Redmine#3868). + . Quoting $exclude and $excludes and avoiding a for loop on $exclude + to not expand wildcards in beforehand (Redmine#3882). + . Quote excludes (Redmine#3882). + . Changing remaining 'exit' to 'fatal' at rsync handler (Redmine#3721). + . Removing duplicated locking support (Redmine#3838). + . Documenting rotation parameters at example.rsync (Redmine#3891). + . Ensure that a non-zero rsync exit status is caught (Redmine#3892). build system changes . Workaround automake sanity check that would prevent us from installing lib/* into lib/backupninja/. Where else are be supposed to install such files anyway? + . Have "make dist" ship handlers/*.in instead of make results. + . Have "make dist" ship the FAQ. + . Install handlers as pkgdata_DATA, instead of their .in files. documentation changes . Document what features available to .sh jobs (Redmine #1558). diff --git a/Makefile.am b/Makefile.am index 3c50ad9..6e8d170 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ # vi: noexpandtab softtabstop=0 ## Process this file with automake to produce Makefile.in -EXTRA_DIST = README COPYING AUTHORS INSTALL NEWS ChangeLog \ +EXTRA_DIST = FAQ README COPYING AUTHORS INSTALL NEWS ChangeLog \ backupninja.spec backupninja.spec.in autogen.sh SUBDIRS = etc examples handlers lib man src diff --git a/configure.in b/configure.in index 2d30687..3cddf61 100644 --- a/configure.in +++ b/configure.in @@ -3,7 +3,7 @@ # The maintainer mode is causing me grief with newest versions of autotools #AM_MAINTAINER_MODE -AC_INIT([backupninja],[0.9.10],[backupninja@lists.riseup.net]) +AC_INIT([backupninja],[1.0-rc1],[backupninja@lists.riseup.net]) AC_CONFIG_SRCDIR([src/backupninja.in]) AM_INIT_AUTOMAKE diff --git a/examples/example.rsync b/examples/example.rsync index a2795db..80365ae 100644 --- a/examples/example.rsync +++ b/examples/example.rsync @@ -36,9 +36,33 @@ backupdir = myserver #tmp = /tmp # specify backup storage format: short, long or mirror (i.e, no rotations) +# +# In the short format, incremental backups are rotated every day the handler +# runs an by a finite number of times (backup.0, backup.1, backup.1, etc), so +# if you want to have incremental backups for longer periods (like months) you +# have to configure rotations for 30 or more using the "days" parameter at the +# [general] section in the handler config. +# +# The short format is better described here: +# http://www.mikerubel.org/computers/rsync_snapshots/#Incremental +# +# The long format is inspired by the maildir handler and allows keeping backups +# of longer periods (weeks and months) using less rotations as it stores +# the increments in folders like daily.1, weekly.1, monthly.1 and has three +# rotation parameters: +# +# keepdaily = number of daily backup increments +# keepweekly = number of weekly backup increments +# keepmonthly = number of monthly backup increments +# format = short # for short storage format, specify the number of backup increments (min = 2, set to 1 or less to disable) +# +# Note that setting days = 0 is almost the same as using format = mirror except +# that with the days config your backup gets a .0 suffix at the destination +# folder, making it easier to turn it later to an incremental backup. +# days = 7 # for long storage format, specify the number of daily backup increments @@ -50,11 +74,6 @@ days = 7 # for long storage format, specify the number of monthly backup increments #keepmonthly = 1 -# use this if you need a lockfile to be kept during backup execution -# this is an useful feature in case you have some tasks that should -# know if the backup is running or not -#lockfile = - # rsync command nice level #nicelevel = 0 diff --git a/handlers/Makefile.am b/handlers/Makefile.am index 54155e8..bad53bb 100644 --- a/handlers/Makefile.am +++ b/handlers/Makefile.am @@ -3,16 +3,20 @@ HANDLERS = dup dup.helper maildir makecd \ makecd.helper mysql mysql.helper pgsql pgsql.helper rdiff \ rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper +DIST_HANDLERS = dup.in dup.helper.in maildir.in makecd.in \ + makecd.helper.in mysql.in mysql.helper.in pgsql.in pgsql.helper.in rdiff.in \ + rdiff.helper.in rsync.in sh.in svn.in sys.in sys.helper.in trac.in tar.in tar.helper.in wget + CLEANFILES = $(HANDLERS) -EXTRA_DIST = Makefile.am $(HANDLERS) +EXTRA_DIST = Makefile.am $(DIST_HANDLERS) edit = sed \ -e "s,@BASH\@,$(BASH),g" \ -e "s,@AWK\@,$(AWK),g" \ -e "s,@SED\@,$(SED),g" -dist_pkgdata_DATA = $(HANDLERS) +pkgdata_DATA = $(HANDLERS) dup: $(srcdir)/dup.in rm -f dup diff --git a/handlers/mysql.in b/handlers/mysql.in index 65deebb..6ade49e 100644 --- a/handlers/mysql.in +++ b/handlers/mysql.in @@ -256,10 +256,15 @@ then for db in $databases do DUMP_BASE="$MYSQLDUMP $defaultsfile $sqldumpoptions" - if [ "$db" = "information_schema" ] || [ "$db" = "performance_schema" ] - then - DUMP_BASE="${DUMP_BASE} --skip-lock-tables" - fi + + case "$db" in + information_schema) + DUMP_BASE="${DUMP_BASE} --skip-lock-tables" + ;; + performance_schema) + DUMP_BASE="${DUMP_BASE} --skip-lock-tables --skip-events" + ;; + esac # Dumping structure and data DUMP="$DUMP_BASE $ignore $db" diff --git a/handlers/rsync.in b/handlers/rsync.in index 884a885..386255e 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -39,7 +39,6 @@ # keepdaily = for long storage format, specify the number of daily backup increments # keepweekly = for long storage format, specify the number of weekly backup increments # keepmonthly = for long storage format, specify the number of monthly backup increments -# lockfile = lockfile to be kept during backup execution # nicelevel = rsync command nice level # enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly # tmp = temp folder @@ -129,7 +128,6 @@ function eval_config { getconf keepdaily 5 getconf keepweekly 3 getconf keepmonthly 1 - getconf lockfile getconf nicelevel 0 getconf enable_mv_timestamp_bug no getconf tmp /tmp @@ -277,9 +275,7 @@ function eval_config { mv=move_files fi - for path in $exclude; do - excludes="$excludes --exclude=$path" - done + excludes=`echo "$exclude" | @SED@ -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"` } @@ -391,8 +387,8 @@ function rotate_long { local metadata if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." - exit + warning "Skipping rotate of $backuproot as it doesn't exist." + return fi for rottype in daily weekly monthly; do @@ -405,12 +401,12 @@ function rotate_long { echo "Debug: $dir.1 does not exist, skipping." continue 1 elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then - echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." + warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." continue 1 fi # Rotate the current list of backups, if we can. - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 for (( i=$oldest; i > 0; i-- )); do if [ -d $dir.$i ]; then @@ -421,11 +417,16 @@ function rotate_long { else created=0 fi + # Validate created date + if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then + warning "Invalid metadata $created. Skipping rotation." + break + fi cutoff_time=$(( now - (seconds*(i-1)) )) if [ ! $created -gt $cutoff_time ]; then next=$(( i + 1 )) if [ ! -d $dir.$next ]; then - echo "Debug: $rottype.$i --> $rottype.$next" + debug "$rottype.$i --> $rottype.$next" $nice mv $dir.$i $dir.$next mkdir -p $metadata.$next date +%c%n%s > $metadata.$next/rotated @@ -433,10 +434,10 @@ function rotate_long { $nice mv $metadata.$i/created $metadata.$next fi else - echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." + debug "skipping rotation of $dir.$i because $dir.$next already exists." fi else - echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." + debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." fi fi done @@ -444,7 +445,7 @@ function rotate_long { max=$((keepdaily+1)) if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then - echo "Debug: daily.$max --> weekly.1" + debug "daily.$max --> weekly.1" $nice mv $backuproot/daily.$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated @@ -455,7 +456,7 @@ function rotate_long { max=$((keepweekly+1)) if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then - echo "Debug: weekly.$max --> monthly.1" + debug "weekly.$max --> monthly.1" $nice mv $backuproot/weekly.$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated @@ -467,16 +468,16 @@ function rotate_long { for rottype in daily weekly monthly; do max=$((keep${rottype}+1)) dir="$backuproot/$rottype" - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=$oldest; i >= $max; i-- )); do if [ -d $dir.$i ]; then if [ -d $backuproot/rotate.tmp ]; then - echo "Debug: removing rotate.tmp" + debug "removing rotate.tmp" $nice rm -rf $backuproot/rotate.tmp fi - echo "Debug: moving $rottype.$i to rotate.tmp" + debug "moving $rottype.$i to rotate.tmp" $nice mv $dir.$i $backuproot/rotate.tmp fi done @@ -509,7 +510,7 @@ function rotate_long_remote { now=\`date +%s\` if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." + echo "Fatal: skipping rotate of $backuproot as it doesn't exist." exit fi @@ -528,7 +529,7 @@ function rotate_long_remote { fi # Rotate the current list of backups, if we can. - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 for (( i=\$oldest; i > 0; i-- )); do if [ -d \$dir.\$i ]; then @@ -539,6 +540,11 @@ function rotate_long_remote { else created=0 fi + # Validate created date + if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then + echo "Warning: Invalid metadata \$created. Skipping rotation." + break + fi cutoff_time=\$(( now - (seconds*(i-1)) )) if [ ! \$created -gt \$cutoff_time ]; then next=\$(( i + 1 )) @@ -585,7 +591,7 @@ function rotate_long_remote { for rottype in daily weekly monthly; do max=\$((keep\${rottype}+1)) dir="$backuproot/\$rottype" - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=\$oldest; i >= \$max; i-- )); do @@ -830,7 +836,7 @@ function update_metadata { # finished, otherwise daily rotations might not take place. # If we used backup end time, in the next handler run # we might not have $now - $created >= 24:00 - echo $starttime > $metadata/created + echo "$starttime" > $metadata/created $touch $backupdir/$SECTION/$suffix else folder="`echo $dest_path | cut -d : -f 2`" @@ -844,7 +850,7 @@ function update_metadata { # finished, otherwise daily rotations might not take place. # If we used backup end time, in the next handler run # we might not have $now - $created >= 24:00 - echo $starttime > $metadata/created + echo "$starttime" > $metadata/created ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -875,45 +881,6 @@ function test_connect { } -function set_lockfile { - - if [ ! -z "$lockfile" ]; then - mkdir -p `dirname $lockfile` - if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then - trap 'unset_lockfile' INT TERM EXIT - else - fatal "Could not create lockfile $lockfile, exiting" - fi - fi - -} - -function unset_lockfile { - - if [ ! -z "$lockfile" ]; then - $rm -f $lockfile || warning "Could not remove lockfile $lockfile" - fi - -} - -function check_lockfile { - - local pid process - - if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then - pid="`cat $lockfile`" - process="`ps --no-headers -o comm $pid`" - if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then - info "Another backup is running for $lockfile, skipping run" - exit - else - info "Found old lockfile $lockfile, removing it" - unset_lockfile - fi - fi - -} - function set_filelist { filelist_flag="" @@ -1100,18 +1067,38 @@ function end_mux { } +function set_pipefail { + + # Save initial pipefail status for later restoration + if echo "$SHELLOPTS" | grep -q ":pipefail"; then + pipefail="-o" + else + pipefail="+o" + fi + + # Ensure that a non-zero rsync exit status is caught by our handler + set -o pipefail + +} + +function restore_pipefail { + + if [ ! -z "$pipefail" ]; then + set $pipefail pipefail + fi + +} + # the backup procedure eval_config -check_lockfile -set_lockfile set_rsync_options start_mux stop_services mount_rw starttime="`date +%c%n%s`" -echo "Starting backup at `echo $starttime | head -n 1`" >> $log +echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log for SECTION in $include; do @@ -1122,13 +1109,15 @@ for SECTION in $include; do set_dest info "Syncing $SECTION on $dest_path..." - debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path - $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log + debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path + set_pipefail + $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then fatal "Rsync error when trying to transfer $SECTION" fi + restore_pipefail update_metadata done @@ -1136,7 +1125,6 @@ done mount_ro run_fsck start_services -unset_lockfile end_mux echo "Finnishing backup at `date`" >> $log diff --git a/src/backupninja.in b/src/backupninja.in index 0400828..0f74906 100755 --- a/src/backupninja.in +++ b/src/backupninja.in @@ -349,7 +349,7 @@ function process_action() { else # a backup is probably ongoing already, so display an error message - debug "failed to acquire lock" + debug "failed to acquire lock $lockfile" echo "Fatal: Could not acquire lock $lockfile. A backup is probably already running for $file." >>$bufferfile fi } 200> $lockfile |