Updating rsync handler from upstream rsync handler development

1 files changed, 80 insertions, 89 deletions

diff --git a/files/handlers/rsync b/files/handlers/rsync
index e098a84..01a2987 100644
--- a/files/handlers/rsync
+++ b/files/handlers/rsync
@@ -39,7 +39,6 @@
 #   keepdaily = for long storage format, specify the number of daily backup increments
 #   keepweekly = for long storage format, specify the number of weekly backup increments
 #   keepmonthly = for long storage format, specify the number of monthly backup increments
-#   lockfile = lockfile to be kept during backup execution
 #   nicelevel = rsync command nice level
 #   enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
 #   tmp = temp folder
@@ -60,7 +59,7 @@
 #   exclude_vserver = vserver-name (valid only if vservers = yes on backupninja.conf)
 #   numericids = when set to 1, use numeric ids instead of user/group mappings on rsync
 #   compress = if set to 1, compress data on rsync (remote source only)
-#   bandwidthlimit = set a badnwidth limit in KB/s (remote source only)
+#   bandwidthlimit = set a bandwidth limit in KB/s (remote source only)
 #   remote_rsync = remote rsync program (remote source only)
 #   id_file = ssh key file (remote source only)
 #   batch = set to "yes" to rsync use a batch file as source
@@ -79,7 +78,7 @@
 #   port = remote port number (remote destination only)
 #   user = remote user name (remote destination only)
 #   id_file = ssh key file (remote destination only)
-#   bandwidthlimit = set a badnwidth limit in KB/s (remote destination only)
+#   bandwidthlimit = set a bandwidth limit in KB/s (remote destination only)
 #   remote_rsync = remote rsync program (remote dest only)
 #   batch = set to "yes" to rsync write a batch file from the changes
 #   batchbase = folder where the batch file should be written
@@ -125,11 +124,10 @@ function eval_config {
   getconf mountpoint
   getconf backupdir
   getconf format short
-  getconf days
+  getconf days 7
   getconf keepdaily 5
   getconf keepweekly 3
   getconf keepmonthly 1
-  getconf lockfile
   getconf nicelevel 0
   getconf enable_mv_timestamp_bug no
   getconf tmp /tmp
@@ -176,7 +174,7 @@ function eval_config {
   getconf include
   getconf exclude
   getconf exclude_vserver
-  getconf numericids
+  getconf numericids 0
   getconf compress 0
   
   # dest section
@@ -217,7 +215,7 @@ function eval_config {
     fi
   fi
 
-  getconf numericids
+  getconf numericids 0
   getconf compress 0
   
   # services section
@@ -230,19 +228,16 @@ function eval_config {
 
   if [ "$dest" != "local" ] && [ "$from" == "remote" ]; then
     fatal "When source is remote, destination should be local."
-    exit 1
   fi
 
   if [ "$from" != "local" ] && [ "$from" != "remote" ]; then
     fatal "Invalid source $from"
-    exit 1
   fi
 
   backupdir="$mountpoint/$backupdir"
 
   if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then 
-    error "Backupdir $backupdir does not exist"
-    exit 1
+    fatal "Backupdir $backupdir does not exist"
   fi
 
   if [ ! -z "$log" ]; then
@@ -280,9 +275,7 @@ function eval_config {
     mv=move_files
   fi
 
-  for path in $exclude; do
-    excludes="$excludes --exclude=$path"
-  done
+  excludes=`echo "$exclude" | sed -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"`
 
   # Make sure we'll run bash at the destination
   ssh_cmd="$ssh_cmd /bin/bash"
@@ -376,7 +369,7 @@ function rotate_short_remote {
       echo "Debug: removing orphaned metadata \$file"
       rm -rf $metadata/\$file
     fi
-  done  
+  done
   ##### END REMOTE SCRIPT #######
 EOF
 ) | (while read a; do passthru $a; done)
@@ -397,8 +390,8 @@ function rotate_long {
   local metadata
 
   if [ ! -d "$backuproot" ]; then
-    echo "Debug: skipping rotate of $backuproot as it doesn't exist."
-    exit
+    warning "Skipping rotate of $backuproot as it doesn't exist."
+    return
   fi
 
   for rottype in daily weekly monthly; do
@@ -411,7 +404,7 @@ function rotate_long {
       echo "Debug: $dir.1 does not exist, skipping."
       continue 1
     elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then
-      echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
+      warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
       continue 1
     fi
     
@@ -427,11 +420,16 @@ function rotate_long {
         else
           created=0
         fi
+        # Validate created date
+        if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then
+           warning "Invalid metadata $created. Skipping rotation."
+           break
+        fi
         cutoff_time=$(( now - (seconds*(i-1)) ))
         if [ ! $created -gt $cutoff_time ]; then
           next=$(( i + 1 ))
           if [ ! -d $dir.$next ]; then
-            echo "Debug: $rottype.$i --> $rottype.$next"
+            debug "$rottype.$i --> $rottype.$next"
             $nice mv $dir.$i $dir.$next
             mkdir -p $metadata.$next
             date +%c%n%s > $metadata.$next/rotated
@@ -439,10 +437,10 @@ function rotate_long {
               $nice mv $metadata.$i/created $metadata.$next
             fi
           else
-            echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
+            debug "skipping rotation of $dir.$i because $dir.$next already exists."
           fi
         else
-          echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
+          debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
         fi
       fi
     done
@@ -450,9 +448,11 @@ function rotate_long {
 
   max=$((keepdaily+1))
   if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
-    echo "Debug: daily.$max --> weekly.1"
+    debug "daily.$max --> weekly.1"
     $nice mv $backuproot/daily.$max $backuproot/weekly.1
     mkdir -p $backuproot/metadata/weekly.1
+    # Update increment folder date and setup metadata
+    $touch $backuproot/weekly.1
     date +%c%n%s > $backuproot/metadata/weekly.1/rotated
     #if [ -f $backuproot/metadata/daily.$max/created  ]; then
     #   $nice mv $backuproot/metadata/daily.$max/created $backuproot/metadata/weekly.1/
@@ -461,9 +461,11 @@ function rotate_long {
 
   max=$((keepweekly+1))
   if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
-    echo "Debug: weekly.$max --> monthly.1"
+    debug "weekly.$max --> monthly.1"
     $nice mv $backuproot/weekly.$max $backuproot/monthly.1
     mkdir -p $backuproot/metadata/monthly.1
+    # Update increment folder date and setup metadata
+    $touch $backuproot/monthly.1
     date +%c%n%s > $backuproot/metadata/monthly.1/rotated
     #if [ -f $backuproot/metadata/weekly.$max/created  ]; then
     #   $nice mv $backuproot/metadata/weekly.$max/created $backuproot/metadata/weekly.1/
@@ -479,10 +481,10 @@ function rotate_long {
     for (( i=$oldest; i >= $max; i-- )); do
       if [ -d $dir.$i ]; then
         if [ -d $backuproot/rotate.tmp ]; then
-          echo "Debug: removing rotate.tmp"
+          debug "removing rotate.tmp"
           $nice rm -rf $backuproot/rotate.tmp
         fi
-        echo "Debug: moving $rottype.$i to rotate.tmp"
+        debug "moving $rottype.$i to rotate.tmp"
         $nice mv $dir.$i $backuproot/rotate.tmp
       fi
     done
@@ -515,7 +517,7 @@ function rotate_long_remote {
   now=\`date +%s\`
 
   if [ ! -d "$backuproot" ]; then
-    echo "Debug: skipping rotate of $backuproot as it doesn't exist."
+    echo "Fatal: skipping rotate of $backuproot as it doesn't exist."
     exit
   fi
 
@@ -545,6 +547,11 @@ function rotate_long_remote {
         else
           created=0
         fi
+        # Validate created date
+        if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then
+           echo "Warning: Invalid metadata \$created. Skipping rotation."
+           break
+        fi
         cutoff_time=\$(( now - (seconds*(i-1)) ))
         if [ ! \$created -gt \$cutoff_time ]; then
           next=\$(( i + 1 ))
@@ -571,6 +578,8 @@ function rotate_long_remote {
     echo "Debug: daily.\$max --> weekly.1"
     $nice mv $backuproot/daily.\$max $backuproot/weekly.1
     mkdir -p $backuproot/metadata/weekly.1
+    # Update increment folder date and setup metadata
+    $touch $backuproot/weekly.1
     date +%c%n%s > $backuproot/metadata/weekly.1/rotated
     #if [ -f $backuproot/metadata/daily.\$max/created  ]; then
     #   $nice mv $backuproot/metadata/daily.\$max/created $backuproot/metadata/weekly.1/
@@ -582,6 +591,8 @@ function rotate_long_remote {
     echo "Debug: weekly.\$max --> monthly.1"
     $nice mv $backuproot/weekly.\$max $backuproot/monthly.1
     mkdir -p $backuproot/metadata/monthly.1
+    # Update increment folder date and setup metadata
+    $touch $backuproot/monthly.1
     date +%c%n%s > $backuproot/metadata/monthly.1/rotated
     #if [ -f $backuproot/metadata/weekly.\$max/created  ]; then
     #   $nice mv $backuproot/metadata/weekly.\$max/created $backuproot/metadata/weekly.1/
@@ -642,22 +653,19 @@ function setup_long_dirs {
     if [ -d $tmpdir ]; then
       mv $tmpdir $dir.1
       if [ $? == 1 ]; then
-        echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
-        exit 1
+        fatal "Could not move $tmpdir to $dir.1 on host $host"
       fi
     else
       mkdir --parents $dir.1
       if [ $? == 1 ]; then
-        echo "Fatal: could not create directory $dir.1 on host $host"
-        exit 1
+        fatal "Could not create directory $dir.1 on host $host"
       fi
     fi
     if [ -d $dir.2 ]; then
       echo "Debug: update links $backuptype.2 --> $backuptype.1"
       cp -alf $dir.2/. $dir.1
       #if [ $? == 1 ]; then
-      #  echo "Fatal: could not create hard links to $dir.1 on host $host"
-      #  exit 1
+      #  fatal "Could not create hard links to $dir.1 on host $host"
       #fi
     fi
   fi
@@ -758,7 +766,6 @@ function prepare_storage {
       btype=monthly
     else
       fatal "keeping no backups";
-      exit 1
     fi
 
     suffix="$btype.1"
@@ -777,7 +784,6 @@ function prepare_storage {
     suffix=""
   else
     fatal "Invalid backup format $format"
-    exit 1
   fi
 
 }
@@ -821,7 +827,6 @@ function set_batch_mode {
       batch_option="--read-batch=$batch_file"
     else
       fatal "Batch file not found: $batch_file"
-      exit 1
     fi
   elif [ "$batch" == "write" ]; then
     mkdir -p `dirname $batch_file`
@@ -838,7 +843,11 @@ function update_metadata {
   if [ "$dest" == "local" ]; then
     metadata="`dirname $dest_path`/metadata/`basename $dest_path`"
     mkdir -p $metadata
-    date +%c%n%s > $metadata/created
+    # Use the backup start time and not the time the backup was
+    # finished, otherwise daily rotations might not take place.
+    # If we used backup end time, in the next handler run
+    # we might not have $now - $created >= 24:00
+    echo "$starttime" > $metadata/created
     $touch $backupdir/$SECTION/$suffix
   else
     folder="`echo $dest_path | cut -d : -f 2`"
@@ -848,7 +857,11 @@ function update_metadata {
   $ssh_cmd <<EOF
     ##### BEGIN REMOTE SCRIPT #####
     mkdir -p $metadata
-    date +%c%n%s > $metadata/created
+    # Use the backup start time and not the time the backup was
+    # finished, otherwise daily rotations might not take place.
+    # If we used backup end time, in the next handler run
+    # we might not have $now - $created >= 24:00
+    echo "$starttime" > $metadata/created
     ##### END REMOTE SCRIPT #######
 EOF
 ) | (while read a; do passthru $a; done)
@@ -866,7 +879,6 @@ function test_connect {
 
   if [ -z "$host" ] || [ -z "$user" ]; then
     fatal "Remote host or user not set"
-    exit 1
   fi
 
   debug "$ssh_cmd 'echo -n 1'"
@@ -874,52 +886,12 @@ function test_connect {
 
   if [ "$result" != "1" ]; then
     fatal "Can't connect to $host as $user."
-    exit 1
   else
     debug "Connected to $host successfully"
   fi
 
 }
 
-function set_lockfile {
-
-  if [ ! -z "$lockfile" ]; then
-    mkdir -p `dirname $lockfile`
-    if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then
-      trap 'unset_lockfile' INT TERM EXIT
-    else
-      fatal "Could not create lockfile $lockfile, exiting"
-    fi
-  fi
-
-}
-
-function unset_lockfile {
-
-  if [ ! -z "$lockfile" ]; then
-    $rm -f $lockfile || warning "Could not remove lockfile $lockfile"
-  fi
-
-}
-
-function check_lockfile {
-
-  local pid process
-
-  if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then
-    pid="`cat $lockfile`"
-    process="`ps --no-headers -o comm $pid`"
-    if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then
-      info "Another backup is running for $lockfile, skipping run"
-      exit
-    else
-      info "Found old lockfile $lockfile, removing it"
-      unset_lockfile  
-    fi
-  fi
-
-}
-
 function set_filelist {
 
   filelist_flag=""
@@ -940,7 +912,7 @@ function set_filelist {
 
 function set_rsync_options {
 
-  if [ ! -z "$numericids" ]; then
+  if [ "$numericids" != "0" ]; then
     rsync_options="$rsync_options --numeric-ids"
   fi
 
@@ -968,7 +940,6 @@ function set_rsync_options {
     if [ "$protocol" == "ssh" ]; then
       if [ ! -e "$id_file" ]; then
         fatal "SSH Identity file $id_file not found"
-        exit 1
       else
         debug RSYNC_RSH=\"$ssh_cmd_base\"
         echo RSYNC_RSH=\"$ssh_cmd_base\" >> $log
@@ -1023,8 +994,7 @@ function mount_rw {
       if [ -d "$mountpoint" ]; then
         mount -o remount,rw $mountpoint
         if (($?)); then
-          error "Could not mount $mountpoint"
-          exit 1
+          fatal "Could not mount $mountpoint"
         fi
       fi
     fi
@@ -1108,17 +1078,39 @@ function end_mux {
 
 }
 
+function set_pipefail {
+
+  # Save initial pipefail status for later restoration
+  if echo "$SHELLOPTS" | grep -q ":pipefail"; then
+     pipefail="-o"
+  else
+     pipefail="+o"
+  fi
+
+  # Ensure that a non-zero rsync exit status is caught by our handler
+  set -o pipefail
+
+}
+
+function restore_pipefail {
+
+  if [ ! -z "$pipefail" ]; then
+    set $pipefail pipefail
+  fi
+
+}
+
 # the backup procedure
 
 eval_config
-check_lockfile
-set_lockfile
 set_rsync_options
 start_mux
 stop_services
 mount_rw
+set_pipefail
 
-echo "Starting backup at `date`" >> $log
+starttime="`date +%c%n%s`"
+echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log
 
 for SECTION in $include; do
 
@@ -1129,22 +1121,21 @@ for SECTION in $include; do
   set_dest
 
   info "Syncing $SECTION on $dest_path..."
-  debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path
-  $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log
+  debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path
+  $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log
 
   if [ "$?" != "0" ]; then
     fatal "Rsync error when trying to transfer $SECTION"
-    exit 1
   fi
 
   update_metadata
 
 done
 
+restore_pipefail
 mount_ro
 run_fsck
 start_services
-unset_lockfile
 end_mux
 
 echo "Finnishing backup at `date`" >> $log