diff options
| -rw-r--r-- | files/handlers/rsync | 169 |
1 files changed, 80 insertions, 89 deletions
diff --git a/files/handlers/rsync b/files/handlers/rsync index e098a84..01a2987 100644 --- a/files/handlers/rsync +++ b/files/handlers/rsync @@ -39,7 +39,6 @@ # keepdaily = for long storage format, specify the number of daily backup increments # keepweekly = for long storage format, specify the number of weekly backup increments # keepmonthly = for long storage format, specify the number of monthly backup increments -# lockfile = lockfile to be kept during backup execution # nicelevel = rsync command nice level # enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly # tmp = temp folder @@ -60,7 +59,7 @@ # exclude_vserver = vserver-name (valid only if vservers = yes on backupninja.conf) # numericids = when set to 1, use numeric ids instead of user/group mappings on rsync # compress = if set to 1, compress data on rsync (remote source only) -# bandwidthlimit = set a badnwidth limit in KB/s (remote source only) +# bandwidthlimit = set a bandwidth limit in KB/s (remote source only) # remote_rsync = remote rsync program (remote source only) # id_file = ssh key file (remote source only) # batch = set to "yes" to rsync use a batch file as source @@ -79,7 +78,7 @@ # port = remote port number (remote destination only) # user = remote user name (remote destination only) # id_file = ssh key file (remote destination only) -# bandwidthlimit = set a badnwidth limit in KB/s (remote destination only) +# bandwidthlimit = set a bandwidth limit in KB/s (remote destination only) # remote_rsync = remote rsync program (remote dest only) # batch = set to "yes" to rsync write a batch file from the changes # batchbase = folder where the batch file should be written @@ -125,11 +124,10 @@ function eval_config { getconf mountpoint getconf backupdir getconf format short - getconf days + getconf days 7 getconf keepdaily 5 getconf keepweekly 3 getconf keepmonthly 1 - getconf lockfile getconf nicelevel 0 getconf enable_mv_timestamp_bug no getconf tmp /tmp @@ -176,7 +174,7 @@ function eval_config { getconf include getconf exclude getconf exclude_vserver - getconf numericids + getconf numericids 0 getconf compress 0 # dest section @@ -217,7 +215,7 @@ function eval_config { fi fi - getconf numericids + getconf numericids 0 getconf compress 0 # services section @@ -230,19 +228,16 @@ function eval_config { if [ "$dest" != "local" ] && [ "$from" == "remote" ]; then fatal "When source is remote, destination should be local." - exit 1 fi if [ "$from" != "local" ] && [ "$from" != "remote" ]; then fatal "Invalid source $from" - exit 1 fi backupdir="$mountpoint/$backupdir" if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then - error "Backupdir $backupdir does not exist" - exit 1 + fatal "Backupdir $backupdir does not exist" fi if [ ! -z "$log" ]; then @@ -280,9 +275,7 @@ function eval_config { mv=move_files fi - for path in $exclude; do - excludes="$excludes --exclude=$path" - done + excludes=`echo "$exclude" | sed -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"` # Make sure we'll run bash at the destination ssh_cmd="$ssh_cmd /bin/bash" @@ -376,7 +369,7 @@ function rotate_short_remote { echo "Debug: removing orphaned metadata \$file" rm -rf $metadata/\$file fi - done + done ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -397,8 +390,8 @@ function rotate_long { local metadata if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." - exit + warning "Skipping rotate of $backuproot as it doesn't exist." + return fi for rottype in daily weekly monthly; do @@ -411,7 +404,7 @@ function rotate_long { echo "Debug: $dir.1 does not exist, skipping." continue 1 elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then - echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." + warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." continue 1 fi @@ -427,11 +420,16 @@ function rotate_long { else created=0 fi + # Validate created date + if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then + warning "Invalid metadata $created. Skipping rotation." + break + fi cutoff_time=$(( now - (seconds*(i-1)) )) if [ ! $created -gt $cutoff_time ]; then next=$(( i + 1 )) if [ ! -d $dir.$next ]; then - echo "Debug: $rottype.$i --> $rottype.$next" + debug "$rottype.$i --> $rottype.$next" $nice mv $dir.$i $dir.$next mkdir -p $metadata.$next date +%c%n%s > $metadata.$next/rotated @@ -439,10 +437,10 @@ function rotate_long { $nice mv $metadata.$i/created $metadata.$next fi else - echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." + debug "skipping rotation of $dir.$i because $dir.$next already exists." fi else - echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." + debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." fi fi done @@ -450,9 +448,11 @@ function rotate_long { max=$((keepdaily+1)) if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then - echo "Debug: daily.$max --> weekly.1" + debug "daily.$max --> weekly.1" $nice mv $backuproot/daily.$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 + # Update increment folder date and setup metadata + $touch $backuproot/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated #if [ -f $backuproot/metadata/daily.$max/created ]; then # $nice mv $backuproot/metadata/daily.$max/created $backuproot/metadata/weekly.1/ @@ -461,9 +461,11 @@ function rotate_long { max=$((keepweekly+1)) if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then - echo "Debug: weekly.$max --> monthly.1" + debug "weekly.$max --> monthly.1" $nice mv $backuproot/weekly.$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 + # Update increment folder date and setup metadata + $touch $backuproot/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated #if [ -f $backuproot/metadata/weekly.$max/created ]; then # $nice mv $backuproot/metadata/weekly.$max/created $backuproot/metadata/weekly.1/ @@ -479,10 +481,10 @@ function rotate_long { for (( i=$oldest; i >= $max; i-- )); do if [ -d $dir.$i ]; then if [ -d $backuproot/rotate.tmp ]; then - echo "Debug: removing rotate.tmp" + debug "removing rotate.tmp" $nice rm -rf $backuproot/rotate.tmp fi - echo "Debug: moving $rottype.$i to rotate.tmp" + debug "moving $rottype.$i to rotate.tmp" $nice mv $dir.$i $backuproot/rotate.tmp fi done @@ -515,7 +517,7 @@ function rotate_long_remote { now=\`date +%s\` if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." + echo "Fatal: skipping rotate of $backuproot as it doesn't exist." exit fi @@ -545,6 +547,11 @@ function rotate_long_remote { else created=0 fi + # Validate created date + if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then + echo "Warning: Invalid metadata \$created. Skipping rotation." + break + fi cutoff_time=\$(( now - (seconds*(i-1)) )) if [ ! \$created -gt \$cutoff_time ]; then next=\$(( i + 1 )) @@ -571,6 +578,8 @@ function rotate_long_remote { echo "Debug: daily.\$max --> weekly.1" $nice mv $backuproot/daily.\$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 + # Update increment folder date and setup metadata + $touch $backuproot/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated #if [ -f $backuproot/metadata/daily.\$max/created ]; then # $nice mv $backuproot/metadata/daily.\$max/created $backuproot/metadata/weekly.1/ @@ -582,6 +591,8 @@ function rotate_long_remote { echo "Debug: weekly.\$max --> monthly.1" $nice mv $backuproot/weekly.\$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 + # Update increment folder date and setup metadata + $touch $backuproot/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated #if [ -f $backuproot/metadata/weekly.\$max/created ]; then # $nice mv $backuproot/metadata/weekly.\$max/created $backuproot/metadata/weekly.1/ @@ -642,22 +653,19 @@ function setup_long_dirs { if [ -d $tmpdir ]; then mv $tmpdir $dir.1 if [ $? == 1 ]; then - echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host" - exit 1 + fatal "Could not move $tmpdir to $dir.1 on host $host" fi else mkdir --parents $dir.1 if [ $? == 1 ]; then - echo "Fatal: could not create directory $dir.1 on host $host" - exit 1 + fatal "Could not create directory $dir.1 on host $host" fi fi if [ -d $dir.2 ]; then echo "Debug: update links $backuptype.2 --> $backuptype.1" cp -alf $dir.2/. $dir.1 #if [ $? == 1 ]; then - # echo "Fatal: could not create hard links to $dir.1 on host $host" - # exit 1 + # fatal "Could not create hard links to $dir.1 on host $host" #fi fi fi @@ -758,7 +766,6 @@ function prepare_storage { btype=monthly else fatal "keeping no backups"; - exit 1 fi suffix="$btype.1" @@ -777,7 +784,6 @@ function prepare_storage { suffix="" else fatal "Invalid backup format $format" - exit 1 fi } @@ -821,7 +827,6 @@ function set_batch_mode { batch_option="--read-batch=$batch_file" else fatal "Batch file not found: $batch_file" - exit 1 fi elif [ "$batch" == "write" ]; then mkdir -p `dirname $batch_file` @@ -838,7 +843,11 @@ function update_metadata { if [ "$dest" == "local" ]; then metadata="`dirname $dest_path`/metadata/`basename $dest_path`" mkdir -p $metadata - date +%c%n%s > $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo "$starttime" > $metadata/created $touch $backupdir/$SECTION/$suffix else folder="`echo $dest_path | cut -d : -f 2`" @@ -848,7 +857,11 @@ function update_metadata { $ssh_cmd <<EOF ##### BEGIN REMOTE SCRIPT ##### mkdir -p $metadata - date +%c%n%s > $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo "$starttime" > $metadata/created ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -866,7 +879,6 @@ function test_connect { if [ -z "$host" ] || [ -z "$user" ]; then fatal "Remote host or user not set" - exit 1 fi debug "$ssh_cmd 'echo -n 1'" @@ -874,52 +886,12 @@ function test_connect { if [ "$result" != "1" ]; then fatal "Can't connect to $host as $user." - exit 1 else debug "Connected to $host successfully" fi } -function set_lockfile { - - if [ ! -z "$lockfile" ]; then - mkdir -p `dirname $lockfile` - if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then - trap 'unset_lockfile' INT TERM EXIT - else - fatal "Could not create lockfile $lockfile, exiting" - fi - fi - -} - -function unset_lockfile { - - if [ ! -z "$lockfile" ]; then - $rm -f $lockfile || warning "Could not remove lockfile $lockfile" - fi - -} - -function check_lockfile { - - local pid process - - if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then - pid="`cat $lockfile`" - process="`ps --no-headers -o comm $pid`" - if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then - info "Another backup is running for $lockfile, skipping run" - exit - else - info "Found old lockfile $lockfile, removing it" - unset_lockfile - fi - fi - -} - function set_filelist { filelist_flag="" @@ -940,7 +912,7 @@ function set_filelist { function set_rsync_options { - if [ ! -z "$numericids" ]; then + if [ "$numericids" != "0" ]; then rsync_options="$rsync_options --numeric-ids" fi @@ -968,7 +940,6 @@ function set_rsync_options { if [ "$protocol" == "ssh" ]; then if [ ! -e "$id_file" ]; then fatal "SSH Identity file $id_file not found" - exit 1 else debug RSYNC_RSH=\"$ssh_cmd_base\" echo RSYNC_RSH=\"$ssh_cmd_base\" >> $log @@ -1023,8 +994,7 @@ function mount_rw { if [ -d "$mountpoint" ]; then mount -o remount,rw $mountpoint if (($?)); then - error "Could not mount $mountpoint" - exit 1 + fatal "Could not mount $mountpoint" fi fi fi @@ -1108,17 +1078,39 @@ function end_mux { } +function set_pipefail { + + # Save initial pipefail status for later restoration + if echo "$SHELLOPTS" | grep -q ":pipefail"; then + pipefail="-o" + else + pipefail="+o" + fi + + # Ensure that a non-zero rsync exit status is caught by our handler + set -o pipefail + +} + +function restore_pipefail { + + if [ ! -z "$pipefail" ]; then + set $pipefail pipefail + fi + +} + # the backup procedure eval_config -check_lockfile -set_lockfile set_rsync_options start_mux stop_services mount_rw +set_pipefail -echo "Starting backup at `date`" >> $log +starttime="`date +%c%n%s`" +echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log for SECTION in $include; do @@ -1129,22 +1121,21 @@ for SECTION in $include; do set_dest info "Syncing $SECTION on $dest_path..." - debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path - $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log + debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path + $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then fatal "Rsync error when trying to transfer $SECTION" - exit 1 fi update_metadata done +restore_pipefail mount_ro run_fsck start_services -unset_lockfile end_mux echo "Finnishing backup at `date`" >> $log |