diff options
Diffstat (limited to 'handlers/rsync.in')
-rw-r--r-- | handlers/rsync.in | 171 |
1 files changed, 101 insertions, 70 deletions
diff --git a/handlers/rsync.in b/handlers/rsync.in index 81bd5b4..d0c211c 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -39,7 +39,6 @@ # keepdaily = for long storage format, specify the number of daily backup increments # keepweekly = for long storage format, specify the number of weekly backup increments # keepmonthly = for long storage format, specify the number of monthly backup increments -# lockfile = lockfile to be kept during backup execution # nicelevel = rsync command nice level # enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly # tmp = temp folder @@ -129,7 +128,6 @@ function eval_config { getconf keepdaily 5 getconf keepweekly 3 getconf keepmonthly 1 - getconf lockfile getconf nicelevel 0 getconf enable_mv_timestamp_bug no getconf tmp /tmp @@ -277,9 +275,15 @@ function eval_config { mv=move_files fi - for path in $exclude; do - excludes="$excludes --exclude=$path" + set -o noglob + SAVEIFS=$IFS + IFS=$(echo -en "\n\b") + for i in $exclude; do + str="${i//__star__/*}" + excludes="${excludes} --exclude='$str'" done + IFS=$SAVEIFS + set +o noglob # Make sure we'll run bash at the destination ssh_cmd="$ssh_cmd /bin/bash" @@ -394,8 +398,8 @@ function rotate_long { local metadata if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." - exit + warning "Skipping rotate of $backuproot as it doesn't exist." + return fi for rottype in daily weekly monthly; do @@ -408,12 +412,12 @@ function rotate_long { echo "Debug: $dir.1 does not exist, skipping." continue 1 elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then - echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." + warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." continue 1 fi # Rotate the current list of backups, if we can. - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 for (( i=$oldest; i > 0; i-- )); do if [ -d $dir.$i ]; then @@ -424,11 +428,16 @@ function rotate_long { else created=0 fi + # Validate created date + if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then + warning "Invalid metadata $created. Skipping rotation." + break + fi cutoff_time=$(( now - (seconds*(i-1)) )) if [ ! $created -gt $cutoff_time ]; then next=$(( i + 1 )) if [ ! -d $dir.$next ]; then - echo "Debug: $rottype.$i --> $rottype.$next" + debug "$rottype.$i --> $rottype.$next" $nice mv $dir.$i $dir.$next mkdir -p $metadata.$next date +%c%n%s > $metadata.$next/rotated @@ -436,10 +445,10 @@ function rotate_long { $nice mv $metadata.$i/created $metadata.$next fi else - echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." + debug "skipping rotation of $dir.$i because $dir.$next already exists." fi else - echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." + debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." fi fi done @@ -447,9 +456,11 @@ function rotate_long { max=$((keepdaily+1)) if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then - echo "Debug: daily.$max --> weekly.1" + debug "daily.$max --> weekly.1" $nice mv $backuproot/daily.$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 + # Update increment folder date and setup metadata + $touch $backuproot/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated #if [ -f $backuproot/metadata/daily.$max/created ]; then # $nice mv $backuproot/metadata/daily.$max/created $backuproot/metadata/weekly.1/ @@ -458,9 +469,11 @@ function rotate_long { max=$((keepweekly+1)) if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then - echo "Debug: weekly.$max --> monthly.1" + debug "weekly.$max --> monthly.1" $nice mv $backuproot/weekly.$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 + # Update increment folder date and setup metadata + $touch $backuproot/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated #if [ -f $backuproot/metadata/weekly.$max/created ]; then # $nice mv $backuproot/metadata/weekly.$max/created $backuproot/metadata/weekly.1/ @@ -470,16 +483,16 @@ function rotate_long { for rottype in daily weekly monthly; do max=$((keep${rottype}+1)) dir="$backuproot/$rottype" - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=$oldest; i >= $max; i-- )); do if [ -d $dir.$i ]; then if [ -d $backuproot/rotate.tmp ]; then - echo "Debug: removing rotate.tmp" + debug "removing rotate.tmp" $nice rm -rf $backuproot/rotate.tmp fi - echo "Debug: moving $rottype.$i to rotate.tmp" + debug "moving $rottype.$i to rotate.tmp" $nice mv $dir.$i $backuproot/rotate.tmp fi done @@ -512,7 +525,7 @@ function rotate_long_remote { now=\`date +%s\` if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." + echo "Fatal: skipping rotate of $backuproot as it doesn't exist." exit fi @@ -531,7 +544,7 @@ function rotate_long_remote { fi # Rotate the current list of backups, if we can. - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 for (( i=\$oldest; i > 0; i-- )); do if [ -d \$dir.\$i ]; then @@ -542,6 +555,11 @@ function rotate_long_remote { else created=0 fi + # Validate created date + if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then + echo "Warning: Invalid metadata \$created. Skipping rotation." + break + fi cutoff_time=\$(( now - (seconds*(i-1)) )) if [ ! \$created -gt \$cutoff_time ]; then next=\$(( i + 1 )) @@ -568,6 +586,8 @@ function rotate_long_remote { echo "Debug: daily.\$max --> weekly.1" $nice mv $backuproot/daily.\$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 + # Update increment folder date and setup metadata + $touch $backuproot/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated #if [ -f $backuproot/metadata/daily.\$max/created ]; then # $nice mv $backuproot/metadata/daily.\$max/created $backuproot/metadata/weekly.1/ @@ -579,6 +599,8 @@ function rotate_long_remote { echo "Debug: weekly.\$max --> monthly.1" $nice mv $backuproot/weekly.\$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 + # Update increment folder date and setup metadata + $touch $backuproot/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated #if [ -f $backuproot/metadata/weekly.\$max/created ]; then # $nice mv $backuproot/metadata/weekly.\$max/created $backuproot/metadata/weekly.1/ @@ -588,7 +610,7 @@ function rotate_long_remote { for rottype in daily weekly monthly; do max=\$((keep\${rottype}+1)) dir="$backuproot/\$rottype" - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=\$oldest; i >= \$max; i-- )); do @@ -829,7 +851,11 @@ function update_metadata { if [ "$dest" == "local" ]; then metadata="`dirname $dest_path`/metadata/`basename $dest_path`" mkdir -p $metadata - date +%c%n%s > $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo "$starttime" > $metadata/created $touch $backupdir/$SECTION/$suffix else folder="`echo $dest_path | cut -d : -f 2`" @@ -839,7 +865,11 @@ function update_metadata { $ssh_cmd <<EOF ##### BEGIN REMOTE SCRIPT ##### mkdir -p $metadata - date +%c%n%s > $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo "$starttime" > $metadata/created ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -870,45 +900,6 @@ function test_connect { } -function set_lockfile { - - if [ ! -z "$lockfile" ]; then - mkdir -p `dirname $lockfile` - if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then - trap 'unset_lockfile' INT TERM EXIT - else - fatal "Could not create lockfile $lockfile, exiting" - fi - fi - -} - -function unset_lockfile { - - if [ ! -z "$lockfile" ]; then - $rm -f $lockfile || warning "Could not remove lockfile $lockfile" - fi - -} - -function check_lockfile { - - local pid process - - if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then - pid="`cat $lockfile`" - process="`ps --no-headers -o comm $pid`" - if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then - info "Another backup is running for $lockfile, skipping run" - exit - else - info "Found old lockfile $lockfile, removing it" - unset_lockfile - fi - fi - -} - function set_filelist { filelist_flag="" @@ -1095,17 +1086,58 @@ function end_mux { } +function set_pipefail { + + # Save initial pipefail status for later restoration + if echo "$SHELLOPTS" | grep -q ":pipefail"; then + pipefail="-o" + else + pipefail="+o" + fi + + # Ensure that a non-zero rsync exit status is caught by our handler + set -o pipefail + +} + +function restore_pipefail { + + if [ ! -z "$pipefail" ]; then + set $pipefail pipefail + fi + +} + +function check_rsync_exit_status { + + if [ -z "$1" ]; then + return + fi + + case $1 in + 0) + return + ;; + 1|2|3|4|5|6|10|11|12|13|14|21) + fatal "Rsync error $1 when trying to transfer $SECTION" + ;; + *) + warning "Rsync error $1 when trying to transfer $SECTION" + ;; + esac + +} + # the backup procedure eval_config -check_lockfile -set_lockfile set_rsync_options start_mux stop_services mount_rw -echo "Starting backup at `date`" >> $log +starttime="`date +%c%n%s`" +echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log for SECTION in $include; do @@ -1116,13 +1148,13 @@ for SECTION in $include; do set_dest info "Syncing $SECTION on $dest_path..." - debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path - $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log - - if [ "$?" != "0" ]; then - fatal "Rsync error when trying to transfer $SECTION" - fi + command="$rsync ${rsync_options[@]} --delete-excluded $filelist_flag $excludes $batch_option $orig $dest_path" + debug $nice su -c "$command" + set_pipefail + $nice su -c "$command" | tee -a $log + check_rsync_exit_status $? + restore_pipefail update_metadata done @@ -1130,7 +1162,6 @@ done mount_ro run_fsck start_services -unset_lockfile end_mux echo "Finnishing backup at `date`" >> $log |