From 46000c030896cb8efe820e762d9d12ee663b5936 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 24 Dec 2011 11:11:48 -0200 Subject: Make numericids defaults to null (fixes #3691) --- handlers/rsync.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 6c772e1..cc255a3 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -887,7 +887,7 @@ function set_filelist { function set_rsync_options { - if [ ! -z "$numericids" ]; then + if [ "$numericids" != "0" ]; then rsync_options="$rsync_options --numeric-ids" fi -- cgit v1.2.3 From b89b684c2cece1f099a512dc7d7cbe0b2ad06b42 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 31 Dec 2011 14:29:15 -0200 Subject: Provide a default number of days for the rsync handler --- handlers/rsync.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index cc255a3..45b9b31 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -125,7 +125,7 @@ function eval_config { getconf mountpoint getconf backupdir getconf format short - getconf days + getconf days 7 getconf keepdaily 5 getconf keepweekly 3 getconf keepmonthly 1 -- cgit v1.2.3 From 926152dfa1c3b076488bf768ae6cd2282cfa5497 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 24 Dec 2011 11:35:06 -0200 Subject: Mangle rsync_options just after all other options were evaluated (#3702) --- ChangeLog | 1 + handlers/rsync.in | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'handlers/rsync.in') diff --git a/ChangeLog b/ChangeLog index e1ac76d..fa319b8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ version 1.0 -- UNRELEASED handler changes rysnc: . Fix numericids option (Redmine#3691). + . Mangle $rsync_options just afterwards (Redmine#3702). dup: . Make the .dup generated by ninjahelper more consistent with example.dup. diff --git a/handlers/rsync.in b/handlers/rsync.in index 45b9b31..a17afbd 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -893,7 +893,7 @@ function set_rsync_options { if [ "$from" == "local" ] || [ "$dest" == "local" ]; then # rsync options for local sources or destinations - rsync_options=($rsync_options) + true fi if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then @@ -912,8 +912,6 @@ function set_rsync_options { remote_rsync="$remote_rsync --fake-super" fi - rsync_options=($rsync_options --rsync-path="$remote_rsync") - if [ "$protocol" == "ssh" ]; then if [ ! -e "$id_file" ]; then fatal "SSH Identity file $id_file not found" @@ -927,6 +925,14 @@ function set_rsync_options { fi + # Mangle rsync_options so we can use quotes after all other + # options were evaluated. + if [ "$from" == "local" ] && [ "$dest" == "local" ]; then + rsync_options=($rsync_options) + else + rsync_options=($rsync_options --rsync-path="$remote_rsync") + fi + include_vservers } -- cgit v1.2.3 From 10d67587efa3f979ea5f9ddca4fff6d2475c3140 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 31 Dec 2011 20:03:02 -0200 Subject: Removing 'exit 1'; using 'fatal' instead (Redmine#3721) --- handlers/rsync.in | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index a17afbd..ef0e636 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -230,19 +230,16 @@ function eval_config { if [ "$dest" != "local" ] && [ "$from" == "remote" ]; then fatal "When source is remote, destination should be local." - exit 1 fi if [ "$from" != "local" ] && [ "$from" != "remote" ]; then fatal "Invalid source $from" - exit 1 fi backupdir="$mountpoint/$backupdir" if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then - error "Backupdir $backupdir does not exist" - exit 1 + fatal "Backupdir $backupdir does not exist" fi if [ ! -z "$log" ]; then @@ -589,22 +586,19 @@ function setup_long_dirs { if [ -d $tmpdir ]; then mv $tmpdir $dir.1 if [ $? == 1 ]; then - echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host" - exit 1 + fatal "Could not move $tmpdir to $dir.1 on host $host" fi else mkdir --parents $dir.1 if [ $? == 1 ]; then - echo "Fatal: could not create directory $dir.1 on host $host" - exit 1 + fatal "Could not create directory $dir.1 on host $host" fi fi if [ -d $dir.2 ]; then echo "Debug: update links $backuptype.2 --> $backuptype.1" cp -alf $dir.2/. $dir.1 #if [ $? == 1 ]; then - # echo "Fatal: could not create hard links to $dir.1 on host $host" - # exit 1 + # fatal "Could not create hard links to $dir.1 on host $host" #fi fi fi @@ -705,7 +699,6 @@ function prepare_storage { btype=monthly else fatal "keeping no backups"; - exit 1 fi suffix="$btype.1" @@ -724,7 +717,6 @@ function prepare_storage { suffix="" else fatal "Invalid backup format $format" - exit 1 fi } @@ -768,7 +760,6 @@ function set_batch_mode { batch_option="--read-batch=$batch_file" else fatal "Batch file not found: $batch_file" - exit 1 fi elif [ "$batch" == "write" ]; then mkdir -p `dirname $batch_file` @@ -813,7 +804,6 @@ function test_connect { if [ -z "$host" ] || [ -z "$user" ]; then fatal "Remote host or user not set" - exit 1 fi debug "$ssh_cmd 'echo -n 1'" @@ -821,7 +811,6 @@ function test_connect { if [ "$result" != "1" ]; then fatal "Can't connect to $host as $user." - exit 1 else debug "Connected to $host successfully" fi @@ -915,7 +904,6 @@ function set_rsync_options { if [ "$protocol" == "ssh" ]; then if [ ! -e "$id_file" ]; then fatal "SSH Identity file $id_file not found" - exit 1 else debug RSYNC_RSH=\"$ssh_cmd_base\" echo RSYNC_RSH=\"$ssh_cmd_base\" >> $log @@ -970,8 +958,7 @@ function mount_rw { if [ -d "$mountpoint" ]; then mount -o remount,rw $mountpoint if (($?)); then - error "Could not mount $mountpoint" - exit 1 + fatal "Could not mount $mountpoint" fi fi fi -- cgit v1.2.3 From aaa53a78b2757ce0e8d4b18cf96f4d45c2c3ec90 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Mon, 30 May 2011 19:22:04 -0300 Subject: Allow disabling rotation or setting 2 days as minimum for backup increments in rsync short format (Issue #2107) --- handlers/rsync.in | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index ef0e636..767b6e0 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -35,7 +35,7 @@ # mountpoint = backup partition mountpoint or backup main folder (either local or remote) # backupdir = folder relative do $mountpoint where the backup should be stored (local or remote) # format = specify backup storage format: short, long or mirror (i.e, no rotations) -# days = for short storage format, specify the number of backup increments (min = 5) +# days = for short storage format, specify the number of backup increments (min = 2, set to 1 or less to disable) # keepdaily = for long storage format, specify the number of daily backup increments # keepweekly = for long storage format, specify the number of weekly backup increments # keepmonthly = for long storage format, specify the number of monthly backup increments @@ -290,9 +290,9 @@ function rotate_short { local keep="$2" local metadata="`dirname $folder`/metadata" - if [[ "$keep" -lt 4 ]]; then - error "Rotate: minimum of 4 rotations" - exit 1 + # No rotations + if [[ "$keep" -lt 1 ]]; then + return fi if [ -d $folder.$keep ]; then @@ -325,9 +325,9 @@ function rotate_short_remote { local metadata="`dirname $folder`/metadata" local keep="$2" - if [[ "$2" -lt 4 ]]; then - error "Rotate: minimum of 4 rotations" - exit 1 + # No rotations + if [[ "$keep" -lt 1 ]]; then + return fi ( -- cgit v1.2.3 From 6933a8724d792147d774a91d08ddf3cdee163e1f Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sun, 12 Dec 2010 16:49:23 -0200 Subject: Fixing metadata rotation --- handlers/rsync.in | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 767b6e0..ab73051 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -413,6 +413,9 @@ function rotate_long { $nice mv $dir.$i $dir.$next mkdir -p $metadata.$next date +%c%n%s > $metadata.$next/rotated + if [ -f $metadata.$i/created ]; then + $nice mv $metadata.$i/created $metadata.$next + fi else echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." fi @@ -429,6 +432,9 @@ function rotate_long { $nice mv $backuproot/daily.$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated + #if [ -f $backuproot/metadata/daily.$max/created ]; then + # $nice mv $backuproot/metadata/daily.$max/created $backuproot/metadata/weekly.1/ + #fi fi max=$((keepweekly+1)) @@ -437,6 +443,9 @@ function rotate_long { $nice mv $backuproot/weekly.$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated + #if [ -f $backuproot/metadata/weekly.$max/created ]; then + # $nice mv $backuproot/metadata/weekly.$max/created $backuproot/metadata/weekly.1/ + #fi fi for rottype in daily weekly monthly; do @@ -514,6 +523,9 @@ function rotate_long_remote { $nice mv \$dir.\$i \$dir.\$next mkdir -p \$metadata.\$next date +%c%n%s > \$metadata.\$next/rotated + if [ -f \$metadata.\$i/created ]; then + $nice mv \$metadata.\$i/created \$metadata.\$next + fi else echo "Debug: skipping rotation of \$dir.\$i because \$dir.\$next already exists." fi @@ -530,6 +542,9 @@ function rotate_long_remote { $nice mv $backuproot/daily.\$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated + #if [ -f $backuproot/metadata/daily.\$max/created ]; then + # $nice mv $backuproot/metadata/daily.\$max/created $backuproot/metadata/weekly.1/ + #fi fi max=\$((keepweekly+1)) @@ -538,6 +553,9 @@ function rotate_long_remote { $nice mv $backuproot/weekly.\$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated + #if [ -f $backuproot/metadata/weekly.\$max/created ]; then + # $nice mv $backuproot/metadata/weekly.\$max/created $backuproot/metadata/weekly.1/ + #fi fi for rottype in daily weekly monthly; do -- cgit v1.2.3 From d680847c3704413e3fbcb7ef108360b40bc4141d Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 24 Dec 2011 11:20:40 -0200 Subject: Abort on rsync error (#3692) --- handlers/rsync.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index ab73051..ab6496f 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -1085,7 +1085,7 @@ for SECTION in $include; do $nice $rsync $rsync_options $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then - warning "Rsync error when trying to transfer $SECTION" + fatal "Rsync error when trying to transfer $SECTION" fi update_metadata -- cgit v1.2.3 From db45b98132b352bc651f5ee0eb7299919eceb31a Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Tue, 3 Jan 2012 18:54:58 -0200 Subject: Cleanup orphaned metadata (#3727) --- handlers/rsync.in | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index ab6496f..f140642 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -317,6 +317,14 @@ function rotate_short { $nice $cp -alf /$folder.1/. /$folder.0 fi + # Cleanup orphaned metadata + for file in `ls $metadata`; do + if [ ! -d "`dirname $folder`/$file" ]; then + debug "removing orphaned metadata $file" + rm -rf $metadata/$file + fi + done + } function rotate_short_remote { @@ -355,6 +363,14 @@ function rotate_short_remote { if [ -d $folder.1 ]; then $nice $cp -alf /$folder.1/. /$folder.0 fi + + # Cleanup orphaned metadata + for file in `ls $metadata`; do + if [ ! -d "`dirname $folder`/\$file" ]; then + debug "removing orphaned metadata \$file" + rm -rf $metadata/\$file + fi + done ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -466,6 +482,14 @@ function rotate_long { done done + # Cleanup orphaned metadata + for file in `ls $backuproot/metadata`; do + if [ ! -d "$backuproot/$file" ]; then + debug "removing orphaned metadata $file" + rm -rf $backuproot/metadata/$file + fi + done + } function rotate_long_remote { @@ -575,6 +599,14 @@ function rotate_long_remote { fi done done + + # Cleanup orphaned metadata + for file in `ls $backuproot/metadata`; do + if [ ! -d "$backuproot/\$file" ]; then + debug "removing orphaned metadata \$file" + rm -rf $backuproot/metadata/\$file + fi + done ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) -- cgit v1.2.3 From dcabf4fe4ee96b8c201c48e5e42e6e07f53f1a06 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Tue, 3 Jan 2012 19:33:19 -0200 Subject: Fixing escaping and debug message for remote metadata cleanup (#3727) --- handlers/rsync.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index f140642..e46f866 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -365,9 +365,9 @@ function rotate_short_remote { fi # Cleanup orphaned metadata - for file in `ls $metadata`; do + for file in \`ls $metadata\`; do if [ ! -d "`dirname $folder`/\$file" ]; then - debug "removing orphaned metadata \$file" + echo "Debug: removing orphaned metadata \$file" rm -rf $metadata/\$file fi done @@ -601,9 +601,9 @@ function rotate_long_remote { done # Cleanup orphaned metadata - for file in `ls $backuproot/metadata`; do + for file in \`ls $backuproot/metadata\`; do if [ ! -d "$backuproot/\$file" ]; then - debug "removing orphaned metadata \$file" + echo "Debug: removing orphaned metadata \$file" rm -rf $backuproot/metadata/\$file fi done -- cgit v1.2.3 From a74db16e51a057d665f7d8ee35733ffce12a4717 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Sat, 25 Feb 2012 14:01:49 +0100 Subject: Fix typos. --- examples/example.rsync | 4 ++-- handlers/rsync.in | 4 ++-- handlers/wget | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'handlers/rsync.in') diff --git a/examples/example.rsync b/examples/example.rsync index 3ac3975..a2795db 100644 --- a/examples/example.rsync +++ b/examples/example.rsync @@ -116,7 +116,7 @@ exclude_vserver = excluded_vserver2 # if set to 1, compress data on rsync (remote source only) #compress = 0 -# set a badnwidth limit in KB/s (remote source only) +# set a bandwidth limit in KB/s (remote source only) #bandwidthlimit = # remote rsync program (remote source only) @@ -169,7 +169,7 @@ exclude_vserver = excluded_vserver2 # ssh key file (remote destination only) #id_file = /root/.ssh/id_dsa -# set a badnwidth limit in KB/s (remote destination only) +# set a bandwidth limit in KB/s (remote destination only) #bandwidthlimit = # remote rsync program (remote dest only) diff --git a/handlers/rsync.in b/handlers/rsync.in index e46f866..ba2b9b5 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -60,7 +60,7 @@ # exclude_vserver = vserver-name (valid only if vservers = yes on backupninja.conf) # numericids = when set to 1, use numeric ids instead of user/group mappings on rsync # compress = if set to 1, compress data on rsync (remote source only) -# bandwidthlimit = set a badnwidth limit in KB/s (remote source only) +# bandwidthlimit = set a bandwidth limit in KB/s (remote source only) # remote_rsync = remote rsync program (remote source only) # id_file = ssh key file (remote source only) # batch = set to "yes" to rsync use a batch file as source @@ -79,7 +79,7 @@ # port = remote port number (remote destination only) # user = remote user name (remote destination only) # id_file = ssh key file (remote destination only) -# bandwidthlimit = set a badnwidth limit in KB/s (remote destination only) +# bandwidthlimit = set a bandwidth limit in KB/s (remote destination only) # remote_rsync = remote rsync program (remote dest only) # batch = set to "yes" to rsync write a batch file from the changes # batchbase = folder where the batch file should be written diff --git a/handlers/wget b/handlers/wget index 67425fc..79aa22c 100644 --- a/handlers/wget +++ b/handlers/wget @@ -26,7 +26,7 @@ # wget = wget program # wget_options = wget command options # url = remote data url -# bandwidthlimit = set a badnwidth limit in kbps (remote source only) +# bandwidthlimit = set a bandwidth limit in kbps (remote source only) # # [destination] # folder = local folder @@ -182,8 +182,8 @@ echo "Rotating $backupdir/$folder/$folder..." >> $log rotate $backupdir/$folder/$folder $keep info "Wget'ing $SECTION on $backupdir/$folder/$folder.0..." -if [ ! -z "$badnwidth" ]; then - limit_rate="--limit-rate=$badnwidth""k" +if [ ! -z "$bandwidth" ]; then + limit_rate="--limit-rate=$bandwidth""k" fi cd $backupdir/$folder/$folder.0 -- cgit v1.2.3 From 102c619341920100a7a9e546e62e5c4dcdbbb5ac Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Mon, 30 May 2011 18:43:57 -0300 Subject: Fixing $rsync_options output when rsync is local (Issue #3001) --- handlers/rsync.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index ba2b9b5..e253c5c 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -1113,8 +1113,8 @@ for SECTION in $include; do set_dest info "Syncing $SECTION on $dest_path..." - debug $nice $rsync $rsync_options $filelist_flag $excludes $batch_option $orig $dest_path - $nice $rsync $rsync_options $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log + debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path + $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then fatal "Rsync error when trying to transfer $SECTION" -- cgit v1.2.3 From 1fda206fa22dc2f97faf90e96ecd4cd8955d2cf1 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Mon, 27 Feb 2012 18:47:21 -0300 Subject: Rsync handler: using more debug/warning instead of regular echo (upstream #3840) --- handlers/rsync.in | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index e253c5c..a9dae06 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -405,7 +405,7 @@ function rotate_long { echo "Debug: $dir.1 does not exist, skipping." continue 1 elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then - echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." + warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." continue 1 fi @@ -425,7 +425,7 @@ function rotate_long { if [ ! $created -gt $cutoff_time ]; then next=$(( i + 1 )) if [ ! -d $dir.$next ]; then - echo "Debug: $rottype.$i --> $rottype.$next" + debug "$rottype.$i --> $rottype.$next" $nice mv $dir.$i $dir.$next mkdir -p $metadata.$next date +%c%n%s > $metadata.$next/rotated @@ -433,10 +433,10 @@ function rotate_long { $nice mv $metadata.$i/created $metadata.$next fi else - echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." + debug "skipping rotation of $dir.$i because $dir.$next already exists." fi else - echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." + debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." fi fi done @@ -444,7 +444,7 @@ function rotate_long { max=$((keepdaily+1)) if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then - echo "Debug: daily.$max --> weekly.1" + debug "daily.$max --> weekly.1" $nice mv $backuproot/daily.$max $backuproot/weekly.1 mkdir -p $backuproot/metadata/weekly.1 date +%c%n%s > $backuproot/metadata/weekly.1/rotated @@ -455,7 +455,7 @@ function rotate_long { max=$((keepweekly+1)) if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then - echo "Debug: weekly.$max --> monthly.1" + debug "weekly.$max --> monthly.1" $nice mv $backuproot/weekly.$max $backuproot/monthly.1 mkdir -p $backuproot/metadata/monthly.1 date +%c%n%s > $backuproot/metadata/monthly.1/rotated @@ -473,10 +473,10 @@ function rotate_long { for (( i=$oldest; i >= $max; i-- )); do if [ -d $dir.$i ]; then if [ -d $backuproot/rotate.tmp ]; then - echo "Debug: removing rotate.tmp" + debug "removing rotate.tmp" $nice rm -rf $backuproot/rotate.tmp fi - echo "Debug: moving $rottype.$i to rotate.tmp" + debug "moving $rottype.$i to rotate.tmp" $nice mv $dir.$i $backuproot/rotate.tmp fi done -- cgit v1.2.3 From 15b92db9913cf0c0098fab02d05bd1c90f9af771 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Tue, 28 Feb 2012 01:00:45 -0300 Subject: Rsync handler: use the backup start time and not the time the backup was finished (Debian #654192) --- handlers/rsync.in | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index e253c5c..884a885 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -826,7 +826,11 @@ function update_metadata { if [ "$dest" == "local" ]; then metadata="`dirname $dest_path`/metadata/`basename $dest_path`" mkdir -p $metadata - date +%c%n%s > $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo $starttime > $metadata/created $touch $backupdir/$SECTION/$suffix else folder="`echo $dest_path | cut -d : -f 2`" @@ -836,7 +840,11 @@ function update_metadata { $ssh_cmd < $metadata/created + # Use the backup start time and not the time the backup was + # finished, otherwise daily rotations might not take place. + # If we used backup end time, in the next handler run + # we might not have $now - $created >= 24:00 + echo $starttime > $metadata/created ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -1102,7 +1110,8 @@ start_mux stop_services mount_rw -echo "Starting backup at `date`" >> $log +starttime="`date +%c%n%s`" +echo "Starting backup at `echo $starttime | head -n 1`" >> $log for SECTION in $include; do -- cgit v1.2.3 From 29286c7c3edefc3d77ea529bbe325718cace64db Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Fri, 6 Apr 2012 17:29:10 -0300 Subject: Quoting $starttime (upstream #3868) --- handlers/rsync.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 884a885..68ab392 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -830,7 +830,7 @@ function update_metadata { # finished, otherwise daily rotations might not take place. # If we used backup end time, in the next handler run # we might not have $now - $created >= 24:00 - echo $starttime > $metadata/created + echo "$starttime" > $metadata/created $touch $backupdir/$SECTION/$suffix else folder="`echo $dest_path | cut -d : -f 2`" @@ -844,7 +844,7 @@ function update_metadata { # finished, otherwise daily rotations might not take place. # If we used backup end time, in the next handler run # we might not have $now - $created >= 24:00 - echo $starttime > $metadata/created + echo "$starttime" > $metadata/created ##### END REMOTE SCRIPT ####### EOF ) | (while read a; do passthru $a; done) @@ -1111,7 +1111,7 @@ stop_services mount_rw starttime="`date +%c%n%s`" -echo "Starting backup at `echo $starttime | head -n 1`" >> $log +echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log for SECTION in $include; do -- cgit v1.2.3 From e22107cf0954f29215052becf848bc28b47ffbe0 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 12 Apr 2012 20:49:04 -0300 Subject: Validate created date on long_rotation to avoid too many arguments at comparison (upstream #3868) --- handlers/rsync.in | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 68ab392..31449c2 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -421,6 +421,11 @@ function rotate_long { else created=0 fi + # Validate created date + if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then + warning "Invalid metatada $created. Skipping rotation." + break + fi cutoff_time=$(( now - (seconds*(i-1)) )) if [ ! $created -gt $cutoff_time ]; then next=$(( i + 1 )) @@ -539,6 +544,11 @@ function rotate_long_remote { else created=0 fi + # Validate created date + if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then + echo "Warning: Invalid metatada \$created. Skipping rotation." + break + fi cutoff_time=\$(( now - (seconds*(i-1)) )) if [ ! \$created -gt \$cutoff_time ]; then next=\$(( i + 1 )) -- cgit v1.2.3 From aa8d0f864830036e12e52696243c05df76d678e2 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 12 Apr 2012 21:29:57 -0300 Subject: Quoting $exclude and $excludes and avoiding a for loop on $exclude to not expand wildcards in beforehand (upstream #3882) --- handlers/rsync.in | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index e253c5c..a77da2a 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -277,9 +277,7 @@ function eval_config { mv=move_files fi - for path in $exclude; do - excludes="$excludes --exclude=$path" - done + excludes=`echo "$exclude" | sed -e 's/^/--exclude=/g' -e 's/ / --exclude=/g'` } @@ -1113,8 +1111,8 @@ for SECTION in $include; do set_dest info "Syncing $SECTION on $dest_path..." - debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path - $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log + debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path + $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then fatal "Rsync error when trying to transfer $SECTION" -- cgit v1.2.3 From 5ea13f4c240cf4c8c9f96dc424a5a14acc709f51 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Fri, 13 Apr 2012 00:31:03 -0300 Subject: Using @SED@ for #3882 --- handlers/rsync.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index a77da2a..ad8e64f 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -277,7 +277,7 @@ function eval_config { mv=move_files fi - excludes=`echo "$exclude" | sed -e 's/^/--exclude=/g' -e 's/ / --exclude=/g'` + excludes=`echo "$exclude" | @SED@ 's/^/--exclude=/g' -e 's/ / --exclude=/g'` } -- cgit v1.2.3 From 39bb9e605f04665f8d838be442d9224745959872 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Fri, 13 Apr 2012 09:58:18 -0300 Subject: Trying additional quoting on excludes (upstream #3882) --- handlers/rsync.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index ad8e64f..3108b37 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -277,7 +277,7 @@ function eval_config { mv=move_files fi - excludes=`echo "$exclude" | @SED@ 's/^/--exclude=/g' -e 's/ / --exclude=/g'` + excludes=`echo "$exclude" | @SED@ "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"` } -- cgit v1.2.3 From bf5ada0f0357dd3b7500d03dd4a4a2f84282e8c1 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 16:53:52 -0300 Subject: Changing remaining 'exit' to 'fatal' at rsync handler (#3721) --- handlers/rsync.in | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 884a885..cb72886 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -391,8 +391,7 @@ function rotate_long { local metadata if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." - exit + fatal "Skipping rotate of $backuproot as it doesn't exist." fi for rottype in daily weekly monthly; do @@ -509,7 +508,7 @@ function rotate_long_remote { now=\`date +%s\` if [ ! -d "$backuproot" ]; then - echo "Debug: skipping rotate of $backuproot as it doesn't exist." + echo "Fatal: skipping rotate of $backuproot as it doesn't exist." exit fi -- cgit v1.2.3 From ba3e37595d7a8fbf9a881ab0b9b14146eb91f3a2 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 17:00:58 -0300 Subject: Changing a 'fatal' to a 'warning' on rsync handler. Using 'warning' and 'return' instead of a fatal at rotate_long if backuproot doesn't exist. Using a warning ensures that the backup action can process the other includes and not just exit. --- handlers/rsync.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index cb72886..6fcda05 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -391,7 +391,8 @@ function rotate_long { local metadata if [ ! -d "$backuproot" ]; then - fatal "Skipping rotate of $backuproot as it doesn't exist." + warning "Skipping rotate of $backuproot as it doesn't exist." + return fi for rottype in daily weekly monthly; do -- cgit v1.2.3 From 5af9b963715efa186feacd0314a793184cc0eea3 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 17:25:34 -0300 Subject: Removing duplicated locking support (#3838) --- handlers/rsync.in | 44 -------------------------------------------- 1 file changed, 44 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 884a885..3378819 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -39,7 +39,6 @@ # keepdaily = for long storage format, specify the number of daily backup increments # keepweekly = for long storage format, specify the number of weekly backup increments # keepmonthly = for long storage format, specify the number of monthly backup increments -# lockfile = lockfile to be kept during backup execution # nicelevel = rsync command nice level # enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly # tmp = temp folder @@ -129,7 +128,6 @@ function eval_config { getconf keepdaily 5 getconf keepweekly 3 getconf keepmonthly 1 - getconf lockfile getconf nicelevel 0 getconf enable_mv_timestamp_bug no getconf tmp /tmp @@ -875,45 +873,6 @@ function test_connect { } -function set_lockfile { - - if [ ! -z "$lockfile" ]; then - mkdir -p `dirname $lockfile` - if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then - trap 'unset_lockfile' INT TERM EXIT - else - fatal "Could not create lockfile $lockfile, exiting" - fi - fi - -} - -function unset_lockfile { - - if [ ! -z "$lockfile" ]; then - $rm -f $lockfile || warning "Could not remove lockfile $lockfile" - fi - -} - -function check_lockfile { - - local pid process - - if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then - pid="`cat $lockfile`" - process="`ps --no-headers -o comm $pid`" - if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then - info "Another backup is running for $lockfile, skipping run" - exit - else - info "Found old lockfile $lockfile, removing it" - unset_lockfile - fi - fi - -} - function set_filelist { filelist_flag="" @@ -1103,8 +1062,6 @@ function end_mux { # the backup procedure eval_config -check_lockfile -set_lockfile set_rsync_options start_mux stop_services @@ -1136,7 +1093,6 @@ done mount_ro run_fsck start_services -unset_lockfile end_mux echo "Finnishing backup at `date`" >> $log -- cgit v1.2.3 From c43bac5911f8d97044b505466f21ed87f43d84fc Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 18:47:11 -0300 Subject: Ensure that a non-zero rsync exit status is caught by our handler (#3892) --- handlers/rsync.in | 3 +++ 1 file changed, 3 insertions(+) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 884a885..73507c8 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -1121,6 +1121,9 @@ for SECTION in $include; do set_filelist set_dest + # Ensure that a non-zero rsync exit status is caught by our handler + set -o pipefail + info "Syncing $SECTION on $dest_path..." debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log -- cgit v1.2.3 From fbf25018bea6b5cb630ee15b401b385d5e9f7084 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 21:03:37 -0300 Subject: Set/restore pipefail to not mess with other scripts (#3892) --- handlers/rsync.in | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 73507c8..85e1c99 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -1100,6 +1100,28 @@ function end_mux { } +function set_pipefail { + + # Save initial pipefail status for later restoration + if echo "$SHELLOPTS" | grep -q ":pipefail"; then + pipefail="-o" + else + pipefail="+o" + fi + + # Ensure that a non-zero rsync exit status is caught by our handler + set -o pipefail + +} + +function restore_pipefail { + + if [ ! -z "$pipefail" ]; then + set $pipefail pipefail + fi + +} + # the backup procedure eval_config @@ -1109,6 +1131,7 @@ set_rsync_options start_mux stop_services mount_rw +set_pipefail starttime="`date +%c%n%s`" echo "Starting backup at `echo $starttime | head -n 1`" >> $log @@ -1121,9 +1144,6 @@ for SECTION in $include; do set_filelist set_dest - # Ensure that a non-zero rsync exit status is caught by our handler - set -o pipefail - info "Syncing $SECTION on $dest_path..." debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log @@ -1136,6 +1156,7 @@ for SECTION in $include; do done +restore_pipefail mount_ro run_fsck start_services -- cgit v1.2.3 From b7a92b20f8c139fff7e0febc0275c25c5c254d94 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 10 May 2012 21:26:38 -0300 Subject: Fixing typos (#3868) --- handlers/rsync.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 31449c2..5d8c777 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -423,7 +423,7 @@ function rotate_long { fi # Validate created date if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then - warning "Invalid metatada $created. Skipping rotation." + warning "Invalid metadata $created. Skipping rotation." break fi cutoff_time=$(( now - (seconds*(i-1)) )) @@ -546,7 +546,7 @@ function rotate_long_remote { fi # Validate created date if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then - echo "Warning: Invalid metatada \$created. Skipping rotation." + echo "Warning: Invalid metadata \$created. Skipping rotation." break fi cutoff_time=\$(( now - (seconds*(i-1)) )) -- cgit v1.2.3 From 118d7587b6d4ad34224b7bf7370bdfbe941ffe3c Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 12 May 2012 21:48:43 -0300 Subject: Changing pipefail just when it's really needed --- handlers/rsync.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index 85e1c99..1f7fb4d 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -1131,7 +1131,6 @@ set_rsync_options start_mux stop_services mount_rw -set_pipefail starttime="`date +%c%n%s`" echo "Starting backup at `echo $starttime | head -n 1`" >> $log @@ -1146,17 +1145,18 @@ for SECTION in $include; do info "Syncing $SECTION on $dest_path..." debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path + set_pipefail $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log if [ "$?" != "0" ]; then fatal "Rsync error when trying to transfer $SECTION" fi + restore_pipefail update_metadata done -restore_pipefail mount_ro run_fsck start_services -- cgit v1.2.3 From 7c66b699f5820a9fab997a98d95e2852dd21a262 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Mon, 14 May 2012 16:30:02 +0200 Subject: Always use explicit sed's -e. Hopefully this fixes http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=656968#31 and future incarnations of this class of bugs. --- handlers/rsync.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'handlers/rsync.in') diff --git a/handlers/rsync.in b/handlers/rsync.in index c5c7cac..2d00267 100644 --- a/handlers/rsync.in +++ b/handlers/rsync.in @@ -275,7 +275,7 @@ function eval_config { mv=move_files fi - excludes=`echo "$exclude" | @SED@ "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"` + excludes=`echo "$exclude" | @SED@ -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"` } @@ -406,7 +406,7 @@ function rotate_long { fi # Rotate the current list of backups, if we can. - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 for (( i=$oldest; i > 0; i-- )); do if [ -d $dir.$i ]; then @@ -468,7 +468,7 @@ function rotate_long { for rottype in daily weekly monthly; do max=$((keep${rottype}+1)) dir="$backuproot/$rottype" - oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1` [ "$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=$oldest; i >= $max; i-- )); do @@ -529,7 +529,7 @@ function rotate_long_remote { fi # Rotate the current list of backups, if we can. - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 for (( i=\$oldest; i > 0; i-- )); do if [ -d \$dir.\$i ]; then @@ -591,7 +591,7 @@ function rotate_long_remote { for rottype in daily weekly monthly; do max=\$((keep\${rottype}+1)) dir="$backuproot/\$rottype" - oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\` + oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\` [ "\$oldest" == "" ] && oldest=0 # if we've rotated the last backup off the stack, remove it. for (( i=\$oldest; i >= \$max; i-- )); do -- cgit v1.2.3