diff options
Diffstat (limited to 'handlers')
| -rw-r--r-- | handlers/wget | 527 |
1 files changed, 396 insertions, 131 deletions
diff --git a/handlers/wget b/handlers/wget index 79aa22c..c5001fa 100644 --- a/handlers/wget +++ b/handlers/wget @@ -16,6 +16,7 @@ # read_only = set to 1 if $partition is mounted read-only # mountpoint = backup partition mountpoint or backup main folder # backupdir = folder relative do $mountpoint where the backup should be stored +# format = specify backup storage format: short, long or mirror (i.e, no rotations) # days = number of backup increments (min = 5) # lockfile = lockfile to be kept during backup execution # nicelevel = wget command nice level @@ -26,7 +27,7 @@ # wget = wget program # wget_options = wget command options # url = remote data url -# bandwidthlimit = set a bandwidth limit in kbps (remote source only) +# bandwidthlimit = set a badnwidth limit in kbps (remote source only) # # [destination] # folder = local folder @@ -40,69 +41,221 @@ # mv = mv command # fsck = fsck command # -# TODO: Daily, weekly and monthly snapshot rotation (like the one present on maildir handler). +# TODO: Test daily, weekly and monthly snapshot rotation (like rsync handler). # -# config file evaluation - -setsection system -getconf rm rm -getconf cp cp -getconf touch touch -getconf mv mv -getconf fsck fsck - -setsection general -getconf log /var/log/backup/wget.log -getconf partition -getconf fscheck -getconf read_only -getconf mountpoint -getconf backupdir -getconf rotate -getconf days -getconf lockfile -getconf nicelevel 0 -getconf enable_mv_timestamp_bug no -getconf tmp /tmp - -setsection source -getconf wget wget -getconf wget_options -getconf url -getconf bandwidthlimit - -setsection destination -getconf folder - -# function definitions - -function rotate { - - if [[ "$2" < 4 ]]; then - error "Rotate: minimum of 4 rotations" - exit 1 - fi +# function definitions + +function eval_config { + + # system section + + setsection system + getconf rm rm + getconf cp cp + getconf touch touch + getconf mv mv + getconf fsck fsck + + # general section + setsection general + getconf log /var/log/backup/wget.log + getconf partition + getconf fscheck + getconf read_only + getconf mountpoint + getconf backupdir + getconf format short + getconf rotate + getconf days + getconf keepdaily 5 + getconf keepweekly 3 + getconf keepmonthly 1 + getconf lockfile + getconf nicelevel 0 + getconf enable_mv_timestamp_bug no + getconf tmp /tmp + + # source section + + setsection source + getconf wget wget + getconf wget_options + getconf url + getconf bandwidthlimit + + # destination section + + setsection destination + getconf folder + + # Just local backups are supported by now + dest="local" + + backupdir="$mountpoint/$backupdir" + + if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then + error "Backupdir $backupdir does not exist" + exit 1 + fi + + if [ "$format" == "short" ]; then + if [ -z "$days" ]; then + keep="4" + else + keep=$[$days - 1] + fi + fi + + if [ ! -z "$nicelevel" ]; then + nice="nice -n $nicelevel" + else + nice="" + fi + + if [ $enable_mv_timestamp_bug == "yes" ]; then + mv=move_files + fi + + if [ ! -z "$bandwidthlimit" ]; then + limit_rate="--limit-rate=$bandwidthlimit""k" + fi - if [ -d $1.$2 ]; then - $nice $mv /$1.$2 /$1.tmp - fi +} - for ((n=$[$2 - 1]; n >= 0; n--)); do - if [ -d $1.$n ]; then - dest=$[$n + 1] - $nice $mv /$1.$n /$1.$dest - $touch /$1.$dest - fi - done +# using rotate_short from rsync handler +function rotate_short { + + local dest + local folder="$1" + local keep="$2" + local metadata="`dirname $folder`/metadata" + + if [[ "$keep" -lt 4 ]]; then + error "Rotate: minimum of 4 rotations" + exit 1 + fi + + if [ -d $folder.$keep ]; then + $nice $mv /$folder.$keep /$folder.tmp + fi + + for ((n=$[$keep - 1]; n >= 0; n--)); do + if [ -d $folder.$n ]; then + dest=$[$n + 1] + $nice $mv /$folder.$n /$folder.$dest + $touch /$folder.$dest + mkdir -p $metadata/`basename $folder`.$dest + date +%c%n%s > $metadata/`basename $folder`.$dest/rotated + fi + done + + if [ -d $folder.tmp ]; then + $nice $mv /$folder.tmp /$folder.0 + fi + + if [ -d $folder.1 ]; then + $nice $cp -alf /$folder.1/. /$folder.0 + fi - if [ -d $1.tmp ]; then - $nice $mv /$1.tmp /$1.0 - fi +} - if [ -d $1.1 ]; then - $nice $cp -alf /$1.1/. /$1.0 - fi +# using rotate_long from rsync handler +function rotate_long { + + backuproot="$1" + seconds_daily=86400 + seconds_weekly=604800 + seconds_monthly=2628000 + keepdaily=$keepdaily + keepweekly=$keepweekly + keepmonthly=$keepmonthly + now=`date +%s` + + local metadata + + if [ ! -d "$backuproot" ]; then + echo "Debug: skipping rotate of $backuproot as it doesn't exist." + exit + fi + + for rottype in daily weekly monthly; do + seconds=$((seconds_${rottype})) + + dir="$backuproot/$rottype" + metadata="$backuproot/metadata/$rottype.1" + mkdir -p $metadata + if [ ! -d $dir.1 ]; then + echo "Debug: $dir.1 does not exist, skipping." + continue 1 + elif [ ! -f $metadata/created ] && [ ! -f $metadata/rotated ]; then + echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation." + continue 1 + fi + + # Rotate the current list of backups, if we can. + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + [ "$oldest" == "" ] && oldest=0 + for (( i=$oldest; i > 0; i-- )); do + if [ -d $dir.$i ]; then + if [ -f $metadata/created ]; then + created=`tail -1 $metadata/created` + elif [ -f $metadata/rotated ]; then + created=`tail -1 $metadata/rotated` + else + created=0 + fi + cutoff_time=$(( now - (seconds*(i-1)) )) + if [ ! $created -gt $cutoff_time ]; then + next=$(( i + 1 )) + if [ ! -d $dir.$next ]; then + echo "Debug: $rottype.$i --> $rottype.$next" + $nice mv $dir.$i $dir.$next + mkdir -p $backuproot/metadata/$rottype.$next + date +%c%n%s > $backuproot/metadata/$rottype.$next/rotated + else + echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists." + fi + else + echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)." + fi + fi + done + done + + max=$((keepdaily+1)) + if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then + echo "Debug: daily.$max --> weekly.1" + $nice mv $backuproot/daily.$max $backuproot/weekly.1 + mkdir -p $backuproot/metadata/weekly.1 + date +%c%n%s > $backuproot/metadata/weekly.1/rotated + fi + + max=$((keepweekly+1)) + if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then + echo "Debug: weekly.$max --> monthly.1" + $nice mv $backuproot/weekly.$max $backuproot/monthly.1 + mkdir -p $backuproot/metadata/monthly.1 + date +%c%n%s > $backuproot/metadata/monthly.1/rotated + fi + + for rottype in daily weekly monthly; do + max=$((keep${rottype}+1)) + dir="$backuproot/$rottype" + oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1` + [ "$oldest" == "" ] && oldest=0 + # if we've rotated the last backup off the stack, remove it. + for (( i=$oldest; i >= $max; i-- )); do + if [ -d $dir.$i ]; then + if [ -d $backuproot/rotate.tmp ]; then + echo "Debug: removing rotate.tmp" + $nice rm -rf $backuproot/rotate.tmp + fi + echo "Debug: moving $rottype.$i to rotate.tmp" + $nice mv $dir.$i $backuproot/rotate.tmp + fi + done + done } @@ -114,107 +267,219 @@ function move_files { $rm $ref; } -backupdir="$mountpoint/$backupdir" - -# does $backupdir exists? +# using setup_long_dirs from rsync handler +function setup_long_dirs { + + local destdir=$1 + local backuptype=$2 + local dir="$destdir/$backuptype" + local tmpdir="$destdir/rotate.tmp" + local metadata="$destdir/metadata/$backuptype.1" + + if [ ! -d $destdir ]; then + echo "Creating destination directory $destdir..." + mkdir -p $destdir + fi + + if [ -d $dir.1 ]; then + if [ -f $metadata/created ]; then + echo "Warning: $dir.1 already exists. Overwriting contents." + else + echo "Warning: we seem to be resuming a partially written $dir.1" + fi + else + if [ -d $tmpdir ]; then + mv $tmpdir $dir.1 + if [ $? == 1 ]; then + echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host" + exit 1 + fi + else + mkdir --parents $dir.1 + if [ $? == 1 ]; then + echo "Fatal: could not create directory $dir.1 on host $host" + exit 1 + fi + fi + if [ -d $dir.2 ]; then + echo "Debug: update links $backuptype.2 --> $backuptype.1" + cp -alf $dir.2/. $dir.1 + #if [ $? == 1 ]; then + # echo "Fatal: could not create hard links to $dir.1 on host $host" + # exit 1 + #fi + fi + fi + [ -f $metadata/created ] && rm $metadata/created + [ -f $metadata/rotated ] && rm $metadata/rotated -if [ ! -d "$backupdir" ]; then - error "Backupdir $backupdir does not exist" - exit 1 -fi +} -# setup number of increments +# using prepare_storage from rsync handler +function prepare_storage { -if [ -z "$days" ]; then - keep="4" -else - keep=$[$days - 1] -fi + section="`basename $SECTION`" -# lockfile setup + if [ "$format" == "short" ]; then -if [ ! -z "$lockfile" ]; then - $touch $lockfile || warning "Could not create lockfile $lockfile" -fi + suffix="$section.0" + info "Rotating $backupdir/$SECTION..." + echo "Rotating $backupdir/$SECTION..." >> $log -# nicelevel setup + if [ "$dest" == "remote" ]; then + rotate_short_remote $backupdir/$SECTION/$section $keep + else + rotate_short $backupdir/$SECTION/$section $keep + if [ ! -d "$backupdir/$SECTION/$section.0" ]; then + mkdir -p $backupdir/$SECTION/$section.0 + fi + fi + + elif [ "$format" == "long" ]; then + + if [ $keepdaily -gt 0 ]; then + btype=daily + elif [ $keepweekly -gt 0 ]; then + btype=weekly + elif [ $keepmonthly -gt 0 ]; then + btype=monthly + else + fatal "keeping no backups"; + exit 1 + fi + + suffix="$btype.1" + info "Rotating $backupdir/$SECTION/..." + echo "Rotating $backupdir/$SECTION/..." >> $log + + if [ "$dest" == "remote" ]; then + rotate_long_remote $backupdir/$SECTION + setup_long_dirs_remote $backupdir/$SECTION $btype + else + rotate_long $backupdir/$SECTION + setup_long_dirs $backupdir/$SECTION $btype + fi + + elif [ "$format" == "mirror" ]; then + suffix="" + else + fatal "Invalid backup format $format" + exit 1 + fi -if [ ! -z "$nicelevel" ]; then - nice="nice -n $nicelevel" -else - nice="" -fi +} -# set mv procedure +# using mount_ro from rsync handler +function mount_ro { -if [ $enable_mv_timestamp_bug == "yes" ]; then - mv=move_files -fi + # remount backup destination as read-only -# set excludes + if [ "$dest" == "local" ]; then + if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then + mount -o remount,ro $mountpoint + fi + fi -for path in $exclude; do - EXCLUDES="$EXCLUDES --exclude=$path" -done +} -echo "Starting backup at `date`" >> $log +# usind run_fsck from rsync handler +function run_fsck { -# mount backup destination folder as read-write + # check partition for errors -if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then - if [ -d "$mountpoint" ]; then - mount -o remount,rw $mountpoint + if [ "$dest" == "local" ]; then + if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then + umount $mountpoint if (($?)); then - error "Could not mount $mountpoint" - exit 1 + warning "Could not umount $mountpoint to run fsck" + else + $nice $fsck -v -y $partition >> $log + mount $mountpoint fi - fi -fi + fi + fi -# the backup procedure +} + +# using mount_rw from rsync handler +function mount_rw { + + # mount backup destination folder as read-write + + if [ "$dest" == "local" ]; then + if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then + if [ -d "$mountpoint" ]; then + mount -o remount,rw $mountpoint + if (($?)); then + error "Could not mount $mountpoint" + exit 1 + fi + fi + fi + fi -if [ ! -d "$backupdir/$folder/$folder.0" ]; then - mkdir -p $backupdir/$folder/$folder.0 -fi +} -info "Rotating $backupdir/$folder/$folder..." -echo "Rotating $backupdir/$folder/$folder..." >> $log -rotate $backupdir/$folder/$folder $keep -info "Wget'ing $SECTION on $backupdir/$folder/$folder.0..." +# using set_lockfile from rsync handler +function set_lockfile { -if [ ! -z "$bandwidth" ]; then - limit_rate="--limit-rate=$bandwidth""k" -fi + if [ ! -z "$lockfile" ]; then + $touch $lockfile || warning "Could not create lockfile $lockfile" + fi -cd $backupdir/$folder/$folder.0 -wget $wget_options $limit-rate -r -c -N -e robots=off $url -cd - +} -$touch $backupdir/$folder/$folder.0 +# using unset_lockfile from rsync handler +function unset_lockfile { -# remount backup destination as read-only + if [ ! -z "$lockfile" ]; then + $rm $lockfile || warning "Could not remove lockfile $lockfile" + fi -if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then - mount -o remount,ro $mountpoint -fi +} -# check partition for errors +# using set_dest from rsync handler +function set_dest { -if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then - umount $mountpoint - if (($?)); then - warning "Could not umount $mountpoint to run fsck" - else - $nice $fsck -v -y $partition >> $log - mount $mountpoint - fi -fi + if [ "$dest" == "local" ]; then + dest_path="$backupdir/$SECTION/$suffix/" + else + if [ "$protocol" == "rsync" ]; then + dest_path="rsync://$user@$host:$port/$backupdir/$SECTION/$suffix/" + else + dest_path="$user@$host:$backupdir/$SECTION/$suffix/" + fi + fi -# removes the lockfile +} + +function do_backup { + + # the backup procedure + + SECTION="$folder" + prepare_storage + set_dest + + cwd="`pwd`" + cd $dest_path + $nice $wget $wget_options $limit_rate -r -c -N -e robots=off $url + cd $cwd + + $touch $dest_path + +} + +eval_config +set_lockfile +mount_rw + +echo "Starting backup at `date`" >> $log -if [ ! -z "$lockfile" ]; then - $rm $lockfile || warning "Could not remove lockfile $lockfile" -fi +do_backup +mount_ro +run_fsck +unset_lockfile echo "Finnishing backup at `date`" >> $log |