aboutsummaryrefslogtreecommitdiff
path: root/handlers/wget
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2012-02-27 19:06:08 -0300
committerSilvio Rhatto <rhatto@riseup.net>2012-02-27 19:06:08 -0300
commit7f220900dc3e9b6af26c775a6415ef9731ab0ea7 (patch)
tree4424162e247be39ce7cae22bb31291bded422783 /handlers/wget
parent1f367b40cdb0c790b9c3a1e45fceb4ee79e93c31 (diff)
downloadbackupninja-feature/wget.tar.gz
backupninja-feature/wget.tar.bz2
Major rewrite of wget handler (re-commiting from old repo)feature/wget
Diffstat (limited to 'handlers/wget')
-rw-r--r--handlers/wget527
1 files changed, 396 insertions, 131 deletions
diff --git a/handlers/wget b/handlers/wget
index 79aa22c..c5001fa 100644
--- a/handlers/wget
+++ b/handlers/wget
@@ -16,6 +16,7 @@
# read_only = set to 1 if $partition is mounted read-only
# mountpoint = backup partition mountpoint or backup main folder
# backupdir = folder relative do $mountpoint where the backup should be stored
+# format = specify backup storage format: short, long or mirror (i.e, no rotations)
# days = number of backup increments (min = 5)
# lockfile = lockfile to be kept during backup execution
# nicelevel = wget command nice level
@@ -26,7 +27,7 @@
# wget = wget program
# wget_options = wget command options
# url = remote data url
-# bandwidthlimit = set a bandwidth limit in kbps (remote source only)
+# bandwidthlimit = set a badnwidth limit in kbps (remote source only)
#
# [destination]
# folder = local folder
@@ -40,69 +41,221 @@
# mv = mv command
# fsck = fsck command
#
-# TODO: Daily, weekly and monthly snapshot rotation (like the one present on maildir handler).
+# TODO: Test daily, weekly and monthly snapshot rotation (like rsync handler).
#
-# config file evaluation
-
-setsection system
-getconf rm rm
-getconf cp cp
-getconf touch touch
-getconf mv mv
-getconf fsck fsck
-
-setsection general
-getconf log /var/log/backup/wget.log
-getconf partition
-getconf fscheck
-getconf read_only
-getconf mountpoint
-getconf backupdir
-getconf rotate
-getconf days
-getconf lockfile
-getconf nicelevel 0
-getconf enable_mv_timestamp_bug no
-getconf tmp /tmp
-
-setsection source
-getconf wget wget
-getconf wget_options
-getconf url
-getconf bandwidthlimit
-
-setsection destination
-getconf folder
-
-# function definitions
-
-function rotate {
-
- if [[ "$2" < 4 ]]; then
- error "Rotate: minimum of 4 rotations"
- exit 1
- fi
+# function definitions
+
+function eval_config {
+
+ # system section
+
+ setsection system
+ getconf rm rm
+ getconf cp cp
+ getconf touch touch
+ getconf mv mv
+ getconf fsck fsck
+
+ # general section
+ setsection general
+ getconf log /var/log/backup/wget.log
+ getconf partition
+ getconf fscheck
+ getconf read_only
+ getconf mountpoint
+ getconf backupdir
+ getconf format short
+ getconf rotate
+ getconf days
+ getconf keepdaily 5
+ getconf keepweekly 3
+ getconf keepmonthly 1
+ getconf lockfile
+ getconf nicelevel 0
+ getconf enable_mv_timestamp_bug no
+ getconf tmp /tmp
+
+ # source section
+
+ setsection source
+ getconf wget wget
+ getconf wget_options
+ getconf url
+ getconf bandwidthlimit
+
+ # destination section
+
+ setsection destination
+ getconf folder
+
+ # Just local backups are supported by now
+ dest="local"
+
+ backupdir="$mountpoint/$backupdir"
+
+ if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then
+ error "Backupdir $backupdir does not exist"
+ exit 1
+ fi
+
+ if [ "$format" == "short" ]; then
+ if [ -z "$days" ]; then
+ keep="4"
+ else
+ keep=$[$days - 1]
+ fi
+ fi
+
+ if [ ! -z "$nicelevel" ]; then
+ nice="nice -n $nicelevel"
+ else
+ nice=""
+ fi
+
+ if [ $enable_mv_timestamp_bug == "yes" ]; then
+ mv=move_files
+ fi
+
+ if [ ! -z "$bandwidthlimit" ]; then
+ limit_rate="--limit-rate=$bandwidthlimit""k"
+ fi
- if [ -d $1.$2 ]; then
- $nice $mv /$1.$2 /$1.tmp
- fi
+}
- for ((n=$[$2 - 1]; n >= 0; n--)); do
- if [ -d $1.$n ]; then
- dest=$[$n + 1]
- $nice $mv /$1.$n /$1.$dest
- $touch /$1.$dest
- fi
- done
+# using rotate_short from rsync handler
+function rotate_short {
+
+ local dest
+ local folder="$1"
+ local keep="$2"
+ local metadata="`dirname $folder`/metadata"
+
+ if [[ "$keep" -lt 4 ]]; then
+ error "Rotate: minimum of 4 rotations"
+ exit 1
+ fi
+
+ if [ -d $folder.$keep ]; then
+ $nice $mv /$folder.$keep /$folder.tmp
+ fi
+
+ for ((n=$[$keep - 1]; n >= 0; n--)); do
+ if [ -d $folder.$n ]; then
+ dest=$[$n + 1]
+ $nice $mv /$folder.$n /$folder.$dest
+ $touch /$folder.$dest
+ mkdir -p $metadata/`basename $folder`.$dest
+ date +%c%n%s > $metadata/`basename $folder`.$dest/rotated
+ fi
+ done
+
+ if [ -d $folder.tmp ]; then
+ $nice $mv /$folder.tmp /$folder.0
+ fi
+
+ if [ -d $folder.1 ]; then
+ $nice $cp -alf /$folder.1/. /$folder.0
+ fi
- if [ -d $1.tmp ]; then
- $nice $mv /$1.tmp /$1.0
- fi
+}
- if [ -d $1.1 ]; then
- $nice $cp -alf /$1.1/. /$1.0
- fi
+# using rotate_long from rsync handler
+function rotate_long {
+
+ backuproot="$1"
+ seconds_daily=86400
+ seconds_weekly=604800
+ seconds_monthly=2628000
+ keepdaily=$keepdaily
+ keepweekly=$keepweekly
+ keepmonthly=$keepmonthly
+ now=`date +%s`
+
+ local metadata
+
+ if [ ! -d "$backuproot" ]; then
+ echo "Debug: skipping rotate of $backuproot as it doesn't exist."
+ exit
+ fi
+
+ for rottype in daily weekly monthly; do
+ seconds=$((seconds_${rottype}))
+
+ dir="$backuproot/$rottype"
+ metadata="$backuproot/metadata/$rottype.1"
+ mkdir -p $metadata
+ if [ ! -d $dir.1 ]; then
+ echo "Debug: $dir.1 does not exist, skipping."
+ continue 1
+ elif [ ! -f $metadata/created ] && [ ! -f $metadata/rotated ]; then
+ echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
+ continue 1
+ fi
+
+ # Rotate the current list of backups, if we can.
+ oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+ [ "$oldest" == "" ] && oldest=0
+ for (( i=$oldest; i > 0; i-- )); do
+ if [ -d $dir.$i ]; then
+ if [ -f $metadata/created ]; then
+ created=`tail -1 $metadata/created`
+ elif [ -f $metadata/rotated ]; then
+ created=`tail -1 $metadata/rotated`
+ else
+ created=0
+ fi
+ cutoff_time=$(( now - (seconds*(i-1)) ))
+ if [ ! $created -gt $cutoff_time ]; then
+ next=$(( i + 1 ))
+ if [ ! -d $dir.$next ]; then
+ echo "Debug: $rottype.$i --> $rottype.$next"
+ $nice mv $dir.$i $dir.$next
+ mkdir -p $backuproot/metadata/$rottype.$next
+ date +%c%n%s > $backuproot/metadata/$rottype.$next/rotated
+ else
+ echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
+ fi
+ else
+ echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
+ fi
+ fi
+ done
+ done
+
+ max=$((keepdaily+1))
+ if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
+ echo "Debug: daily.$max --> weekly.1"
+ $nice mv $backuproot/daily.$max $backuproot/weekly.1
+ mkdir -p $backuproot/metadata/weekly.1
+ date +%c%n%s > $backuproot/metadata/weekly.1/rotated
+ fi
+
+ max=$((keepweekly+1))
+ if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
+ echo "Debug: weekly.$max --> monthly.1"
+ $nice mv $backuproot/weekly.$max $backuproot/monthly.1
+ mkdir -p $backuproot/metadata/monthly.1
+ date +%c%n%s > $backuproot/metadata/monthly.1/rotated
+ fi
+
+ for rottype in daily weekly monthly; do
+ max=$((keep${rottype}+1))
+ dir="$backuproot/$rottype"
+ oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+ [ "$oldest" == "" ] && oldest=0
+ # if we've rotated the last backup off the stack, remove it.
+ for (( i=$oldest; i >= $max; i-- )); do
+ if [ -d $dir.$i ]; then
+ if [ -d $backuproot/rotate.tmp ]; then
+ echo "Debug: removing rotate.tmp"
+ $nice rm -rf $backuproot/rotate.tmp
+ fi
+ echo "Debug: moving $rottype.$i to rotate.tmp"
+ $nice mv $dir.$i $backuproot/rotate.tmp
+ fi
+ done
+ done
}
@@ -114,107 +267,219 @@ function move_files {
$rm $ref;
}
-backupdir="$mountpoint/$backupdir"
-
-# does $backupdir exists?
+# using setup_long_dirs from rsync handler
+function setup_long_dirs {
+
+ local destdir=$1
+ local backuptype=$2
+ local dir="$destdir/$backuptype"
+ local tmpdir="$destdir/rotate.tmp"
+ local metadata="$destdir/metadata/$backuptype.1"
+
+ if [ ! -d $destdir ]; then
+ echo "Creating destination directory $destdir..."
+ mkdir -p $destdir
+ fi
+
+ if [ -d $dir.1 ]; then
+ if [ -f $metadata/created ]; then
+ echo "Warning: $dir.1 already exists. Overwriting contents."
+ else
+ echo "Warning: we seem to be resuming a partially written $dir.1"
+ fi
+ else
+ if [ -d $tmpdir ]; then
+ mv $tmpdir $dir.1
+ if [ $? == 1 ]; then
+ echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
+ exit 1
+ fi
+ else
+ mkdir --parents $dir.1
+ if [ $? == 1 ]; then
+ echo "Fatal: could not create directory $dir.1 on host $host"
+ exit 1
+ fi
+ fi
+ if [ -d $dir.2 ]; then
+ echo "Debug: update links $backuptype.2 --> $backuptype.1"
+ cp -alf $dir.2/. $dir.1
+ #if [ $? == 1 ]; then
+ # echo "Fatal: could not create hard links to $dir.1 on host $host"
+ # exit 1
+ #fi
+ fi
+ fi
+ [ -f $metadata/created ] && rm $metadata/created
+ [ -f $metadata/rotated ] && rm $metadata/rotated
-if [ ! -d "$backupdir" ]; then
- error "Backupdir $backupdir does not exist"
- exit 1
-fi
+}
-# setup number of increments
+# using prepare_storage from rsync handler
+function prepare_storage {
-if [ -z "$days" ]; then
- keep="4"
-else
- keep=$[$days - 1]
-fi
+ section="`basename $SECTION`"
-# lockfile setup
+ if [ "$format" == "short" ]; then
-if [ ! -z "$lockfile" ]; then
- $touch $lockfile || warning "Could not create lockfile $lockfile"
-fi
+ suffix="$section.0"
+ info "Rotating $backupdir/$SECTION..."
+ echo "Rotating $backupdir/$SECTION..." >> $log
-# nicelevel setup
+ if [ "$dest" == "remote" ]; then
+ rotate_short_remote $backupdir/$SECTION/$section $keep
+ else
+ rotate_short $backupdir/$SECTION/$section $keep
+ if [ ! -d "$backupdir/$SECTION/$section.0" ]; then
+ mkdir -p $backupdir/$SECTION/$section.0
+ fi
+ fi
+
+ elif [ "$format" == "long" ]; then
+
+ if [ $keepdaily -gt 0 ]; then
+ btype=daily
+ elif [ $keepweekly -gt 0 ]; then
+ btype=weekly
+ elif [ $keepmonthly -gt 0 ]; then
+ btype=monthly
+ else
+ fatal "keeping no backups";
+ exit 1
+ fi
+
+ suffix="$btype.1"
+ info "Rotating $backupdir/$SECTION/..."
+ echo "Rotating $backupdir/$SECTION/..." >> $log
+
+ if [ "$dest" == "remote" ]; then
+ rotate_long_remote $backupdir/$SECTION
+ setup_long_dirs_remote $backupdir/$SECTION $btype
+ else
+ rotate_long $backupdir/$SECTION
+ setup_long_dirs $backupdir/$SECTION $btype
+ fi
+
+ elif [ "$format" == "mirror" ]; then
+ suffix=""
+ else
+ fatal "Invalid backup format $format"
+ exit 1
+ fi
-if [ ! -z "$nicelevel" ]; then
- nice="nice -n $nicelevel"
-else
- nice=""
-fi
+}
-# set mv procedure
+# using mount_ro from rsync handler
+function mount_ro {
-if [ $enable_mv_timestamp_bug == "yes" ]; then
- mv=move_files
-fi
+ # remount backup destination as read-only
-# set excludes
+ if [ "$dest" == "local" ]; then
+ if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
+ mount -o remount,ro $mountpoint
+ fi
+ fi
-for path in $exclude; do
- EXCLUDES="$EXCLUDES --exclude=$path"
-done
+}
-echo "Starting backup at `date`" >> $log
+# usind run_fsck from rsync handler
+function run_fsck {
-# mount backup destination folder as read-write
+ # check partition for errors
-if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
- if [ -d "$mountpoint" ]; then
- mount -o remount,rw $mountpoint
+ if [ "$dest" == "local" ]; then
+ if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
+ umount $mountpoint
if (($?)); then
- error "Could not mount $mountpoint"
- exit 1
+ warning "Could not umount $mountpoint to run fsck"
+ else
+ $nice $fsck -v -y $partition >> $log
+ mount $mountpoint
fi
- fi
-fi
+ fi
+ fi
-# the backup procedure
+}
+
+# using mount_rw from rsync handler
+function mount_rw {
+
+ # mount backup destination folder as read-write
+
+ if [ "$dest" == "local" ]; then
+ if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
+ if [ -d "$mountpoint" ]; then
+ mount -o remount,rw $mountpoint
+ if (($?)); then
+ error "Could not mount $mountpoint"
+ exit 1
+ fi
+ fi
+ fi
+ fi
-if [ ! -d "$backupdir/$folder/$folder.0" ]; then
- mkdir -p $backupdir/$folder/$folder.0
-fi
+}
-info "Rotating $backupdir/$folder/$folder..."
-echo "Rotating $backupdir/$folder/$folder..." >> $log
-rotate $backupdir/$folder/$folder $keep
-info "Wget'ing $SECTION on $backupdir/$folder/$folder.0..."
+# using set_lockfile from rsync handler
+function set_lockfile {
-if [ ! -z "$bandwidth" ]; then
- limit_rate="--limit-rate=$bandwidth""k"
-fi
+ if [ ! -z "$lockfile" ]; then
+ $touch $lockfile || warning "Could not create lockfile $lockfile"
+ fi
-cd $backupdir/$folder/$folder.0
-wget $wget_options $limit-rate -r -c -N -e robots=off $url
-cd -
+}
-$touch $backupdir/$folder/$folder.0
+# using unset_lockfile from rsync handler
+function unset_lockfile {
-# remount backup destination as read-only
+ if [ ! -z "$lockfile" ]; then
+ $rm $lockfile || warning "Could not remove lockfile $lockfile"
+ fi
-if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
- mount -o remount,ro $mountpoint
-fi
+}
-# check partition for errors
+# using set_dest from rsync handler
+function set_dest {
-if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
- umount $mountpoint
- if (($?)); then
- warning "Could not umount $mountpoint to run fsck"
- else
- $nice $fsck -v -y $partition >> $log
- mount $mountpoint
- fi
-fi
+ if [ "$dest" == "local" ]; then
+ dest_path="$backupdir/$SECTION/$suffix/"
+ else
+ if [ "$protocol" == "rsync" ]; then
+ dest_path="rsync://$user@$host:$port/$backupdir/$SECTION/$suffix/"
+ else
+ dest_path="$user@$host:$backupdir/$SECTION/$suffix/"
+ fi
+ fi
-# removes the lockfile
+}
+
+function do_backup {
+
+ # the backup procedure
+
+ SECTION="$folder"
+ prepare_storage
+ set_dest
+
+ cwd="`pwd`"
+ cd $dest_path
+ $nice $wget $wget_options $limit_rate -r -c -N -e robots=off $url
+ cd $cwd
+
+ $touch $dest_path
+
+}
+
+eval_config
+set_lockfile
+mount_rw
+
+echo "Starting backup at `date`" >> $log
-if [ ! -z "$lockfile" ]; then
- $rm $lockfile || warning "Could not remove lockfile $lockfile"
-fi
+do_backup
+mount_ro
+run_fsck
+unset_lockfile
echo "Finnishing backup at `date`" >> $log