path: root/handlers/wget

# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:
#
# backupninja handler to do incremental backups using
# wget and hardlinks, based on rsync handler
#
# feedback: rhatto at riseup.net | gpl
#
# Config file options
# -------------------
#
#   [general]
#   log = wget log file
#   partition = partition where the backup lives
#   fscheck = set to 1 if fsck should run on $partition after the backup is made
#   read_only = set to 1 if $partition is mounted read-only
#   mountpoint = backup partition mountpoint or backup main folder
#   backupdir = folder relative do $mountpoint where the backup should be stored
#   format = specify backup storage format: short, long or mirror (i.e, no rotations) 
#   days = number of backup increments (min = 5)
#   lockfile = lockfile to be kept during backup execution
#   nicelevel = wget command nice level
#   enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
#   tmp = temp folder
#
#   [source]
#   wget = wget program
#   wget_options = wget command options
#   url = remote data url
#   bandwidthlimit = set a badnwidth limit in kbps (remote source only)
#
#   [destination]
#   folder = local folder
#
# You can also specify some system comands if you don't want the default system values:
#
#   [system]
#   rm = rm command
#   cp = cp command
#   touch = touch command
#   mv = mv command
#   fsck = fsck command
#
# TODO: Test daily, weekly and monthly snapshot rotation (like rsync handler).
#

# function definitions 

function eval_config { 

  # system section

  setsection system
  getconf rm rm
  getconf cp cp
  getconf touch touch
  getconf mv mv
  getconf fsck fsck
  
  # general section
  setsection general
  getconf log /var/log/backup/wget.log
  getconf partition
  getconf fscheck
  getconf read_only
  getconf mountpoint
  getconf backupdir
  getconf format short
  getconf rotate
  getconf days
  getconf keepdaily 5
  getconf keepweekly 3
  getconf keepmonthly 1
  getconf lockfile
  getconf nicelevel 0
  getconf enable_mv_timestamp_bug no
  getconf tmp /tmp
  
  # source section

  setsection source
  getconf wget wget
  getconf wget_options
  getconf url
  getconf bandwidthlimit
  
  # destination section

  setsection destination
  getconf folder

  # Just local backups are supported by now
  dest="local"

  backupdir="$mountpoint/$backupdir"

  if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then 
    error "Backupdir $backupdir does not exist"
    exit 1
  fi

  if [ "$format" == "short" ]; then
     if [ -z "$days" ]; then
        keep="4"
     else
        keep=$[$days - 1]
     fi
  fi

  if [ ! -z "$nicelevel" ]; then 
    nice="nice -n $nicelevel"
  else 
    nice=""
  fi

  if [ $enable_mv_timestamp_bug == "yes" ]; then
    mv=move_files
  fi

  if [ ! -z "$bandwidthlimit" ]; then
     limit_rate="--limit-rate=$bandwidthlimit""k"
  fi

}

# using rotate_short from rsync handler
function rotate_short {

  local dest
  local folder="$1"
  local keep="$2"
  local metadata="`dirname $folder`/metadata"

  if [[ "$keep" -lt 4 ]]; then
    error "Rotate: minimum of 4 rotations"
    exit 1
  fi

  if [ -d $folder.$keep ]; then
    $nice $mv /$folder.$keep /$folder.tmp
  fi

  for ((n=$[$keep - 1]; n >= 0; n--)); do
    if [ -d $folder.$n ]; then
      dest=$[$n + 1]
      $nice $mv /$folder.$n /$folder.$dest
      $touch /$folder.$dest
      mkdir -p $metadata/`basename $folder`.$dest
      date +%c%n%s > $metadata/`basename $folder`.$dest/rotated
    fi
  done

  if [ -d $folder.tmp ]; then
    $nice $mv /$folder.tmp /$folder.0
  fi

  if [ -d $folder.1 ]; then
    $nice $cp -alf /$folder.1/. /$folder.0
  fi

}

# using rotate_long from rsync handler
function rotate_long {

  backuproot="$1"
  seconds_daily=86400
  seconds_weekly=604800
  seconds_monthly=2628000
  keepdaily=$keepdaily
  keepweekly=$keepweekly
  keepmonthly=$keepmonthly
  now=`date +%s`

  local metadata

  if [ ! -d "$backuproot" ]; then
    echo "Debug: skipping rotate of $backuproot as it doesn't exist."
    exit
  fi

  for rottype in daily weekly monthly; do
    seconds=$((seconds_${rottype}))

    dir="$backuproot/$rottype"
    metadata="$backuproot/metadata/$rottype.1"
    mkdir -p $metadata
    if [ ! -d $dir.1 ]; then
      echo "Debug: $dir.1 does not exist, skipping."
      continue 1
    elif [ ! -f $metadata/created ] && [ ! -f $metadata/rotated ]; then
      echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
      continue 1
    fi
    
    # Rotate the current list of backups, if we can.
    oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
    [ "$oldest" == "" ] && oldest=0
    for (( i=$oldest; i > 0; i-- )); do
      if [ -d $dir.$i ]; then
        if [ -f $metadata/created ]; then
          created=`tail -1 $metadata/created`
        elif [ -f $metadata/rotated ]; then
          created=`tail -1 $metadata/rotated`
        else
          created=0
        fi
        cutoff_time=$(( now - (seconds*(i-1)) ))
        if [ ! $created -gt $cutoff_time ]; then
          next=$(( i + 1 ))
          if [ ! -d $dir.$next ]; then
            echo "Debug: $rottype.$i --> $rottype.$next"
            $nice mv $dir.$i $dir.$next
            mkdir -p $backuproot/metadata/$rottype.$next
            date +%c%n%s > $backuproot/metadata/$rottype.$next/rotated
          else
            echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
          fi
        else
          echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
        fi
      fi
    done
  done

  max=$((keepdaily+1))
  if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
    echo "Debug: daily.$max --> weekly.1"
    $nice mv $backuproot/daily.$max $backuproot/weekly.1
    mkdir -p $backuproot/metadata/weekly.1
    date +%c%n%s > $backuproot/metadata/weekly.1/rotated
  fi

  max=$((keepweekly+1))
  if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
    echo "Debug: weekly.$max --> monthly.1"
    $nice mv $backuproot/weekly.$max $backuproot/monthly.1
    mkdir -p $backuproot/metadata/monthly.1
    date +%c%n%s > $backuproot/metadata/monthly.1/rotated
  fi

  for rottype in daily weekly monthly; do
    max=$((keep${rottype}+1))
    dir="$backuproot/$rottype"
    oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
    [ "$oldest" == "" ] && oldest=0 
    # if we've rotated the last backup off the stack, remove it.
    for (( i=$oldest; i >= $max; i-- )); do
      if [ -d $dir.$i ]; then
        if [ -d $backuproot/rotate.tmp ]; then
          echo "Debug: removing rotate.tmp"
          $nice rm -rf $backuproot/rotate.tmp
        fi
        echo "Debug: moving $rottype.$i to rotate.tmp"
        $nice mv $dir.$i $backuproot/rotate.tmp
      fi
    done
  done

}

function move_files {
   ref=$tmp/makesnapshot-mymv-$$;
   $touch -r $1 $ref;
   $mv $1 $2;
   $touch -r $ref $2;
   $rm $ref;
}

# using setup_long_dirs from rsync handler
function setup_long_dirs {

  local destdir=$1
  local backuptype=$2
  local dir="$destdir/$backuptype"
  local tmpdir="$destdir/rotate.tmp"
  local metadata="$destdir/metadata/$backuptype.1"

  if [ ! -d $destdir ]; then
    echo "Creating destination directory $destdir..."
    mkdir -p $destdir
  fi

  if [ -d $dir.1 ]; then
    if [ -f $metadata/created ]; then
      echo "Warning: $dir.1 already exists. Overwriting contents."
    else
      echo "Warning: we seem to be resuming a partially written $dir.1"
    fi
  else
    if [ -d $tmpdir ]; then
      mv $tmpdir $dir.1
      if [ $? == 1 ]; then
        echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
        exit 1
      fi
    else
      mkdir --parents $dir.1
      if [ $? == 1 ]; then
        echo "Fatal: could not create directory $dir.1 on host $host"
        exit 1
      fi
    fi
    if [ -d $dir.2 ]; then
      echo "Debug: update links $backuptype.2 --> $backuptype.1"
      cp -alf $dir.2/. $dir.1
      #if [ $? == 1 ]; then
      #  echo "Fatal: could not create hard links to $dir.1 on host $host"
      #  exit 1
      #fi
    fi
  fi
  [ -f $metadata/created ] && rm $metadata/created
  [ -f $metadata/rotated ] && rm $metadata/rotated

}

# using prepare_storage from rsync handler
function prepare_storage {

  section="`basename $SECTION`"

  if [ "$format" == "short" ]; then

    suffix="$section.0"
    info "Rotating $backupdir/$SECTION..."
    echo "Rotating $backupdir/$SECTION..." >> $log

    if [ "$dest" == "remote" ]; then
      rotate_short_remote $backupdir/$SECTION/$section $keep
    else
      rotate_short $backupdir/$SECTION/$section $keep
      if [ ! -d "$backupdir/$SECTION/$section.0" ]; then
        mkdir -p $backupdir/$SECTION/$section.0
      fi
    fi

  elif [ "$format" == "long" ]; then

    if [ $keepdaily -gt 0 ]; then
      btype=daily
    elif [ $keepweekly -gt 0 ]; then
      btype=weekly
    elif [ $keepmonthly -gt 0 ]; then
      btype=monthly
    else
      fatal "keeping no backups";
      exit 1
    fi

    suffix="$btype.1"
    info "Rotating $backupdir/$SECTION/..."
    echo "Rotating $backupdir/$SECTION/..." >> $log

    if [ "$dest" == "remote" ]; then
      rotate_long_remote $backupdir/$SECTION
      setup_long_dirs_remote $backupdir/$SECTION $btype
    else
      rotate_long $backupdir/$SECTION
      setup_long_dirs $backupdir/$SECTION $btype
    fi

  elif [ "$format" == "mirror" ]; then
    suffix=""
  else
    fatal "Invalid backup format $format"
    exit 1
  fi

}

# using mount_ro from rsync handler
function mount_ro {

  # remount backup destination as read-only

  if [ "$dest" == "local" ]; then
    if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
      mount -o remount,ro $mountpoint
    fi
  fi

}

# usind run_fsck from rsync handler
function run_fsck {

  # check partition for errors

  if [ "$dest" == "local" ]; then
    if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
      umount $mountpoint
      if (($?)); then
        warning "Could not umount $mountpoint to run fsck"
      else
        $nice $fsck -v -y $partition >> $log
        mount $mountpoint
      fi
    fi
  fi

}

# using mount_rw from rsync handler
function mount_rw {

  # mount backup destination folder as read-write

  if [ "$dest" == "local" ]; then
    if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
      if [ -d "$mountpoint" ]; then
        mount -o remount,rw $mountpoint
        if (($?)); then
          error "Could not mount $mountpoint"
          exit 1
        fi
      fi
    fi
  fi

}

# using set_lockfile from rsync handler
function set_lockfile {

  if [ ! -z "$lockfile" ]; then
    $touch $lockfile || warning "Could not create lockfile $lockfile"
  fi

}

# using unset_lockfile from rsync handler
function unset_lockfile {

  if [ ! -z "$lockfile" ]; then
    $rm $lockfile || warning "Could not remove lockfile $lockfile"
  fi

}

# using set_dest from rsync handler
function set_dest { 

  if [ "$dest" == "local" ]; then
    dest_path="$backupdir/$SECTION/$suffix/"
  else
    if [ "$protocol" == "rsync" ]; then
      dest_path="rsync://$user@$host:$port/$backupdir/$SECTION/$suffix/"
    else
      dest_path="$user@$host:$backupdir/$SECTION/$suffix/"
    fi
  fi

}

function do_backup {

  # the backup procedure
  
  SECTION="$folder"
  prepare_storage
  set_dest
  
  cwd="`pwd`"
  cd $dest_path
  $nice $wget $wget_options $limit_rate -r -c -N -e robots=off $url
  cd $cwd
  
  $touch $dest_path

}

eval_config
set_lockfile
mount_rw

echo "Starting backup at `date`" >> $log

do_backup
mount_ro
run_fsck
unset_lockfile

echo "Finnishing backup at `date`" >> $log