Enhanced rsync handler.

- Added support for:
  - Remote destinations
  - Long rotation format similar to maildir handler
  - Batch files through --read-batch and --write-batch
  - Custom file list using --files-from
  - SSH persistent connection using ControlMaster
  - The rsync:// protocol
- Metadata folder for each backup folder
- General refactoring
- Code cleanup

Not all options were tested and it might contain bugs.
Tests, comments and patches are welcome. :)

1 files changed, 922 insertions, 206 deletions

diff --git a/handlers/rsync.in b/handlers/rsync.in
index 9baa2f2..de746d5 100644
--- a/handlers/rsync.in
+++ b/handlers/rsync.in
@@ -2,10 +2,11 @@
 # backupninja handler to do incremental backups using
 # rsync and hardlinks, based on
 #
-#   http://www.mikerubel.org/computers/rsync_snapshots/
+#  - http://www.mikerubel.org/computers/rsync_snapshots/
+#  - rsnap handler by paulv at bikkel.org
+#  - maildir handler from backupninja
 #
 # feedback: rhatto at riseup.net | gpl
-# lot of enhancements grabbed from "rsnap" handler by paulv at bikkel.org
 #
 # Config file options
 # -------------------
@@ -15,21 +16,29 @@
 #   partition = partition where the backup lives
 #   fscheck = set to 1 if fsck should run on $partition after the backup is made
 #   read_only = set to 1 if $partition is mounted read-only
-#   mountpoint = backup partition mountpoint or backup main folder
-#   backupdir = folder relative do $mountpoint where the backup should be stored
-#   days = number of backup increments (min = 5)
+#   mountpoint = backup partition mountpoint or backup main folder (either local or remote)
+#   backupdir = folder relative do $mountpoint where the backup should be stored (local or remote)
+#   format = specify backup storage format: short, long or mirror (i.e, no rotations)
+#   days = for short storage format, specify the number of backup increments (min = 5)
+#   keepdaily = for long storage format, specify the number of daily backup increments
+#   keepweekly = for long storage format, specify the number of weekly backup increments
+#   keepmonthly = for long storage format, specify the number of monthly backup increments
 #   lockfile = lockfile to be kept during backup execution
 #   nicelevel = rsync command nice level
 #   enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
 #   tmp = temp folder
+#   multiconnection = set to "yes" if you want to use multiconnection ssh support
 #
 #   [source]
 #   from = local or remote
 #   host = source hostname or ip, if remote backup
+#   port = remote port number (remote source only)
+#   user = remote user name (remote source only)
 #   testconnect = when "yes", test the connection for a remote source before backup
 #   include = include folder on backup
 #   exclude = exclude folder on backup
-#   ssh = ssh command line (remote only)
+#   ssh = ssh command line (remote source only)
+#   protocol = ssh or rsync (remote source only)
 #   rsync = rsync program
 #   rsync_options = rsync command options
 #   exclude_vserver = vserver-name (valid only if vservers = yes on backupninja.conf)
@@ -37,6 +46,28 @@
 #   compress = if set to 1, compress data on rsync (remote source only)
 #   bandwidthlimit = set a badnwidth limit in kbps (remote source only)
 #   remote_rsync = remote rsync program (remote source only)
+#   id_file = ssh key file (remote source only)
+#   batch = set to "yes" to rsync use a batch file as source
+#   batchbase = folder where the batch file is located
+#   filelist = set yes if you want rsync to use a file list source
+#   filelistbase = folder where the file list is placed
+#
+#   [dest]
+#   dest = backup destination type (local or remote)
+#   testconnect = when "yes", test the connection for a remote source before backup
+#   ssh = ssh command line (remote dest only)
+#   protocol = ssh or rsync (remote dest only)
+#   numericids = when set to 1, use numeric ids instead of user/group mappings on rsync
+#   compress = if set to 1, compress data on rsync (remote source only)
+#   host = destination host name (remote destination only)
+#   port = remote port number (remote destination only)
+#   user = remote user name (remote destination only)
+#   id_file = ssh key file (remote destination only)
+#   bandwidthlimit = set a badnwidth limit in kbps (remote destination only)
+#   remote_rsync = remote rsync program (remote dest only)
+#   batch = set to "yes" to rsync write a batch file from the changes
+#   batchbase = folder where the batch file should be written
+#   fakesuper = set to yes so rsync use the --fake-super flag (remote destination only)
 #
 #   [services]
 #   initscripts = absolute path where scripts are located
@@ -54,297 +85,982 @@
 # You dont need to manually specify vservers using "include = /vservers".
 # They are automatically backuped if vserver is set to "yes" on you backupninja.conf.
 #
-
-# config file evaluation
-
-setsection system
-getconf rm rm
-getconf cp cp
-getconf touch touch
-getconf mv mv
-getconf fsck fsck
-
-setsection general
-getconf log /var/log/backup/rsync.log
-getconf partition
-getconf fscheck
-getconf read_only
-getconf mountpoint
-getconf backupdir
-getconf rotate
-getconf days
-getconf lockfile
-getconf nicelevel 0
-getconf enable_mv_timestamp_bug no
-getconf tmp /tmp
-
-setsection source
-getconf from local
-getconf testconnect no
-getconf rsync $RSYNC
-getconf rsync_options "-av --delete"
-getconf ssh ssh
-getconf user
-getconf host
-getconf include
-getconf exclude
-getconf exclude_vserver
-getconf numericids 0
-getconf compress 0
-getconf bandwidthlimit
-getconf remote_rsync rsync
-
-setsection services
-getconf initscripts
-getconf service
+# Changelog
+# ---------
+# 
+# 20090329 - rhatto at riseup.net
+#
+#   - Added support for: 
+#     - Remote destinations
+#     - Long rotation format similar to maildir handler
+#     - Batch files through --read-batch and --write-batch
+#     - Custom file list using --files-from
+#     - SSH persistent connection using ControlMaster
+#     - The rsync:// protocol
+#   - Metadata folder for each backup folder
+#   - General refactoring
+#   - Code cleanup
+#
 
 # function definitions
 
-function rotate {
+function eval_config {
+  
+  # system section
+  
+  setsection system
+  getconf rm rm
+  getconf cp cp
+  getconf touch touch
+  getconf mv mv
+  getconf fsck fsck
+  
+  # general section
+  
+  setsection general
+  getconf log /var/log/backup/rsync.log
+  getconf partition
+  getconf fscheck
+  getconf read_only
+  getconf mountpoint
+  getconf backupdir
+  getconf format short
+  getconf days
+  getconf keepdaily 5
+  getconf keepweekly 3
+  getconf keepmonthly 1
+  getconf lockfile
+  getconf nicelevel 0
+  getconf enable_mv_timestamp_bug no
+  getconf tmp /tmp
+  getconf multiconnection no
+  
+  # source section
+  
+  setsection source
+  getconf from local
+  getconf rsync $RSYNC
+  getconf rsync_options "-av --delete --recursive"
+  
+  if [ "$from" == "remote" ]; then
+    getconf testconnect no
+    getconf protocol ssh
+    getconf ssh ssh
+    getconf host
+
+    if [ "$protocol" == "ssh" ]; then
+      # sshd default listen port
+      getconf port 22
+    else
+      # rsyncd default listen port
+      getconf port 873
+    fi
+
+    getconf user
+    getconf bandwidthlimit
+    getconf remote_rsync rsync
+    getconf id_file /root/.ssh/id_dsa
+  fi
+  
+  getconf batch no
 
-  if [[ "$2" < 4 ]]; then
+  if [ "$batch" == "yes" ]; then
+    getconf batchbase
+    if [ ! -z "$batchbase" ]; then
+      batch="read"
+    fi
+  fi
+
+  getconf filelist no
+  getconf filelistbase
+  getconf include
+  getconf exclude
+  getconf exclude_vserver
+  getconf numericids 0
+  getconf compress 0
+  
+  # dest section
+  
+  setsection dest
+  getconf dest local
+  getconf fakesuper no
+  
+  if [ "$dest" == "remote" ]; then
+    getconf testconnect no
+    getconf protocol ssh
+    getconf ssh ssh
+    getconf host
+
+    if [ "$protocol" == "ssh" ]; then
+      # sshd default listen port
+      getconf port 22
+    else
+      # rsyncd default listen port
+      getconf port 873
+    fi
+
+    getconf user
+    getconf bandwidthlimit
+    getconf remote_rsync rsync
+    getconf id_file /root/.ssh/id_dsa
+  fi
+  
+  getconf batch no
+
+  if [ "$batch" != "yes" ]; then
+    getconf batch no
+    if [ "$batch" == "yes" ]; then
+      getconf batchbase
+      if [ ! -z "$batchbase" ]; then
+        batch="write"
+      fi
+    fi
+  fi
+
+  getconf numericids 0
+  getconf compress 0
+  
+  # services section
+  
+  setsection services
+  getconf initscripts /etc/init.d
+  getconf service
+
+  # config check
+
+  if [ "$dest" != "local" ] && [ "$from" == "remote" ]; then
+    fatal "When source is remote, destination should be local."
+    exit 1
+  fi
+
+  if [ "$from" != "local" ] && [ "$from" != "remote" ]; then
+    fatal "Invalid source $from"
+    exit 1
+  fi
+
+  backupdir="$mountpoint/$backupdir"
+
+  if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then 
+    error "Backupdir $backupdir does not exist"
+    exit 1
+  fi
+
+  if [ ! -z "$log" ]; then
+    mkdir -p `dirname $log`
+  fi
+
+  if [ "$format" == "short" ]; then
+    if [ -z "$days" ]; then
+      keep="4"
+    else
+      keep="`echo $days - 1 | bc -l`"
+    fi
+  fi
+
+  if [ ! -z "$nicelevel" ]; then 
+    nice="nice -n $nicelevel"
+  else 
+    nice=""
+  fi
+
+  ssh_cmd="ssh -T -o PasswordAuthentication=no $host -p $port -l $user -i $id_file"
+
+  if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then
+    if [ "$testconnect" == "yes" ] && [ "$protocol" == "ssh" ]; then
+      test_connect $host $port $user $id_file
+    fi
+  fi
+
+  if [ "$multiconnection" == "yes" ]; then
+    ssh_cmd="$ssh_cmd -S $tmp/%r@%h:%p"
+  fi
+
+  if [ $enable_mv_timestamp_bug == "yes" ]; then
+    mv=move_files
+  fi
+
+  for path in $exclude; do
+    excludes="$excludes --exclude=$path"
+  done
+
+}
+
+function rotate_short {
+
+  local dest
+  local folder="$1"
+  local keep="$2"
+  local metadata="`dirname $folder`/metadata"
+
+  if [[ "$keep" < 4 ]]; then
     error "Rotate: minimum of 4 rotations"
     exit 1
   fi
 
-  if [ -d $1.$2 ]; then
-    $nice $mv /$1.$2 /$1.tmp
+  if [ -d $folder.$keep ]; then
+    $nice $mv /$folder.$keep /$folder.tmp
   fi
 
-  for ((n=`echo "$2 - 1" | bc`; n >= 0; n--)); do
-    if [ -d $1.$n ]; then
+  for ((n=`echo "$keep - 1" | bc`; n >= 0; n--)); do
+    if [ -d $folder.$n ]; then
       dest=`echo "$n + 1" | bc`
-      $nice $mv /$1.$n /$1.$dest
-      $touch /$1.$dest
+      $nice $mv /$folder.$n /$folder.$dest
+      $touch /$folder.$dest
+      mkdir -p $metadata/`basename $folder`.$dest
+      date +%c%n%s > $metadata/`basename $folder`.$dest/rotated
+    fi
+  done
+
+  if [ -d $folder.tmp ]; then
+    $nice $mv /$folder.tmp /$folder.0
+  fi
+
+  if [ -d $folder.1 ]; then
+    $nice $cp -alf /$folder.1/. /$folder.0
+  fi
+
+}
+
+function rotate_short_remote {
+
+  local folder="$1"
+  local metadata="`dirname $folder`/metadata"
+  local keep="$2"
+
+  if [[ "$2" < 4 ]]; then
+    error "Rotate: minimum of 4 rotations"
+    exit 1
+  fi
+
+(
+  $ssh_cmd <<EOF
+  ##### BEGIN REMOTE SCRIPT #####
+
+  if [ -d $folder.$keep ]; then
+    $nice mv /$folder.$keep /$folder.tmp
+  fi
+
+  for ((n=$(($keep - 1)); n >= 0; n--)); do
+    if [ -d $folder.\$n ]; then
+      dest=\$((\$n + 1))
+      $nice mv /$folder.\$n /$folder.\$dest
+      touch /$folder.\$dest
+      mkdir -p $metadata/`basename $folder`.\$dest
+      date +%c%n%s > $metadata/`basename $folder`.\$dest/rotated
+    fi
+  done
+
+  if [ -d $folder.tmp ]; then
+    $nice mv /$folder.tmp /$folder.0
+  fi
+
+  if [ -d $folder.1 ]; then
+    $nice $cp -alf /$folder.1/. /$folder.0
+  fi
+  ##### END REMOTE SCRIPT #######
+EOF
+) | (while read a; do passthru $a; done)
+
+}
+
+function rotate_long {
+
+  backuproot="$1"
+  seconds_daily=86400
+  seconds_weekly=604800
+  seconds_monthly=2628000
+  keepdaily=$keepdaily
+  keepweekly=$keepweekly
+  keepmonthly=$keepmonthly
+  now=`date +%s`
+
+  local metadata
+
+  if [ ! -d "$backuproot" ]; then
+    echo "Debug: skipping rotate of $backuproot as it doesn't exist."
+    exit
+  fi
+
+  for rottype in daily weekly monthly; do
+    seconds=$((seconds_${rottype}))
+
+    dir="$backuproot/$rottype"
+    metadata="$backuproot/metadata/$rottype.1"
+    mkdir -p $metadata
+    if [ ! -d $dir.1 ]; then
+      echo "Debug: $dir.1 does not exist, skipping."
+      continue 1
+    elif [ ! -f $metadata/created ] && [ ! -f $metadata/rotated ]; then
+      echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
+      continue 1
+    fi
+    
+    # Rotate the current list of backups, if we can.
+    oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+    [ "$oldest" == "" ] && oldest=0
+    for (( i=$oldest; i > 0; i-- )); do
+      if [ -d $dir.$i ]; then
+        if [ -f $metadata/created ]; then
+          created=`tail -1 $metadata/created`
+        elif [ -f $metadata/rotated ]; then
+          created=`tail -1 $metadata/rotated`
+        else
+          created=0
+        fi
+        cutoff_time=$(( now - (seconds*(i-1)) ))
+        if [ ! $created -gt $cutoff_time ]; then
+          next=$(( i + 1 ))
+          if [ ! -d $dir.$next ]; then
+            echo "Debug: $rottype.$i --> $rottype.$next"
+            $nice mv $dir.$i $dir.$next
+            mkdir -p $backuproot/metadata/$rottype.$next
+            date +%c%n%s > $backuproot/metadata/$rottype.$next/rotated
+          else
+            echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
+          fi
+        else
+          echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
+        fi
+      fi
+    done
+  done
+
+  max=$((keepdaily+1))
+  if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
+    echo "Debug: daily.$max --> weekly.1"
+    $nice mv $backuproot/daily.$max $backuproot/weekly.1
+    mkdir -p $backuproot/metadata/weekly.1
+    date +%c%n%s > $backuproot/metadata/weekly.1/rotated
+  fi
+
+  max=$((keepweekly+1))
+  if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
+    echo "Debug: weekly.$max --> monthly.1"
+    $nice mv $backuproot/weekly.$max $backuproot/monthly.1
+    mkdir -p $backuproot/metadata/monthly.1
+    date +%c%n%s > $backuproot/metadata/monthly.1/rotated
+  fi
+
+  for rottype in daily weekly monthly; do
+    max=$((keep${rottype}+1))
+    dir="$backuproot/$rottype"
+    oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
+    [ "$oldest" == "" ] && oldest=0 
+    # if we've rotated the last backup off the stack, remove it.
+    for (( i=$oldest; i >= $max; i-- )); do
+      if [ -d $dir.$i ]; then
+        if [ -d $backuproot/rotate.tmp ]; then
+          echo "Debug: removing rotate.tmp"
+          $nice rm -rf $backuproot/rotate.tmp
+        fi
+        echo "Debug: moving $rottype.$i to rotate.tmp"
+        $nice mv $dir.$i $backuproot/rotate.tmp
+      fi
+    done
+  done
+
+}
+
+function rotate_long_remote {
+
+  local backuproot="$1"
+
+(
+  $ssh_cmd <<EOF
+  ##### BEGIN REMOTE SCRIPT #####
+
+  seconds_daily=86400
+  seconds_weekly=604800
+  seconds_monthly=2628000
+  keepdaily=$keepdaily
+  keepweekly=$keepweekly
+  keepmonthly=$keepmonthly
+  now=\`date +%s\`
+
+  if [ ! -d "$backuproot" ]; then
+    echo "Debug: skipping rotate of $backuproot as it doesn't exist."
+    exit
+  fi
+
+  for rottype in daily weekly monthly; do
+    seconds=\$((seconds_\${rottype}))
+
+    dir="$backuproot/\$rottype"
+    metadata="$backuproot/metadata/\$rottype.1"
+    mkdir -p \$metadata
+    if [ ! -d \$dir.1 ]; then
+      echo "Debug: \$dir.1 does not exist, skipping."
+      continue 1
+    elif [ ! -f \$metadata/created ] && [ ! -f \$metadata/rotated ]; then
+      echo "Warning: metadata does not exist for \$dir.1. This backup may be only partially completed. Skipping rotation."
+      continue 1
     fi
+    
+    # Rotate the current list of backups, if we can.
+    oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
+    [ "\$oldest" == "" ] && oldest=0
+    for (( i=\$oldest; i > 0; i-- )); do
+      if [ -d \$dir.\$i ]; then
+        if [ -f \$metadata/created ]; then
+          created=\`tail -1 \$metadata/created\`
+        elif [ -f \$metadata/rotated ]; then
+          created=\`tail -1 \$metadata/rotated\`
+        else
+          created=0
+        fi
+        cutoff_time=\$(( now - (seconds*(i-1)) ))
+        if [ ! \$created -gt \$cutoff_time ]; then
+          next=\$(( i + 1 ))
+          if [ ! -d \$dir.\$next ]; then
+            echo "Debug: \$rottype.\$i --> \$rottype.\$next"
+            $nice mv \$dir.\$i \$dir.\$next
+            mkdir -p $backuproot/metadata/\$rottype.\$next
+            date +%c%n%s > $backuproot/metadata/\$rottype.\$next/rotated
+          else
+            echo "Debug: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
+          fi
+        else
+          echo "Debug: skipping rotation of \$dir.\$i because it was created" \$(( (now-created)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
+        fi
+      fi
+    done
+  done
+
+  max=\$((keepdaily+1))
+  if [ \$keepweekly -gt 0 -a -d $backuproot/daily.\$max -a ! -d \$backuproot/weekly.1 ]; then
+    echo "Debug: daily.\$max --> weekly.1"
+    $nice mv $backuproot/daily.\$max $backuproot/weekly.1
+    mkdir -p $backuproot/metadata/weekly.1
+    date +%c%n%s > $backuproot/metadata/weekly.1/rotated
+  fi
+
+  max=\$((keepweekly+1))
+  if [ \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max -a ! -d $backuproot/monthly.1 ]; then
+    echo "Debug: weekly.\$max --> monthly.1"
+    $nice mv $backuproot/weekly.\$max $backuproot/monthly.1
+    mkdir -p $backuproot/metadata/monthly.1
+    date +%c%n%s > $backuproot/metadata/monthly.1/rotated
+  fi
+
+  for rottype in daily weekly monthly; do
+    max=\$((keep\${rottype}+1))
+    dir="$backuproot/\$rottype"
+    oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
+    [ "\$oldest" == "" ] && oldest=0 
+    # if we've rotated the last backup off the stack, remove it.
+    for (( i=\$oldest; i >= \$max; i-- )); do
+      if [ -d \$dir.\$i ]; then
+        if [ -d $backuproot/rotate.tmp ]; then
+          echo "Debug: removing rotate.tmp"
+          $nice rm -rf $backuproot/rotate.tmp
+        fi
+        echo "Debug: moving \$rottype.\$i to rotate.tmp"
+        $nice mv \$dir.\$i $backuproot/rotate.tmp
+      fi
+    done
   done
+  ##### END REMOTE SCRIPT #######
+EOF
+) | (while read a; do passthru $a; done)
+
+}
+
+function setup_long_dirs {
+
+  local destdir=$1
+  local backuptype=$2
+  local dir="$destdir/$backuptype"
+  local tmpdir="$destdir/rotate.tmp"
+  local metadata="$destdir/metadata/$backuptype.1"
+
+  if [ ! -d $destdir ]; then
+    echo "Creating destination directory $destdir..."
+    mkdir -p $destdir
+  fi
+
+  if [ -d $dir.1 ]; then
+    if [ -f $metadata/created ]; then
+      echo "Warning: $dir.1 already exists. Overwriting contents."
+    else
+      echo "Warning: we seem to be resuming a partially written $dir.1"
+    fi
+  else
+    if [ -d $tmpdir ]; then
+      mv $tmpdir $dir.1
+      if [ $? == 1 ]; then
+        echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
+        exit 1
+      fi
+    else
+      mkdir --parents $dir.1
+      if [ $? == 1 ]; then
+        echo "Fatal: could not create directory $dir.1 on host $host"
+        exit 1
+      fi
+    fi
+    if [ -d $dir.2 ]; then
+      echo "Debug: update links $backuptype.2 --> $backuptype.1"
+      cp -alf $dir.2/. $dir.1
+      #if [ $? == 1 ]; then
+      #  echo "Fatal: could not create hard links to $dir.1 on host $host"
+      #  exit 1
+      #fi
+    fi
+  fi
+  [ -f $metadata/created ] && rm $metadata/created
+  [ -f $metadata/rotated ] && rm $metadata/rotated
+
+}
+
+function setup_long_dirs_remote {
 
-  if [ -d $1.tmp ]; then
-    $nice $mv /$1.tmp /$1.0
+  local destdir=$1
+  local backuptype=$2
+  local dir="$destdir/$backuptype"
+  local tmpdir="$destdir/rotate.tmp"
+  local metadata="$destdir/metadata/$backuptype.1"
+
+(
+  $ssh_cmd <<EOF
+  ##### BEGIN REMOTE SCRIPT #####
+  if [ ! -d $destdir ]; then
+    echo "Creating destination directory $destdir on $host..."
+    mkdir -p $destdir
   fi
 
-  if [ -d $1.1 ]; then
-    $nice $cp -alf /$1.1/. /$1.0
+  if [ -d $dir.1 ]; then
+    if [ -f $metadata/created ]; then
+      echo "Warning: $dir.1 already exists. Overwriting contents."
+    else
+      echo "Warning: we seem to be resuming a partially written $dir.1"
+    fi
+  else
+    if [ -d $tmpdir ]; then
+      mv $tmpdir $dir.1
+      if [ \$? == 1 ]; then
+        echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
+        exit 1
+      fi
+    else
+      mkdir --parents $dir.1
+      if [ \$? == 1 ]; then
+        echo "Fatal: could not create directory $dir.1 on host $host"
+        exit 1
+      fi
+    fi
+    if [ -d $dir.2 ]; then
+      echo "Debug: update links $backuptype.2 --> $backuptype.1"
+      cp -alf $dir.2/. $dir.1
+      #if [ \$? == 1 ]; then
+      #  echo "Fatal: could not create hard links to $dir.1 on host $host"
+      #  exit 1
+      #fi
+    fi
   fi
+  [ -f $metadata/created ] && rm $metadata/created
+  [ -f $metadata/rotated ] && rm $metadata/rotated
+  ##### END REMOTE SCRIPT #######
+EOF
+) | (while read a; do passthru $a; done)
 
 }
 
 function move_files {
 
- ref=$tmp/makesnapshot-mymv-$$;
- $touch -r $1 $ref;
- $mv $1 $2;
- $touch -r $ref $2;
- $rm $ref;
+  ref=$tmp/makesnapshot-mymv-$$;
+  $touch -r $1 $ref;
+  $mv $1 $2;
+  $touch -r $ref $2;
+  $rm $ref;
 
 }
 
-backupdir="$mountpoint/$backupdir"
+function prepare_storage {
 
-# does $backupdir exists?
+  section="`basename $SECTION`"
 
-if [ ! -d "$backupdir" ]; then 
-  error "Backupdir $backupdir does not exist"
-  exit 1
-fi
+  if [ "$format" == "short" ]; then
 
-# setup number of increments
+    suffix="$section.0"
+    info "Rotating $backupdir/$SECTION..."
+    echo "Rotating $backupdir/$SECTION..." >> $log
 
-if [ -z "$days" ]; then
-  keep="4"
-else
-  keep="`echo $days - 1 | bc -l`"
-fi
+    if [ "$dest" == "remote" ]; then
+      rotate_short_remote $backupdir/$SECTION/$section $keep
+    else
+      rotate_short $backupdir/$SECTION/$section $keep
+      if [ ! -d "$backupdir/$SECTION/$section.0" ]; then
+        mkdir -p $backupdir/$SECTION/$section.0
+      fi
+    fi
 
-# lockfile setup
+  elif [ "$format" == "long" ]; then
 
-if [ ! -z "$lockfile" ]; then
-  $touch $lockfile || warning "Could not create lockfile $lockfile"
-fi
+    if [ $keepdaily -gt 0 ]; then
+      btype=daily
+    elif [ $keepweekly -gt 0 ]; then
+      btype=weekly
+    elif [ $keepmonthly -gt 0 ]; then
+      btype=monthly
+    else
+      fatal "keeping no backups";
+      exit 1
+    fi
 
-# nicelevel setup
+    suffix="$btype.1"
+    info "Rotating $backupdir/$SECTION/..."
+    echo "Rotating $backupdir/$SECTION/..." >> $log
 
-if [ ! -z "$nicelevel" ]; then 
-  nice="nice -n $nicelevel"
-else 
-  nice=""
-fi
+    if [ "$dest" == "remote" ]; then
+      rotate_long_remote $backupdir/$SECTION
+      setup_long_dirs_remote $backupdir/$SECTION $btype
+    else
+      rotate_long $backupdir/$SECTION
+      setup_long_dirs $backupdir/$SECTION $btype
+    fi
 
-# connection test
+  elif [ "$format" == "mirror" ]; then
+    suffix=""
+  else
+    fatal "Invalid backup format $format"
+    exit 1
+  fi
 
-if [ "$from" == "remote" ] && [ "$testconnect" == "yes" ]; then
-  debug "$ssh -o PasswordAuthentication=no $user@$host 'echo -n 1'"
-  result=`ssh -o PasswordAuthentication=no $user@$host 'echo -n 1'`
-  if [ "$result" != "1" ]; then
-    fatal "Can't connect to $host as $user."
+}
+
+function set_orig {
+
+  if [ "$from" == "local" ]; then
+    orig="/$SECTION/"
+  elif [ "$from" == "remote" ]; then
+    if [ "$protocol" == "rsync" ]; then
+      orig="rsync://$user@$host:$port/$SECTION/"
+    else
+      orig="$user@$host:/$SECTION/"
+    fi
+  fi
+
+}
+
+function set_dest { 
+
+  if [ "$dest" == "local" ]; then
+    dest_path="$backupdir/$SECTION/$suffix/"
   else
-    debug "Connected to $srchost successfully"
+    if [ "$protocol" == "rsync" ]; then
+      dest_path="rsync://$user@$host:$port/$backupdir/$SECTION/$suffix/"
+    else
+      dest_path="$user@$host:$backupdir/$SECTION/$suffix/"
+    fi
   fi
-fi
 
-# rsync options for local sources
+}
 
-if [ "$from" == "local" ]; then
+function set_batch_mode {
 
-  rsync_local_options="$rsync_options"
+  local batch_file="$batchbase/$SECTION/$suffix"
 
-  if [ ! -z "$numericids" ]; then
-    rsync_local_options="$rsync_local_options --numeric-ids "
+  if [ "$batch" == "read" ]; then
+    if [ -e "$batch_file" ]; then
+      orig=""
+      excludes=""
+      batch_option="--read-batch=$batch_file"
+    else
+      fatal "Batch file not found: $batch_file"
+      exit 1
+    fi
+  elif [ "$batch" == "write" ]; then
+    mkdir -p `dirname $batch_file`
+    batch_option="--write-batch=$batch_file"
   fi
 
-fi
+}
 
-# rsync options for remote sources
+function update_metadata {
 
-if [ "$from" == "remote" ]; then
+  local metadata
+  local folder
 
-  rsync_remote_options="$rsync_options --rsync-path=$remote_rsync"
+  if [ "$dest" == "local" ]; then
+    metadata="`dirname $dest_path`/metadata/`basename $dest_path`"
+    mkdir -p $metadata
+    date +%c%n%s > $metadata/created
+    $touch $backupdir/$SECTION/$suffix
+  else
+    folder="`echo $dest_path | cut -d : -f 2`"
+    metadata="`dirname $folder`/metadata/`basename $folder`"
+
+(
+  $ssh_cmd <<EOF
+    ##### BEGIN REMOTE SCRIPT #####
+    mkdir -p $metadata
+    date +%c%n%s > $metadata/created
+    ##### END REMOTE SCRIPT #######
+EOF
+) | (while read a; do passthru $a; done)
 
-  if [ "$compress" == "1" ]; then
-    rsync_remote_options="$rsync_remote_options --compress"
   fi
 
-  if [ ! -z "$bandwidthlimit" ]; then
-    rsync_remote_options="$rsync_remote_options --bwlimit=$bandwidthlimit"
+}
+
+function test_connect {
+
+  local host="$1"
+  local port="$2"
+  local user="$3"
+  local id_file="$4"
+
+  if [ -z "$host" ] || [ -z "$user" ]; then
+    fatal "Remote host or user not set"
+    exit 1
   fi
 
-  if [ ! -z "$numericids" ]; then
-    rsync_remote_options="$rsync_remote_options --numeric-ids"
+  debug "$ssh_cmd 'echo -n 1'"
+  result=`$ssh_cmd 'echo -n 1'`
+
+  if [ "$result" != "1" ]; then
+    fatal "Can't connect to $host as $user."
+    exit 1
+  else
+    debug "Connected to $host successfully"
   fi
 
-fi
+}
 
-# set mv procedure
+function set_lockfile {
 
-if [ $enable_mv_timestamp_bug == "yes" ]; then
-  mv=move_files
-fi
+  if [ ! -z "$lockfile" ]; then
+    $touch $lockfile || warning "Could not create lockfile $lockfile"
+  fi
 
-# set excludes
+}
 
-for path in $exclude; do
-  EXCLUDES="$EXCLUDES --exclude=$path"
-done
+function unset_lockfile {
 
-# stop services
+  if [ ! -z "$lockfile" ]; then
+    $rm $lockfile || warning "Could not remove lockfile $lockfile"
+  fi
 
-if [ ! -z "$service" ]; then
-  for daemon in $service; do
-    info "Stopping service $daemon..."
-    $initscripts/$daemon stop
-  done
-fi
+}
 
-echo "Starting backup at `date`" >> $log
+function set_filelist {
 
-# mount backup destination folder as read-write
+  filelist_flag=""
 
-if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
-  if [ -d "$mountpoint" ]; then
-    mount -o remount,rw $mountpoint
-    if (($?)); then
-      error "Could not mount $mountpoint"
-      exit 1
+  if [ "$filelist" == "yes" ]; then
+    if [ ! -z "$filelistbase" ]; then
+      if [ -e "$filelistbase/$SECTION/$suffix" ]; then
+        filelist_flag="--files-from=$filelistbase/$SECTION/$suffix"
+      else
+        warning "File list $filelistbase/$SECTION/$suffix not found."
+      fi
+    else
+      warning "No filelistbase set."
     fi
   fi
-fi
 
-# add vservers to included folders
+}
+
+function set_rsync_options {
+
+  if [ ! -z "$numericids" ]; then
+    rsync_options="$rsync_options --numeric-ids"
+  fi
+
+  if [ "$from" == "local" ] || [ "$dest" == "local" ]; then
+    # rsync options for local sources or destinations
+    rsync_options="$rsync_options"
+  fi
+
+  if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then
+
+    # rsync options for remote sources or destinations
+
+    if [ "$compress" == "1" ]; then
+      rsync_options="$rsync_options --compress"
+    fi
 
-if [ "$vservers_are_available" == "yes" ]; then
+    if [ ! -z "$bandwidthlimit" ]; then
+      rsync_options="$rsync_options --bwlimit=$bandwidthlimit"
+    fi
+    
+    if [ "$fakesuper" == "yes" ]; then
+      remote_rsync="$remote_rsync --fake-super"
+    fi
 
-  # sane permission on backup
-  mkdir -p $backupdir/$VROOTDIR
-  chmod 000 $backupdir/$VROOTDIR
+    rsync_options=($rsync_options --rsync-path="$remote_rsync")
 
-  for candidate in $found_vservers; do
-    candidate="`basename $candidate`"
-    found_excluded_vserver="0"
-    for excluded_vserver in $exclude_vserver; do
-      if [ "$excluded_vserver" == "$candidate" ]; then
-        found_excluded_vserver="1"
-        break
+    if [ "$protocol" == "ssh" ]; then
+      if [ ! -e "$id_file" ]; then
+        fatal "SSH Identity file $id_file not found"
+        exit 1
+      else
+        debug RSYNC_RSH=\"$ssh_cmd\"
+        echo RSYNC_RSH=\"$ssh_cmd\" >> $log
+        RSYNC_RSH="$ssh_cmd"
       fi
-    done
-    if [ "$found_excluded_vserver" == "0" ]; then
-      include="$include $VROOTDIR/$candidate"
     fi
-  done
-fi
 
-# the backup procedure
+  fi
 
-for SECTION in $include; do
+  include_vservers
 
-  section="`basename $SECTION`"
+}
+
+function stop_services {
 
-  if [ ! -d "$backupdir/$SECTION/$section.0" ]; then
-    mkdir -p $backupdir/$SECTION/$section.0
+  if [ ! -z "$service" ]; then
+    for daemon in $service; do
+      info "Stopping service $daemon..."
+      $initscripts/$daemon stop
+    done
   fi
- 
-  info "Rotating $backupdir/$SECTION/$section..."
-  echo "Rotating $backupdir/$SECTION/$section..." >> $log
-  rotate $backupdir/$SECTION/$section $keep
-  info "Syncing $SECTION on $backupdir/$SECTION/$section.0..."
 
-  if [ "$from" == "local" ]; then
-    debug $rsync $rsync_local_options $EXCLUDES /$SECTION/ $backupdir/$SECTION/$section.0/ 
-    $nice $rsync $rsync_local_options $EXCLUDES /$SECTION/ $backupdir/$SECTION/$section.0/ >> $log
-    if [ "$?" != "0" ]; then
-      warning "Rsync error when trying to transfer $SECTION"
+}
+
+function start_services {
+
+  # restart services
+
+  if [ ! -z "$service" ]; then
+    for daemon in $service; do
+      info "Starting service $daemon..."
+      $initscripts/$daemon start
+    done
+  fi
+
+}
+
+function mount_rw {
+
+  # mount backup destination folder as read-write
+
+  if [ "$dest" == "local" ]; then
+    if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
+      if [ -d "$mountpoint" ]; then
+        mount -o remount,rw $mountpoint
+        if (($?)); then
+          error "Could not mount $mountpoint"
+          exit 1
+        fi
+      fi
     fi
-  elif [ "$from" == "remote" ]; then
-    if [ -z "$user" ] || [ -z "$host" ]; then
-      error "Config file error: either user or host was not specified"
-      exit 1
-    else
-      debug $nice $rsync $rsync_remote_options $EXCLUDES -e "$ssh" $user@$host:/$SECTION/ $backupdir/$SECTION/$section.0
-      $nice $rsync $rsync_remote_options $EXCLUDES -e "$ssh" $user@$host:/$SECTION/ $backupdir/$SECTION/$section.0 >> $log
-      if [ "$?" != "0" ]; then
-        warning "Rsync error when trying to transfer $SECTION"
+  fi
+
+}
+
+function mount_ro {
+
+  # remount backup destination as read-only
+
+  if [ "$dest" == "local" ]; then
+    if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
+      mount -o remount,ro $mountpoint
+    fi
+  fi
+
+}
+
+function run_fsck {
+
+  # check partition for errors
+
+  if [ "$dest" == "local" ]; then
+    if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
+      umount $mountpoint
+      if (($?)); then
+        warning "Could not umount $mountpoint to run fsck"
+      else
+        $nice $fsck -v -y $partition >> $log
+        mount $mountpoint
       fi
     fi
-  else
-    error "Invalid source $from"
-    exit 1
   fi
 
-  $touch $backupdir/$SECTION/$section.0
+}
 
-done
+function include_vservers {
 
-# remount backup destination as read-only
+  # add vservers to included folders
 
-if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
-  mount -o remount,ro $mountpoint
-fi
+  if [ "$vservers_are_available" == "yes" ]; then
 
-# check partition for errors
+    # sane permission on backup
+    mkdir -p $backupdir/$VROOTDIR
+    chmod 000 $backupdir/$VROOTDIR
 
-if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
-  umount $mountpoint
-  if (($?)); then
-    warning "Could not umount $mountpoint to run fsck"
-  else
-    $nice $fsck -v -y $partition >> $log
-    mount $mountpoint
+    for candidate in $found_vservers; do
+      candidate="`basename $candidate`"
+      found_excluded_vserver="0"
+      for excluded_vserver in $exclude_vserver; do
+        if [ "$excluded_vserver" == "$candidate" ]; then
+          found_excluded_vserver="1"
+          break
+        fi
+      done
+      if [ "$found_excluded_vserver" == "0" ]; then
+        include="$include $VROOTDIR/$candidate"
+      fi
+    done
   fi
-fi
 
-# restart services
+}
 
-if [ ! -z "$service" ]; then
-  for daemon in $service; do
-    info "Starting service $daemon..."
-    $initscripts/$daemon start
-  done
-fi
+function start_mux {
+
+  if [ "$multiconnection" == "yes" ]; then
+    debug "Starting master ssh connection"
+    $ssh_cmd -M sleep 1d &
+    sleep 1
+  fi
+
+}
 
-# removes the lockfile
+function end_mux {
+
+  if [ "$multiconnection" == "yes" ]; then
+    debug "Stopping master ssh connection"
+    $ssh_cmd pkill sleep
+  fi
+
+}
+
+# the backup procedure
+
+eval_config
+set_lockfile
+set_rsync_options
+start_mux
+stop_services
+mount_rw
+
+echo "Starting backup at `date`" >> $log
+
+for SECTION in $include; do
+
+  prepare_storage
+  set_orig
+  set_batch_mode
+  set_filelist
+  set_dest
+
+  info "Syncing $SECTION on $dest_path..."
+  debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path
+  $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path >> $log
+
+  if [ "$?" != "0" ]; then
+    warning "Rsync error when trying to transfer $SECTION"
+  fi
+
+  update_metadata
+
+done
 
-if [ ! -z "$lockfile" ]; then
-  $rm $lockfile || warning "Could not remove lockfile $lockfile"
-fi
+mount_ro
+run_fsck
+start_services
+unset_lockfile
+end_mux
 
 echo "Finnishing backup at `date`" >> $log