Adds hydractl sync-guestfs

2 files changed, 122 insertions, 3 deletions

diff --git a/share/hydractl/sync-guestfs b/share/hydractl/sync-guestfs
new file mode 100755
index 0000000..96e74cd
--- /dev/null
+++ b/share/hydractl/sync-guestfs
@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# sync-guestfs: sync a tree containing virtual machine guest images
+#
+
+# Load
+source $APP_BASE/lib/hydra/functions || exit 1
+hydra_config_load
+
+# Parameters
+BASENAME="$0"
+ORIG="$1"
+DEST="$2"
+TMP="/tmp"
+RSYNC="ionice -c 3 nice -n 19 rsync -av --delete --progress"
+
+# Dependencies
+hydra_install_package rsync
+hydra_install_package libguestfs-tools
+
+# Syntax
+if [ -z "$DEST" ]; then
+  echo "usage: $BASENAME <orig-folder> <dest-folder>"
+  exit 1
+elif [ ! -d "$ORIG" ]; then
+  echo "folder $DEST does not exist."
+  exit 1
+fi
+
+# Sudo
+if [ "$USER" != 'root' ]; then
+  export SUDO="sudo"
+fi
+
+# Ensure the dest folder exist
+if [ ! -d "$DEST" ]; then
+  $SUDO mkdir -p $DEST
+
+  OWNER="$(stat -c %U $ORIG)"
+  GROUP="$(stat -c %G $ORIG)"
+
+  # With the right permissions
+  $SUDO chown $OWNER.$GROUP $DEST
+fi
+
+# First sync the folder structure
+$SUDO $RSYNC --exclude='*.img' $ORIG/ $DEST/
+
+# Temporary folders
+ORIG_MOUNT="`mktemp -d $TMP/sync-guestfs-orig.XXXXXXXXXX`"
+DEST_MOUNT="`mktemp -d $TMP/sync-guestfs-dest.XXXXXXXXXX`"
+
+# Main loop
+(
+  cd $ORIG
+  find -name '*.img' | sort | sed -e 's|^./||' | while read image; do
+    if [ ! -e "$DEST/$image" ]; then
+      # Make the first copy
+      # See https://gergap.wordpress.com/2013/08/10/rsync-and-sparse-files/
+      #     https://stackoverflow.com/questions/19821186/rsync-sparse-does-transfer-whole-data#19827546
+      echo "Making the first copy of $image to $DEST, which might take a while..."
+      $SUDO $RSYNC --sparse $image $DEST/$image
+    else
+      echo "Destination $DEST/$image exists, syncing..."
+      $SUDO $RSYNC --inplace $image $DEST/$image
+
+      #
+      # Sync using libguestfs-tools: mount existing
+      # images and use rsync to copy the differences.
+      #
+      ## Mount and sync
+      #$SUDO guestmount -a $image       -i --ro $ORIG_MOUNT/ && \
+      #$SUDO guestmount -a $DEST/$image -i      $DEST_MOUNT/ && \
+      #time $SUDO $RSYNC $ORIG_MOUNT/ $DEST_MOUNT/           || \
+      #echo "Failed to sync $image with $DEST/$image."
+      #
+      ## Umount orig image
+      #if mount | grep -q "$ORIG_MOUNT"; then
+      #  $SUDO guestunmount $ORIG_MOUNT
+      #fi
+      #
+      ## Umount dest image
+      #if mount | grep -q "$DEST_MOUNT"; then
+      #  $SUDO guestunmount $DEST_MOUNT
+      #fi
+      #
+      #echo "Optimizing $DEST/$image size..."
+      #time $SUDO ionice -c 3 nice -n 19 virt-sparsify --in-place $DEST/$image
+      #time $SUDO ionice -c 3 nice -n 19 virt-sparsify --tmp $TMP --compress $DEST/$image $DEST/$image.new && \
+      #     $SUDO mv $DEST/$image.new $DEST/$image
+    fi
+  done
+)
+
+# Remove images that does not exist anymore in the origin
+(
+  cd $DEST
+  find -name '*.img' | sort | sed -e 's|^./||' | while read image; do
+    if [ ! -e "$ORIG/$image" ]; then
+      echo "Removing old $DEST/$image..."
+      $SUDO rm -f $image
+    fi
+  done
+
+  # Sync again to prune any dangling folder structure
+  echo "Cleaning up..."
+  $SUDO $RSYNC --exclude='*.img' $ORIG/ $DEST/
+)
+
+# Teardown
+$SUDO rmdir $ORIG_MOUNT
+$SUDO rmdir $DEST_MOUNT
diff --git a/share/hydractl/sync-tpc b/share/hydractl/sync-tpc
index 6c1b5d2..5b990bb 100755
--- a/share/hydractl/sync-tpc
+++ b/share/hydractl/sync-tpc
@@ -34,9 +34,6 @@ done
 
 # Sync files
 $SUDO $RSYNC /home/                 $DEST/home/
-$SUDO $RSYNC /var/cache/virtualbox/ $DEST/var/cache/virtualbox/
-$SUDO $RSYNC /var/cache/qemu/       $DEST/var/cache/qemu/
-$SUDO $RSYNC /var/cache/vagrant/    $DEST/var/cache/vagrant/
 $SUDO $RSYNC /var/cache/pbuilder/   $DEST/var/cache/pbuilder/
 $SUDO $RSYNC /var/chroot/           $DEST/var/chroot/
 $SUDO $RSYNC /var/lib/wicd/         $DEST/var/lib/wicd/
@@ -68,5 +65,15 @@ if [ ! -d "$MEDIA" ]; then
   $SUDO chown $OWNER.$GROUP $MEDIA
 fi
 
+# Guest images were previously synced using rsync
+#$SUDO $RSYNC /var/cache/virtualbox/ $DEST/var/cache/virtualbox/
+#$SUDO $RSYNC /var/cache/qemu/       $DEST/var/cache/qemu/
+#$SUDO $RSYNC /var/cache/vagrant/    $DEST/var/cache/vagrant/
+
+# Sync qemu guest images
+if [ -d "/var/cache/qemu" ]; then
+  hydractl sync-guestfs /var/cache/qemu $DEST/var/cache/qemu
+fi
+
 # Sync media remotes
 hydractl sync-media-remotes $MEDIA