Adds media and backup scripts originally from puppet-backup

6 files changed, 520 insertions, 0 deletions

diff --git a/share/hydractl/mount-media b/share/hydractl/mount-media
new file mode 100755
index 0000000..0073292
--- /dev/null
+++ b/share/hydractl/mount-media
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# mount-media
+#
+
+# Parameters
+MEDIA="$1"
+VOLUME="$2"
+BASENAME="`basename $0`"
+MOUNTPOINT="/media/$MEDIA"
+
+# Check media config
+if [ -z "$MEDIA" ]; then
+  echo "usage: $BASENAME <media> [volume]"
+  echo "example: $BASENAME mymedia sdb1"
+  exit 1
+fi
+
+# Check volume config
+if [ -z "$VOLUME" ]; then
+  VOLUME="sdb1"
+fi
+
+DISK="`echo ${VOLUME} | sed -e s/[0-9]\$//g`"
+
+# Set sudo config
+if [ "`whoami`" != 'root' ]; then
+  sudo="sudo"
+fi
+
+if [ "$BASENAME" == "mount-media" ]; then
+  echo "Checking drive health status..."
+  $sudo smartctl -H /dev/$DISK
+  echo "Disabling STANDBY on drive..."
+  $sudo sdparm --clear STANDBY -6 /dev/$DISK
+  echo "Initializing crypto layer..."
+  $sudo cryptsetup luksOpen /dev/$VOLUME $MEDIA && \
+  echo "Checking filesystem..."
+  $sudo fsck -v -y /dev/mapper/$MEDIA && \
+  echo "Mounting volume at $MOUNTPOINT..."
+  $sudo mkdir -p $MOUNTPOINT
+  $sudo mount /dev/mapper/$MEDIA $MOUNTPOINT
+elif [ "$BASENAME" == "umount-media" ]; then
+  $sudo umount $MOUNTPOINT && \
+  $sudo cryptsetup luksClose $MEDIA
+fi
diff --git a/share/hydractl/sync-backups b/share/hydractl/sync-backups
new file mode 100755
index 0000000..aa92754
--- /dev/null
+++ b/share/hydractl/sync-backups
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# sync a removable volume with system backups.
+#
+
+# Parameters.
+BASENAME="`basename $0`"
+VOLUME="$1"
+MEDIA="/media/$VOLUME"
+BWLIMIT=${BWLIMIT:=32000}
+IMAGES="/var/data/crypt/"
+RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete --bwlimit=$BWLIMIT"
+CP="ionice -c 3 nice -n 19 cp"
+
+# Sync backups for a node.
+function sync_backups_node {
+  if [ ! -z "$NODE" ]; then
+    # Get full node hostname.
+    NODE_HOSTNAME="`cat /var/vservers/$NODE/etc/hostname`"
+    if [ -z "$NODE_HOSTNAME" ]; then
+      NODE_HOSTNAME="$NODE"
+    fi
+
+    # Sync local encrypted backup.
+    echo "Syncing /var/vservers/$NODE/var/backups/duplicity/..."
+    mkdir -p /$MEDIA/$NODE_HOSTNAME/duplicity
+    $RSYNC /var/vservers/$NODE/var/backups/duplicity/ /$MEDIA/$NODE_HOSTNAME/duplicity/
+
+    # Sync remote backups.
+    for node in `ls /var/vservers/$NODE/var/backups/remote/`; do
+      echo "Syncing /var/vservers/$NODE/var/backups/remote/$node/..."
+      mkdir -p /$MEDIA/$NODE_HOSTNAME/remote/$node
+      $RSYNC /var/vservers/$NODE/var/backups/remote/$node/ /$MEDIA/$NODE_HOSTNAME/remote/$node/
+    done
+  else
+    # Sync local encrypted backup.
+    echo "Syncing /var/backups/duplicity/..."
+    mkdir -p /$MEDIA/$HOSTNAME/duplicity
+    $RSYNC /var/backups/duplicity/ /$MEDIA/$HOSTNAME/duplicity/
+
+    # Sync remote backups.
+    for node in `ls /var/backups/remote/`; do
+      echo "Syncing /var/backups/remote/$node/..."
+      mkdir -p /$MEDIA/$HOSTNAME/remote/$node/
+      $RSYNC /var/backups/remote/$node/ /$MEDIA/$HOSTNAME/remote/$node/
+    done
+
+    # Copy encrypted images.
+    if [ -d "$IMAGES" ]; then
+      for image in `find $IMAGES -name '*.img' -type f`; do
+        echo "Copying image to /$MEDIA/$HOSTNAME/images/`dirname $image`"
+        mkdir -p /$MEDIA/$HOSTNAME/images/`dirname $image`
+        $CP $image /$MEDIA/$HOSTNAME/images/`dirname $image`
+      done
+    fi
+  fi
+}
+
+# Parsing.
+if [ -z "$VOLUME" ]; then
+  echo "usage: $BASENAME <media> [nodes]"
+  exit 1
+else
+  shift
+fi
+
+# Sync backups for each node.
+if [ ! -z "$*" ]; then
+  for NODE in $*; do
+    sync_backups_node
+  done
+else
+  sync_backups_node
+fi
diff --git a/share/hydractl/sync-media b/share/hydractl/sync-media
new file mode 100755
index 0000000..d610245
--- /dev/null
+++ b/share/hydractl/sync-media
@@ -0,0 +1,270 @@
+#!/bin/bash
+#
+# sync-media assets using git-annex
+#
+
+REMOTE="$1"
+VOLUME="/media/$REMOTE"
+DOMAIN="`facter domain`"
+HOST="`facter hostname`"
+CACHE="/var/cache/$HOST/media"
+MEDIA="media.$DOMAIN"
+INCOMING="$CACHE/incoming"
+WHOAMI="`whoami`"
+OPTIONS="$*"
+
+# Fix identity
+function sync_media_identity {
+  if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
+    repo="$(basename `pwd`)"
+    git config user.name "${repo^} Archive"
+    git config user.email "$repo@localhost"
+  fi
+}
+
+# Add files into the annex
+function sync_media_add {
+  git annex add .
+
+  # Adding hidden files and symlinks, git+while version
+  git status --porcelain -u | sed -e 's/?? //' | while read file; do
+    if [ -h "$file" ]; then
+      git add "$file"
+    else
+      git annex add "$file"
+    fi
+  done
+}
+
+# If there is a playlists folder, make sure mpd user can write to it
+function sync_media_playlist_perms {
+  if [ -d "playlists" ]; then
+    $sudo chmod 775 playlists
+    $sudo chown -R mpd.audio playlists
+    find playlists -type f -exec sudo chmod 664 {} \;
+    find playlists -type d -exec sudo chmod 775 {} \;
+  fi
+}
+
+# Fix incoming permissions
+function sync_media_incoming_perms {
+  if [ -d "$INCOMING" ]; then
+    echo "Fixing $INCOMING permissions..."
+    $sudo find $INCOMING -type f -exec chmod 664 {} \;
+    $sudo find $INCOMING -type d -exec chmod 775 {} \;
+    $sudo chown -R $WHOAMI.incoming $INCOMING
+  fi
+}
+
+# Run fsck
+function sync_media_fsck {
+  if [ "$FSCK" == "true" ]; then
+    git annex fsck --fast
+  fi
+}
+
+# Run dropunused
+function sync_media_dropunused {
+  if [ "$DROPUNUSED" == "true" ]; then
+    git annex unused
+    git annex dropunused 1-1000
+  fi
+}
+
+# Get copies of annexed files
+function sync_media_get {
+  local repo="$1"
+  local numcopies
+
+  if [ "`git -C $repo config sync-media.get`" != "false" ]; then
+    if git -C $repo config sync-media.numcopies &> /dev/null; then
+      numcopies="`git -C $repo config sync-media.numcopies`"
+    else
+      numcopies="3"
+    fi
+
+    git annex get . --numcopies=$numcopies
+  fi
+}
+
+# Control whether the repository should have a copy of everything
+function sync_media_getall {
+  local repo="$1"
+
+  if [ "`git -C $repo config sync-media.getall`" == "true" ]; then
+    git annex get .
+  fi
+}
+
+# Ensure we have a reference to the remote repository
+function sync_media_ensure_remote {
+  local remote="$1"
+  local path="$2"
+
+  if [ -z "$remote" ] || [ -z "$path" ]; then
+    return
+  fi
+
+  # Check for local or remote repo
+  if [ -z "$DRIVE" ]; then
+    path="$REMOTE.$DOMAIN:$path"
+  elif [ ! -d "$path/.git/annex" ]; then
+    return
+  fi
+
+  if ! git remote | grep -q "^$remote$"; then
+    echo git remote add $remote $path
+  fi
+}
+
+# Set sudo config
+if [ "$WHOAMI" != 'root' ]; then
+  sudo="sudo"
+else
+  echo "Sorry, cannot run as root"
+  exit 1
+fi
+
+# Set fsck config
+if echo $OPTIONS | grep -q -- "--fsck"; then
+  FSCK="true"
+fi
+
+# Set unused config
+if echo $OPTIONS | grep -q -- "--dropunused"; then
+  DROPUNUSED="true"
+fi
+
+# Set drive config
+if [ ! -z "$REMOTE" ]; then
+  # Check storage media
+  MOUNT="`mount | grep $VOLUME`"
+  if [ ! -z "$MOUNT" ]; then
+    DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)"
+  fi
+fi
+
+# Commit changes
+if [ -d "$CACHE" ]; then
+  # Fix cache permissions
+  #echo "Fixing $CACHE permissions..."
+  #$sudo find $CACHE -type f -exec chmod 644 {} \;
+  #$sudo find $CACHE -type d -exec chmod 755 {} \;
+
+  sync_media_incoming_perms
+
+  # Add and update local repositories
+  for folder in `ls $CACHE`; do
+    if [ -d "$CACHE/$folder/.git/annex" ]; then
+      if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+        continue
+      fi
+
+      #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
+      #  echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
+      #  continue
+      #fi
+
+      (
+      cd $CACHE/$folder
+      echo "Syncing $CACHE/$folder..."
+
+      sync_media_playlist_perms
+      sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder
+      sync_media_identity
+      sync_media_add
+      git annex sync
+      sync_media_getall $CACHE/$folder
+      sync_media_fsck
+      sync_media_dropunused
+      )
+    fi
+  done
+
+  if [ ! -z "$DRIVE" ]; then
+    if [ ! -d "$VOLUME/$MEDIA" ]; then
+      echo "Folder $VOLUME/$MEDIA does not exist..."
+    else
+      for folder in `ls $CACHE`; do
+        if [ -d "$CACHE/$folder/.git/annex" ]; then
+          if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+            continue
+          fi
+
+          #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
+          #  echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
+          #  continue
+          #fi
+
+          if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
+            (
+            cd $VOLUME/$MEDIA
+            echo "Initializing $VOLUME/$MEDIA/$folder..."
+            git clone $CACHE/$folder && cd $folder && sync_media_identity && git annex init $DRIVE && \
+              git remote rename origin $HOST && cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder
+            )
+          fi
+        elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
+          if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+            echo "Syncing $VOLUME/$MEDIA/$folder..."
+            rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
+          fi
+        fi
+      done
+    fi
+  fi
+fi
+
+# Retrieve changes at media volumes
+if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
+  for folder in `ls $VOLUME/$MEDIA`; do
+    if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then
+      if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then
+        continue
+      fi
+
+      (
+      cd $VOLUME/$MEDIA/$folder
+      echo "Syncing $VOLUME/$MEDIA/$folder..."
+
+      sync_media_ensure_remote $HOST $CACHE/$folder
+      sync_media_playlist_perms
+      sync_media_identity
+      sync_media_add
+      git annex sync
+      sync_media_get    $VOLUME/$MEDIA/$folder
+      sync_media_getall $VOLUME/$MEDIA/$folder
+      sync_media_fsck
+      sync_media_dropunused
+      #git annex drop --auto --numcopies=2
+      )
+    elif [ -d "$CACHE/$folder" ]; then
+      if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+        echo "Syncing $VOLUME/$MEDIA/$folder..."
+        rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
+      fi
+    fi
+  done
+elif [ ! -z "$REMOTE" ]; then
+  # Try to copy to a remote
+  for folder in `ls $CACHE`; do
+    if [ -d "$CACHE/$folder/.git/annex" ]; then
+      if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+        continue
+      fi
+
+      sync_media_ensure_remote $REMOTE $CACHE/$folder
+
+      (
+      cd $CACHE/$folder
+      git annex copy . --to $REMOTE
+      git annex sync
+      )
+    else
+      if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+        echo "Syncing $VOLUME/$MEDIA/$folder..."
+        rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/
+      fi
+    fi
+  done
+fi
diff --git a/share/hydractl/sync-media-export b/share/hydractl/sync-media-export
new file mode 100755
index 0000000..7b47639
--- /dev/null
+++ b/share/hydractl/sync-media-export
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Copy git-annex repositories to remote server.
+#
+
+# Parameters
+BASENAME="`basename $0`"
+DESTINATION="$1"
+DOMAIN="`facter DOMAIN`"
+MEDIA="/var/cache/media"
+
+# Syntax check
+if [ -z "$DESTINATION" ]; then
+  echo "usage: $BASENAME <DESTINATION>"
+  exit 1
+fi
+
+# Process each repository
+for file in `ls $MEDIA`; do
+  if [ -d "$MEDIA/$file/.git/annex" ]; then
+    (
+    echo Processing "$MEDIA/$file..."
+
+    cd $MEDIA
+
+    if ssh $DESTINATION if [ -d \"/var/cache/media/$file\" ] \; then echo exists\; fi | grep -q exists; then
+      echo "Remote $file already exists, skipping..."
+      continue;
+    fi
+
+    git clone $file $file.git && \
+    rsync -avz $file.git/ $DESTINATION:/var/cache/media/$file/ && \
+    rm -rf $file.git
+    echo ""
+    )
+  fi
+done
diff --git a/share/hydractl/sync-media-initremotes b/share/hydractl/sync-media-initremotes
new file mode 100755
index 0000000..839deea
--- /dev/null
+++ b/share/hydractl/sync-media-initremotes
@@ -0,0 +1,92 @@
+#!/bin/bash
+#
+# Add git-annex remotes to repository in removable media or local cache.
+#
+
+# Parameters
+BASENAME="`basename $0`"
+VOLUME="$1"
+TYPES="$2"
+DOMAIN="`facter DOMAIN`"
+HOST="`facter hostname`"
+CACHES=""
+VOLUMES=""
+BOXES=""
+
+# Fix identity
+function sync_media_identity {
+  if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
+    repo="$(basename `pwd`)"
+    git config user.name "${repo^} Archive"
+    git config user.email "$repo@localhost"
+  fi
+}
+
+# Syntax check
+if [ -z "$VOLUME" ]; then
+  echo "usage: $BASENAME <volume> [<caches|volumes|boxes> [remotes]]"
+  exit 1
+fi
+
+# Determine media folder
+if [ "$VOLUME" == "$HOST" ] || [ "$VOLUME" == "localhost" ]; then
+  MEDIA="/var/cache/media"
+else
+  MEDIA="/media/$VOLUME/media.$DOMAIN"
+fi
+
+# Determine remote type
+shift 2
+if [ "$TYPES" == 'caches' ]; then
+  # Remotes are caches of local boxes
+  CACHES="$*"
+elif [ "$TYPES" == 'volumes' ]; then
+  # Remotes are removable media
+  VOLUMES="$*"
+elif [ "$TYPES" == 'boxes' ]; then
+  # Remotes are remote boxes
+  BOXES="$*"
+fi
+
+# Process each repository
+for file in `ls $MEDIA`; do
+  if [ -d "$MEDIA/$file/.git" ]; then
+    (
+    echo Processing "$MEDIA/$file..."
+    cd $MEDIA/$file
+    sync_media_identity
+
+    if git remote | grep -q "^origin$"; then
+      echo "Removing origin..."
+      git remote rm origin
+    fi
+
+    if [ ! -d "$MEDIA/$file/.git/annex" ]; then
+      git annex init $VOLUME
+    fi
+
+    for remote in $CACHES; do
+      echo "Adding /var/cache/$remote/media/$file remote..."
+      git remote add $remote /var/cache/$remote/media/$file
+    done
+
+    for remote in $VOLUMES; do
+      echo "Adding /media/$remote/media.$DOMAIN/$file..."
+      git remote add $remote /media/$remote/media.$DOMAIN/$file
+    done
+
+    for remote in $BOXES; do
+      if ! echo $remote | grep -q '\.'; then
+        host="$remote.$DOMAIN"
+      else
+        host="$remote"
+      fi
+
+      echo "Adding ssh://$host/var/cache/media/$file..."
+      git remote add $remote ssh://$host/var/cache/media/$file
+    done
+
+    echo ""
+    )
+  fi
+done
diff --git a/share/hydractl/umount-media b/share/hydractl/umount-media
new file mode 120000
index 0000000..5848693
--- /dev/null
+++ b/share/hydractl/umount-media
@@ -0,0 +1 @@
+mount-media
\ No newline at end of file