summaryrefslogtreecommitdiff
path: root/share/hydractl
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2016-10-02 15:31:42 -0300
committerSilvio Rhatto <rhatto@riseup.net>2016-10-02 15:31:42 -0300
commitfdb07b542860089eb208ec5967826a2225899d10 (patch)
tree1684d5eee1520e8416c6775416f459afed04c02b /share/hydractl
parentd13a337e8bc54ace1ea98ff5faae01d525d9dd21 (diff)
downloadhydra-fdb07b542860089eb208ec5967826a2225899d10.tar.gz
hydra-fdb07b542860089eb208ec5967826a2225899d10.tar.bz2
Adds media and backup scripts originally from puppet-backup
Diffstat (limited to 'share/hydractl')
-rwxr-xr-xshare/hydractl/mount-media46
-rwxr-xr-xshare/hydractl/sync-backups74
-rwxr-xr-xshare/hydractl/sync-media270
-rwxr-xr-xshare/hydractl/sync-media-export37
-rwxr-xr-xshare/hydractl/sync-media-initremotes92
l---------share/hydractl/umount-media1
6 files changed, 520 insertions, 0 deletions
diff --git a/share/hydractl/mount-media b/share/hydractl/mount-media
new file mode 100755
index 0000000..0073292
--- /dev/null
+++ b/share/hydractl/mount-media
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# mount-media
+#
+
+# Parameters
+MEDIA="$1"
+VOLUME="$2"
+BASENAME="`basename $0`"
+MOUNTPOINT="/media/$MEDIA"
+
+# Check media config
+if [ -z "$MEDIA" ]; then
+ echo "usage: $BASENAME <media> [volume]"
+ echo "example: $BASENAME mymedia sdb1"
+ exit 1
+fi
+
+# Check volume config
+if [ -z "$VOLUME" ]; then
+ VOLUME="sdb1"
+fi
+
+DISK="`echo ${VOLUME} | sed -e s/[0-9]\$//g`"
+
+# Set sudo config
+if [ "`whoami`" != 'root' ]; then
+ sudo="sudo"
+fi
+
+if [ "$BASENAME" == "mount-media" ]; then
+ echo "Checking drive health status..."
+ $sudo smartctl -H /dev/$DISK
+ echo "Disabling STANDBY on drive..."
+ $sudo sdparm --clear STANDBY -6 /dev/$DISK
+ echo "Initializing crypto layer..."
+ $sudo cryptsetup luksOpen /dev/$VOLUME $MEDIA && \
+ echo "Checking filesystem..."
+ $sudo fsck -v -y /dev/mapper/$MEDIA && \
+ echo "Mounting volume at $MOUNTPOINT..."
+ $sudo mkdir -p $MOUNTPOINT
+ $sudo mount /dev/mapper/$MEDIA $MOUNTPOINT
+elif [ "$BASENAME" == "umount-media" ]; then
+ $sudo umount $MOUNTPOINT && \
+ $sudo cryptsetup luksClose $MEDIA
+fi
diff --git a/share/hydractl/sync-backups b/share/hydractl/sync-backups
new file mode 100755
index 0000000..aa92754
--- /dev/null
+++ b/share/hydractl/sync-backups
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# sync a removable volume with system backups.
+#
+
+# Parameters.
+BASENAME="`basename $0`"
+VOLUME="$1"
+MEDIA="/media/$VOLUME"
+BWLIMIT=${BWLIMIT:=32000}
+IMAGES="/var/data/crypt/"
+RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete --bwlimit=$BWLIMIT"
+CP="ionice -c 3 nice -n 19 cp"
+
+# Sync backups for a node.
+function sync_backups_node {
+ if [ ! -z "$NODE" ]; then
+ # Get full node hostname.
+ NODE_HOSTNAME="`cat /var/vservers/$NODE/etc/hostname`"
+ if [ -z "$NODE_HOSTNAME" ]; then
+ NODE_HOSTNAME="$NODE"
+ fi
+
+ # Sync local encrypted backup.
+ echo "Syncing /var/vservers/$NODE/var/backups/duplicity/..."
+ mkdir -p /$MEDIA/$NODE_HOSTNAME/duplicity
+ $RSYNC /var/vservers/$NODE/var/backups/duplicity/ /$MEDIA/$NODE_HOSTNAME/duplicity/
+
+ # Sync remote backups.
+ for node in `ls /var/vservers/$NODE/var/backups/remote/`; do
+ echo "Syncing /var/vservers/$NODE/var/backups/remote/$node/..."
+ mkdir -p /$MEDIA/$NODE_HOSTNAME/remote/$node
+ $RSYNC /var/vservers/$NODE/var/backups/remote/$node/ /$MEDIA/$NODE_HOSTNAME/remote/$node/
+ done
+ else
+ # Sync local encrypted backup.
+ echo "Syncing /var/backups/duplicity/..."
+ mkdir -p /$MEDIA/$HOSTNAME/duplicity
+ $RSYNC /var/backups/duplicity/ /$MEDIA/$HOSTNAME/duplicity/
+
+ # Sync remote backups.
+ for node in `ls /var/backups/remote/`; do
+ echo "Syncing /var/backups/remote/$node/..."
+ mkdir -p /$MEDIA/$HOSTNAME/remote/$node/
+ $RSYNC /var/backups/remote/$node/ /$MEDIA/$HOSTNAME/remote/$node/
+ done
+
+ # Copy encrypted images.
+ if [ -d "$IMAGES" ]; then
+ for image in `find $IMAGES -name '*.img' -type f`; do
+ echo "Copying image to /$MEDIA/$HOSTNAME/images/`dirname $image`"
+ mkdir -p /$MEDIA/$HOSTNAME/images/`dirname $image`
+ $CP $image /$MEDIA/$HOSTNAME/images/`dirname $image`
+ done
+ fi
+ fi
+}
+
+# Parsing.
+if [ -z "$VOLUME" ]; then
+ echo "usage: $BASENAME <media> [nodes]"
+ exit 1
+else
+ shift
+fi
+
+# Sync backups for each node.
+if [ ! -z "$*" ]; then
+ for NODE in $*; do
+ sync_backups_node
+ done
+else
+ sync_backups_node
+fi
diff --git a/share/hydractl/sync-media b/share/hydractl/sync-media
new file mode 100755
index 0000000..d610245
--- /dev/null
+++ b/share/hydractl/sync-media
@@ -0,0 +1,270 @@
+#!/bin/bash
+#
+# sync-media assets using git-annex
+#
+
+REMOTE="$1"
+VOLUME="/media/$REMOTE"
+DOMAIN="`facter domain`"
+HOST="`facter hostname`"
+CACHE="/var/cache/$HOST/media"
+MEDIA="media.$DOMAIN"
+INCOMING="$CACHE/incoming"
+WHOAMI="`whoami`"
+OPTIONS="$*"
+
+# Fix identity
+function sync_media_identity {
+ if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
+ repo="$(basename `pwd`)"
+ git config user.name "${repo^} Archive"
+ git config user.email "$repo@localhost"
+ fi
+}
+
+# Add files into the annex
+function sync_media_add {
+ git annex add .
+
+ # Adding hidden files and symlinks, git+while version
+ git status --porcelain -u | sed -e 's/?? //' | while read file; do
+ if [ -h "$file" ]; then
+ git add "$file"
+ else
+ git annex add "$file"
+ fi
+ done
+}
+
+# If there is a playlists folder, make sure mpd user can write to it
+function sync_media_playlist_perms {
+ if [ -d "playlists" ]; then
+ $sudo chmod 775 playlists
+ $sudo chown -R mpd.audio playlists
+ find playlists -type f -exec sudo chmod 664 {} \;
+ find playlists -type d -exec sudo chmod 775 {} \;
+ fi
+}
+
+# Fix incoming permissions
+function sync_media_incoming_perms {
+ if [ -d "$INCOMING" ]; then
+ echo "Fixing $INCOMING permissions..."
+ $sudo find $INCOMING -type f -exec chmod 664 {} \;
+ $sudo find $INCOMING -type d -exec chmod 775 {} \;
+ $sudo chown -R $WHOAMI.incoming $INCOMING
+ fi
+}
+
+# Run fsck
+function sync_media_fsck {
+ if [ "$FSCK" == "true" ]; then
+ git annex fsck --fast
+ fi
+}
+
+# Run dropunused
+function sync_media_dropunused {
+ if [ "$DROPUNUSED" == "true" ]; then
+ git annex unused
+ git annex dropunused 1-1000
+ fi
+}
+
+# Get copies of annexed files
+function sync_media_get {
+ local repo="$1"
+ local numcopies
+
+ if [ "`git -C $repo config sync-media.get`" != "false" ]; then
+ if git -C $repo config sync-media.numcopies &> /dev/null; then
+ numcopies="`git -C $repo config sync-media.numcopies`"
+ else
+ numcopies="3"
+ fi
+
+ git annex get . --numcopies=$numcopies
+ fi
+}
+
+# Control whether the repository should have a copy of everything
+function sync_media_getall {
+ local repo="$1"
+
+ if [ "`git -C $repo config sync-media.getall`" == "true" ]; then
+ git annex get .
+ fi
+}
+
+# Ensure we have a reference to the remote repository
+function sync_media_ensure_remote {
+ local remote="$1"
+ local path="$2"
+
+ if [ -z "$remote" ] || [ -z "$path" ]; then
+ return
+ fi
+
+ # Check for local or remote repo
+ if [ -z "$DRIVE" ]; then
+ path="$REMOTE.$DOMAIN:$path"
+ elif [ ! -d "$path/.git/annex" ]; then
+ return
+ fi
+
+ if ! git remote | grep -q "^$remote$"; then
+ echo git remote add $remote $path
+ fi
+}
+
+# Set sudo config
+if [ "$WHOAMI" != 'root' ]; then
+ sudo="sudo"
+else
+ echo "Sorry, cannot run as root"
+ exit 1
+fi
+
+# Set fsck config
+if echo $OPTIONS | grep -q -- "--fsck"; then
+ FSCK="true"
+fi
+
+# Set unused config
+if echo $OPTIONS | grep -q -- "--dropunused"; then
+ DROPUNUSED="true"
+fi
+
+# Set drive config
+if [ ! -z "$REMOTE" ]; then
+ # Check storage media
+ MOUNT="`mount | grep $VOLUME`"
+ if [ ! -z "$MOUNT" ]; then
+ DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)"
+ fi
+fi
+
+# Commit changes
+if [ -d "$CACHE" ]; then
+ # Fix cache permissions
+ #echo "Fixing $CACHE permissions..."
+ #$sudo find $CACHE -type f -exec chmod 644 {} \;
+ #$sudo find $CACHE -type d -exec chmod 755 {} \;
+
+ sync_media_incoming_perms
+
+ # Add and update local repositories
+ for folder in `ls $CACHE`; do
+ if [ -d "$CACHE/$folder/.git/annex" ]; then
+ if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+ continue
+ fi
+
+ #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
+ # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
+ # continue
+ #fi
+
+ (
+ cd $CACHE/$folder
+ echo "Syncing $CACHE/$folder..."
+
+ sync_media_playlist_perms
+ sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder
+ sync_media_identity
+ sync_media_add
+ git annex sync
+ sync_media_getall $CACHE/$folder
+ sync_media_fsck
+ sync_media_dropunused
+ )
+ fi
+ done
+
+ if [ ! -z "$DRIVE" ]; then
+ if [ ! -d "$VOLUME/$MEDIA" ]; then
+ echo "Folder $VOLUME/$MEDIA does not exist..."
+ else
+ for folder in `ls $CACHE`; do
+ if [ -d "$CACHE/$folder/.git/annex" ]; then
+ if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+ continue
+ fi
+
+ #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then
+ # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo."
+ # continue
+ #fi
+
+ if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
+ (
+ cd $VOLUME/$MEDIA
+ echo "Initializing $VOLUME/$MEDIA/$folder..."
+ git clone $CACHE/$folder && cd $folder && sync_media_identity && git annex init $DRIVE && \
+ git remote rename origin $HOST && cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder
+ )
+ fi
+ elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
+ if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+ echo "Syncing $VOLUME/$MEDIA/$folder..."
+ rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
+ fi
+ fi
+ done
+ fi
+ fi
+fi
+
+# Retrieve changes at media volumes
+if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
+ for folder in `ls $VOLUME/$MEDIA`; do
+ if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then
+ if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then
+ continue
+ fi
+
+ (
+ cd $VOLUME/$MEDIA/$folder
+ echo "Syncing $VOLUME/$MEDIA/$folder..."
+
+ sync_media_ensure_remote $HOST $CACHE/$folder
+ sync_media_playlist_perms
+ sync_media_identity
+ sync_media_add
+ git annex sync
+ sync_media_get $VOLUME/$MEDIA/$folder
+ sync_media_getall $VOLUME/$MEDIA/$folder
+ sync_media_fsck
+ sync_media_dropunused
+ #git annex drop --auto --numcopies=2
+ )
+ elif [ -d "$CACHE/$folder" ]; then
+ if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+ echo "Syncing $VOLUME/$MEDIA/$folder..."
+ rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
+ fi
+ fi
+ done
+elif [ ! -z "$REMOTE" ]; then
+ # Try to copy to a remote
+ for folder in `ls $CACHE`; do
+ if [ -d "$CACHE/$folder/.git/annex" ]; then
+ if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
+ continue
+ fi
+
+ sync_media_ensure_remote $REMOTE $CACHE/$folder
+
+ (
+ cd $CACHE/$folder
+ git annex copy . --to $REMOTE
+ git annex sync
+ )
+ else
+ if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
+ echo "Syncing $VOLUME/$MEDIA/$folder..."
+ rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/
+ fi
+ fi
+ done
+fi
diff --git a/share/hydractl/sync-media-export b/share/hydractl/sync-media-export
new file mode 100755
index 0000000..7b47639
--- /dev/null
+++ b/share/hydractl/sync-media-export
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Copy git-annex repositories to remote server.
+#
+
+# Parameters
+BASENAME="`basename $0`"
+DESTINATION="$1"
+DOMAIN="`facter DOMAIN`"
+MEDIA="/var/cache/media"
+
+# Syntax check
+if [ -z "$DESTINATION" ]; then
+ echo "usage: $BASENAME <DESTINATION>"
+ exit 1
+fi
+
+# Process each repository
+for file in `ls $MEDIA`; do
+ if [ -d "$MEDIA/$file/.git/annex" ]; then
+ (
+ echo Processing "$MEDIA/$file..."
+
+ cd $MEDIA
+
+ if ssh $DESTINATION if [ -d \"/var/cache/media/$file\" ] \; then echo exists\; fi | grep -q exists; then
+ echo "Remote $file already exists, skipping..."
+ continue;
+ fi
+
+ git clone $file $file.git && \
+ rsync -avz $file.git/ $DESTINATION:/var/cache/media/$file/ && \
+ rm -rf $file.git
+ echo ""
+ )
+ fi
+done
diff --git a/share/hydractl/sync-media-initremotes b/share/hydractl/sync-media-initremotes
new file mode 100755
index 0000000..839deea
--- /dev/null
+++ b/share/hydractl/sync-media-initremotes
@@ -0,0 +1,92 @@
+#!/bin/bash
+#
+# Add git-annex remotes to repository in removable media or local cache.
+#
+
+# Parameters
+BASENAME="`basename $0`"
+VOLUME="$1"
+TYPES="$2"
+DOMAIN="`facter DOMAIN`"
+HOST="`facter hostname`"
+CACHES=""
+VOLUMES=""
+BOXES=""
+
+# Fix identity
+function sync_media_identity {
+ if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
+ repo="$(basename `pwd`)"
+ git config user.name "${repo^} Archive"
+ git config user.email "$repo@localhost"
+ fi
+}
+
+# Syntax check
+if [ -z "$VOLUME" ]; then
+ echo "usage: $BASENAME <volume> [<caches|volumes|boxes> [remotes]]"
+ exit 1
+fi
+
+# Determine media folder
+if [ "$VOLUME" == "$HOST" ] || [ "$VOLUME" == "localhost" ]; then
+ MEDIA="/var/cache/media"
+else
+ MEDIA="/media/$VOLUME/media.$DOMAIN"
+fi
+
+# Determine remote type
+shift 2
+if [ "$TYPES" == 'caches' ]; then
+ # Remotes are caches of local boxes
+ CACHES="$*"
+elif [ "$TYPES" == 'volumes' ]; then
+ # Remotes are removable media
+ VOLUMES="$*"
+elif [ "$TYPES" == 'boxes' ]; then
+ # Remotes are remote boxes
+ BOXES="$*"
+fi
+
+# Process each repository
+for file in `ls $MEDIA`; do
+ if [ -d "$MEDIA/$file/.git" ]; then
+ (
+ echo Processing "$MEDIA/$file..."
+ cd $MEDIA/$file
+ sync_media_identity
+
+ if git remote | grep -q "^origin$"; then
+ echo "Removing origin..."
+ git remote rm origin
+ fi
+
+ if [ ! -d "$MEDIA/$file/.git/annex" ]; then
+ git annex init $VOLUME
+ fi
+
+ for remote in $CACHES; do
+ echo "Adding /var/cache/$remote/media/$file remote..."
+ git remote add $remote /var/cache/$remote/media/$file
+ done
+
+ for remote in $VOLUMES; do
+ echo "Adding /media/$remote/media.$DOMAIN/$file..."
+ git remote add $remote /media/$remote/media.$DOMAIN/$file
+ done
+
+ for remote in $BOXES; do
+ if ! echo $remote | grep -q '\.'; then
+ host="$remote.$DOMAIN"
+ else
+ host="$remote"
+ fi
+
+ echo "Adding ssh://$host/var/cache/media/$file..."
+ git remote add $remote ssh://$host/var/cache/media/$file
+ done
+
+ echo ""
+ )
+ fi
+done
diff --git a/share/hydractl/umount-media b/share/hydractl/umount-media
new file mode 120000
index 0000000..5848693
--- /dev/null
+++ b/share/hydractl/umount-media
@@ -0,0 +1 @@
+mount-media \ No newline at end of file