From 6d9d7b39bbf87b32a2e12fa6b29f463382c7027e Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sun, 2 Oct 2016 17:21:26 -0300 Subject: Move scripts to the Hydra Suite --- files/mount-media | 46 -------- files/sync-backups | 74 ------------ files/sync-media | 270 ------------------------------------------- files/sync-media-export | 37 ------ files/sync-media-initremotes | 92 --------------- manifests/init.pp | 25 ++-- 6 files changed, 13 insertions(+), 531 deletions(-) delete mode 100755 files/mount-media delete mode 100755 files/sync-backups delete mode 100755 files/sync-media delete mode 100755 files/sync-media-export delete mode 100755 files/sync-media-initremotes diff --git a/files/mount-media b/files/mount-media deleted file mode 100755 index 0073292..0000000 --- a/files/mount-media +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# -# mount-media -# - -# Parameters -MEDIA="$1" -VOLUME="$2" -BASENAME="`basename $0`" -MOUNTPOINT="/media/$MEDIA" - -# Check media config -if [ -z "$MEDIA" ]; then - echo "usage: $BASENAME [volume]" - echo "example: $BASENAME mymedia sdb1" - exit 1 -fi - -# Check volume config -if [ -z "$VOLUME" ]; then - VOLUME="sdb1" -fi - -DISK="`echo ${VOLUME} | sed -e s/[0-9]\$//g`" - -# Set sudo config -if [ "`whoami`" != 'root' ]; then - sudo="sudo" -fi - -if [ "$BASENAME" == "mount-media" ]; then - echo "Checking drive health status..." - $sudo smartctl -H /dev/$DISK - echo "Disabling STANDBY on drive..." - $sudo sdparm --clear STANDBY -6 /dev/$DISK - echo "Initializing crypto layer..." - $sudo cryptsetup luksOpen /dev/$VOLUME $MEDIA && \ - echo "Checking filesystem..." - $sudo fsck -v -y /dev/mapper/$MEDIA && \ - echo "Mounting volume at $MOUNTPOINT..." - $sudo mkdir -p $MOUNTPOINT - $sudo mount /dev/mapper/$MEDIA $MOUNTPOINT -elif [ "$BASENAME" == "umount-media" ]; then - $sudo umount $MOUNTPOINT && \ - $sudo cryptsetup luksClose $MEDIA -fi diff --git a/files/sync-backups b/files/sync-backups deleted file mode 100755 index aa92754..0000000 --- a/files/sync-backups +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# sync a removable volume with system backups. -# - -# Parameters. -BASENAME="`basename $0`" -VOLUME="$1" -MEDIA="/media/$VOLUME" -BWLIMIT=${BWLIMIT:=32000} -IMAGES="/var/data/crypt/" -RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete --bwlimit=$BWLIMIT" -CP="ionice -c 3 nice -n 19 cp" - -# Sync backups for a node. -function sync_backups_node { - if [ ! -z "$NODE" ]; then - # Get full node hostname. - NODE_HOSTNAME="`cat /var/vservers/$NODE/etc/hostname`" - if [ -z "$NODE_HOSTNAME" ]; then - NODE_HOSTNAME="$NODE" - fi - - # Sync local encrypted backup. - echo "Syncing /var/vservers/$NODE/var/backups/duplicity/..." - mkdir -p /$MEDIA/$NODE_HOSTNAME/duplicity - $RSYNC /var/vservers/$NODE/var/backups/duplicity/ /$MEDIA/$NODE_HOSTNAME/duplicity/ - - # Sync remote backups. - for node in `ls /var/vservers/$NODE/var/backups/remote/`; do - echo "Syncing /var/vservers/$NODE/var/backups/remote/$node/..." - mkdir -p /$MEDIA/$NODE_HOSTNAME/remote/$node - $RSYNC /var/vservers/$NODE/var/backups/remote/$node/ /$MEDIA/$NODE_HOSTNAME/remote/$node/ - done - else - # Sync local encrypted backup. - echo "Syncing /var/backups/duplicity/..." - mkdir -p /$MEDIA/$HOSTNAME/duplicity - $RSYNC /var/backups/duplicity/ /$MEDIA/$HOSTNAME/duplicity/ - - # Sync remote backups. - for node in `ls /var/backups/remote/`; do - echo "Syncing /var/backups/remote/$node/..." - mkdir -p /$MEDIA/$HOSTNAME/remote/$node/ - $RSYNC /var/backups/remote/$node/ /$MEDIA/$HOSTNAME/remote/$node/ - done - - # Copy encrypted images. - if [ -d "$IMAGES" ]; then - for image in `find $IMAGES -name '*.img' -type f`; do - echo "Copying image to /$MEDIA/$HOSTNAME/images/`dirname $image`" - mkdir -p /$MEDIA/$HOSTNAME/images/`dirname $image` - $CP $image /$MEDIA/$HOSTNAME/images/`dirname $image` - done - fi - fi -} - -# Parsing. -if [ -z "$VOLUME" ]; then - echo "usage: $BASENAME [nodes]" - exit 1 -else - shift -fi - -# Sync backups for each node. -if [ ! -z "$*" ]; then - for NODE in $*; do - sync_backups_node - done -else - sync_backups_node -fi diff --git a/files/sync-media b/files/sync-media deleted file mode 100755 index d610245..0000000 --- a/files/sync-media +++ /dev/null @@ -1,270 +0,0 @@ -#!/bin/bash -# -# sync-media assets using git-annex -# - -REMOTE="$1" -VOLUME="/media/$REMOTE" -DOMAIN="`facter domain`" -HOST="`facter hostname`" -CACHE="/var/cache/$HOST/media" -MEDIA="media.$DOMAIN" -INCOMING="$CACHE/incoming" -WHOAMI="`whoami`" -OPTIONS="$*" - -# Fix identity -function sync_media_identity { - if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then - repo="$(basename `pwd`)" - git config user.name "${repo^} Archive" - git config user.email "$repo@localhost" - fi -} - -# Add files into the annex -function sync_media_add { - git annex add . - - # Adding hidden files and symlinks, git+while version - git status --porcelain -u | sed -e 's/?? //' | while read file; do - if [ -h "$file" ]; then - git add "$file" - else - git annex add "$file" - fi - done -} - -# If there is a playlists folder, make sure mpd user can write to it -function sync_media_playlist_perms { - if [ -d "playlists" ]; then - $sudo chmod 775 playlists - $sudo chown -R mpd.audio playlists - find playlists -type f -exec sudo chmod 664 {} \; - find playlists -type d -exec sudo chmod 775 {} \; - fi -} - -# Fix incoming permissions -function sync_media_incoming_perms { - if [ -d "$INCOMING" ]; then - echo "Fixing $INCOMING permissions..." - $sudo find $INCOMING -type f -exec chmod 664 {} \; - $sudo find $INCOMING -type d -exec chmod 775 {} \; - $sudo chown -R $WHOAMI.incoming $INCOMING - fi -} - -# Run fsck -function sync_media_fsck { - if [ "$FSCK" == "true" ]; then - git annex fsck --fast - fi -} - -# Run dropunused -function sync_media_dropunused { - if [ "$DROPUNUSED" == "true" ]; then - git annex unused - git annex dropunused 1-1000 - fi -} - -# Get copies of annexed files -function sync_media_get { - local repo="$1" - local numcopies - - if [ "`git -C $repo config sync-media.get`" != "false" ]; then - if git -C $repo config sync-media.numcopies &> /dev/null; then - numcopies="`git -C $repo config sync-media.numcopies`" - else - numcopies="3" - fi - - git annex get . --numcopies=$numcopies - fi -} - -# Control whether the repository should have a copy of everything -function sync_media_getall { - local repo="$1" - - if [ "`git -C $repo config sync-media.getall`" == "true" ]; then - git annex get . - fi -} - -# Ensure we have a reference to the remote repository -function sync_media_ensure_remote { - local remote="$1" - local path="$2" - - if [ -z "$remote" ] || [ -z "$path" ]; then - return - fi - - # Check for local or remote repo - if [ -z "$DRIVE" ]; then - path="$REMOTE.$DOMAIN:$path" - elif [ ! -d "$path/.git/annex" ]; then - return - fi - - if ! git remote | grep -q "^$remote$"; then - echo git remote add $remote $path - fi -} - -# Set sudo config -if [ "$WHOAMI" != 'root' ]; then - sudo="sudo" -else - echo "Sorry, cannot run as root" - exit 1 -fi - -# Set fsck config -if echo $OPTIONS | grep -q -- "--fsck"; then - FSCK="true" -fi - -# Set unused config -if echo $OPTIONS | grep -q -- "--dropunused"; then - DROPUNUSED="true" -fi - -# Set drive config -if [ ! -z "$REMOTE" ]; then - # Check storage media - MOUNT="`mount | grep $VOLUME`" - if [ ! -z "$MOUNT" ]; then - DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)" - fi -fi - -# Commit changes -if [ -d "$CACHE" ]; then - # Fix cache permissions - #echo "Fixing $CACHE permissions..." - #$sudo find $CACHE -type f -exec chmod 644 {} \; - #$sudo find $CACHE -type d -exec chmod 755 {} \; - - sync_media_incoming_perms - - # Add and update local repositories - for folder in `ls $CACHE`; do - if [ -d "$CACHE/$folder/.git/annex" ]; then - if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then - continue - fi - - #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then - # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo." - # continue - #fi - - ( - cd $CACHE/$folder - echo "Syncing $CACHE/$folder..." - - sync_media_playlist_perms - sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder - sync_media_identity - sync_media_add - git annex sync - sync_media_getall $CACHE/$folder - sync_media_fsck - sync_media_dropunused - ) - fi - done - - if [ ! -z "$DRIVE" ]; then - if [ ! -d "$VOLUME/$MEDIA" ]; then - echo "Folder $VOLUME/$MEDIA does not exist..." - else - for folder in `ls $CACHE`; do - if [ -d "$CACHE/$folder/.git/annex" ]; then - if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then - continue - fi - - #if [ "`git -C $CACHE/$folder config sync-media.ready`" != "true" ]; then - # echo "Skipping $CACHE/$folder: not sync-media ready, please config your repo." - # continue - #fi - - if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then - ( - cd $VOLUME/$MEDIA - echo "Initializing $VOLUME/$MEDIA/$folder..." - git clone $CACHE/$folder && cd $folder && sync_media_identity && git annex init $DRIVE && \ - git remote rename origin $HOST && cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder - ) - fi - elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then - if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then - echo "Syncing $VOLUME/$MEDIA/$folder..." - rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/ - fi - fi - done - fi - fi -fi - -# Retrieve changes at media volumes -if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then - for folder in `ls $VOLUME/$MEDIA`; do - if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then - if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then - continue - fi - - ( - cd $VOLUME/$MEDIA/$folder - echo "Syncing $VOLUME/$MEDIA/$folder..." - - sync_media_ensure_remote $HOST $CACHE/$folder - sync_media_playlist_perms - sync_media_identity - sync_media_add - git annex sync - sync_media_get $VOLUME/$MEDIA/$folder - sync_media_getall $VOLUME/$MEDIA/$folder - sync_media_fsck - sync_media_dropunused - #git annex drop --auto --numcopies=2 - ) - elif [ -d "$CACHE/$folder" ]; then - if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then - echo "Syncing $VOLUME/$MEDIA/$folder..." - rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/ - fi - fi - done -elif [ ! -z "$REMOTE" ]; then - # Try to copy to a remote - for folder in `ls $CACHE`; do - if [ -d "$CACHE/$folder/.git/annex" ]; then - if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then - continue - fi - - sync_media_ensure_remote $REMOTE $CACHE/$folder - - ( - cd $CACHE/$folder - git annex copy . --to $REMOTE - git annex sync - ) - else - if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then - echo "Syncing $VOLUME/$MEDIA/$folder..." - rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/ - fi - fi - done -fi diff --git a/files/sync-media-export b/files/sync-media-export deleted file mode 100755 index 7b47639..0000000 --- a/files/sync-media-export +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# -# Copy git-annex repositories to remote server. -# - -# Parameters -BASENAME="`basename $0`" -DESTINATION="$1" -DOMAIN="`facter DOMAIN`" -MEDIA="/var/cache/media" - -# Syntax check -if [ -z "$DESTINATION" ]; then - echo "usage: $BASENAME " - exit 1 -fi - -# Process each repository -for file in `ls $MEDIA`; do - if [ -d "$MEDIA/$file/.git/annex" ]; then - ( - echo Processing "$MEDIA/$file..." - - cd $MEDIA - - if ssh $DESTINATION if [ -d \"/var/cache/media/$file\" ] \; then echo exists\; fi | grep -q exists; then - echo "Remote $file already exists, skipping..." - continue; - fi - - git clone $file $file.git && \ - rsync -avz $file.git/ $DESTINATION:/var/cache/media/$file/ && \ - rm -rf $file.git - echo "" - ) - fi -done diff --git a/files/sync-media-initremotes b/files/sync-media-initremotes deleted file mode 100755 index 839deea..0000000 --- a/files/sync-media-initremotes +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -# -# Add git-annex remotes to repository in removable media or local cache. -# - -# Parameters -BASENAME="`basename $0`" -VOLUME="$1" -TYPES="$2" -DOMAIN="`facter DOMAIN`" -HOST="`facter hostname`" -CACHES="" -VOLUMES="" -BOXES="" - -# Fix identity -function sync_media_identity { - if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then - repo="$(basename `pwd`)" - git config user.name "${repo^} Archive" - git config user.email "$repo@localhost" - fi -} - -# Syntax check -if [ -z "$VOLUME" ]; then - echo "usage: $BASENAME [ [remotes]]" - exit 1 -fi - -# Determine media folder -if [ "$VOLUME" == "$HOST" ] || [ "$VOLUME" == "localhost" ]; then - MEDIA="/var/cache/media" -else - MEDIA="/media/$VOLUME/media.$DOMAIN" -fi - -# Determine remote type -shift 2 -if [ "$TYPES" == 'caches' ]; then - # Remotes are caches of local boxes - CACHES="$*" -elif [ "$TYPES" == 'volumes' ]; then - # Remotes are removable media - VOLUMES="$*" -elif [ "$TYPES" == 'boxes' ]; then - # Remotes are remote boxes - BOXES="$*" -fi - -# Process each repository -for file in `ls $MEDIA`; do - if [ -d "$MEDIA/$file/.git" ]; then - ( - echo Processing "$MEDIA/$file..." - cd $MEDIA/$file - sync_media_identity - - if git remote | grep -q "^origin$"; then - echo "Removing origin..." - git remote rm origin - fi - - if [ ! -d "$MEDIA/$file/.git/annex" ]; then - git annex init $VOLUME - fi - - for remote in $CACHES; do - echo "Adding /var/cache/$remote/media/$file remote..." - git remote add $remote /var/cache/$remote/media/$file - done - - for remote in $VOLUMES; do - echo "Adding /media/$remote/media.$DOMAIN/$file..." - git remote add $remote /media/$remote/media.$DOMAIN/$file - done - - for remote in $BOXES; do - if ! echo $remote | grep -q '\.'; then - host="$remote.$DOMAIN" - else - host="$remote" - fi - - echo "Adding ssh://$host/var/cache/media/$file..." - git remote add $remote ssh://$host/var/cache/media/$file - done - - echo "" - ) - fi -done diff --git a/manifests/init.pp b/manifests/init.pp index b5ba119..e272b56 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -132,33 +132,34 @@ class backup( require => Package['backupninja'], } + # TODO: remove in the future # sync-backups script file { "/usr/local/sbin/sync-backups": owner => root, group => root, mode => 0755, - ensure => present, - source => "puppet:///modules/backup/sync-backups", + ensure => absent, } + # TODO: remove in the future # sync-media script file { "/usr/local/sbin/sync-media": owner => root, group => root, mode => 0755, - ensure => present, - source => "puppet:///modules/backup/sync-media", + ensure => absent, } + # TODO: remove in the future # sync-media-export script file { "/usr/local/sbin/sync-media-export": owner => root, group => root, mode => 0755, - ensure => present, - source => "puppet:///modules/backup/sync-media-export", + ensure => absent, } + # TODO: remove in the future # sync-media-init script file { "/usr/local/sbin/sync-media-init": owner => root, @@ -167,29 +168,29 @@ class backup( ensure => absent, } + # TODO: remove in the future # sync-media-init script file { "/usr/local/sbin/sync-media-initremotes": owner => root, group => root, mode => 0755, - ensure => present, - source => "puppet:///modules/backup/sync-media-initremotes", + ensure => absent, } + # TODO: remove in the future # mount-media script file { "/usr/local/sbin/mount-media": owner => root, group => root, mode => 0755, - ensure => present, - source => "puppet:///modules/backup/mount-media", + ensure => absent, } + # TODO: remove in the future # umount-media script file { "/usr/local/sbin/umount-media": owner => root, group => root, - ensure => '/usr/local/sbin/mount-media', - require => File['/usr/local/sbin/mount-media'], + ensure => absent, } } -- cgit v1.2.3