#!/bin/bash # # Sync media assets. # # Load source $APP_BASE/lib/hydra/functions || exit 1 hydra_config_load # Parameters REMOTE="$1" VOLUME="/media/$REMOTE" DOMAIN="`facter domain`" HOST="`facter hostname`" CACHE="/var/cache/$HOST/media" MEDIA="media.$DOMAIN" INCOMING="$CACHE/incoming" WHOAMI="`whoami`" OPTIONS="$*" # Dependencies hydra_install_package rsync hydra_install_package unison # Fix identity function sync_media_identity { if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then repo="$(basename `pwd`)" git config user.name "${repo^} Archive" git config user.email "$repo@localhost" fi } # Add files into the annex function sync_media_add { git annex add . # Handle playlists which are generally not managed by git-annex if [ -d "playlists" ]; then git add playlists fi # Adds hidden files and symlinks, git+while version git status --porcelain -u | grep '^?? ' | sed -e 's/?? //' | while read file; do if [ -h "$file" ]; then git add "$file" else git annex add "$file" fi done } # Add meta files, making sure they're handled directly by Git function sync_media_add_metadata { # Koreader metadata files find -name metadata.pdf.lua -exec git annex unlock {} \; find -name metadata.pdf.lua -exec git add {} \; find -name metadata.pdf.lua.old -exec git annex unlock {} \; find -name metadata.pdf.lua.old -exec git add {} \; # Darktable sidecar files find -name '*.xmp' -exec git annex unlock {} \; find -name '*.xmp' -exec git add {} \; } # If there is a playlists folder, make sure mpd user can write to it function sync_media_playlist_perms { if [ -d "playlists" ]; then $sudo chmod 775 playlists $sudo chown -R mpd.audio playlists find playlists -type f -exec sudo chmod 664 {} \; find playlists -type d -exec sudo chmod 775 {} \; fi } # Fix incoming permissions function sync_media_incoming_perms { if [ -d "$INCOMING" ]; then echo "Fixing $INCOMING permissions..." $sudo find $INCOMING -type f -exec chmod 664 {} \; $sudo find $INCOMING -type d -exec chmod 775 {} \; $sudo chown -R $WHOAMI.incoming $INCOMING fi } # Run fsck function sync_media_fsck { if [ "$FSCK" == "true" ]; then git annex fsck --fast fi } # Run dropunused function sync_media_dropunused { if [ "$DROPUNUSED" == "true" ]; then git annex unused git annex dropunused 1-1000 fi } # Get copies of annexed files function sync_media_get { local repo="$1" local numcopies if [ "`git -C $repo config sync-media.get`" != "false" ]; then if git -C $repo config sync-media.numcopies &> /dev/null; then numcopies="`git -C $repo config sync-media.numcopies`" else numcopies="3" fi git annex get . --numcopies=$numcopies fi } # Control whether the repository should have a copy of everything function sync_media_getall { local repo="$1" if [ "`git -C $repo config sync-media.getall`" == "true" ]; then git annex get . fi } # Ensure we have a reference to the remote repository function sync_media_ensure_remote { local remote="$1" local path="$2" if [ -z "$remote" ] || [ -z "$path" ]; then return fi # Check for local or remote repo if [ -z "$DRIVE" ]; then path="$REMOTE.$DOMAIN:$path" elif [ ! -d "$path/.git/annex" ]; then return fi if ! git remote | grep -q "^$remote$"; then git remote add $remote $path fi } # Set sudo config if [ "$WHOAMI" != 'root' ]; then sudo="sudo" else echo "Sorry, cannot run as root" exit 1 fi # Set fsck config if echo $OPTIONS | grep -q -- "--fsck"; then FSCK="true" fi # Set unused config if echo $OPTIONS | grep -q -- "--dropunused"; then DROPUNUSED="true" fi # Set drive config if [ ! -z "$REMOTE" ]; then # Check storage media MOUNT="`mount | grep $VOLUME`" if [ ! -z "$MOUNT" ]; then DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)" fi fi # Ensure cache exists mkdir -p $CACHE # Fix cache permissions #echo "Fixing $CACHE permissions..." #$sudo find $CACHE -type f -exec chmod 644 {} \; #$sudo find $CACHE -type d -exec chmod 755 {} \; $sudo chown $WHOAMI. $CACHE $sudo chown $WHOAMI. $CACHE/* sync_media_incoming_perms # Iterate over existing repositories in the local cache for folder in `ls $CACHE`; do # Sync each repository in the local cache if [ -d "$CACHE/$folder/.git/annex" ]; then if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then continue fi ( cd $CACHE/$folder echo "Syncing $CACHE/$folder..." sync_media_add_metadata sync_media_playlist_perms sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder sync_media_identity sync_media_add git annex sync sync_media_getall $CACHE/$folder sync_media_fsck sync_media_dropunused ) fi # Ensure the removable media has each repository listed in the local cache if [ ! -z "$DRIVE" ]; then mkdir -p $VOLUME/$MEDIA if [ -d "$CACHE/$folder/.git" ]; then if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then ( cd $VOLUME/$MEDIA echo "Initializing $VOLUME/$MEDIA/$folder..." git clone $CACHE/$folder && cd $folder && sync_media_identity && git remote rename origin $HOST cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder ) if [ -d "$CACHE/$folder/.git/annex" ]; then ( cd $VOLUME/$MEDIA/$folder git annex init $DRIVE && git config sync-media.getall true ) fi fi elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then mkdir -p $VOLUME/$MEDIA/$folder # Set timestamp old enough to help unison guess which copy is newer touch -t 197001010000 $VOLUME/$MEDIA/$folder fi fi done # Process removable or remote media if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then # Iterate over existing repositories in the removable media for folder in `ls $VOLUME/$MEDIA`; do # Sync each local repository in the removable media if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then continue fi ( cd $VOLUME/$MEDIA/$folder echo "Syncing $VOLUME/$MEDIA/$folder..." sync_media_playlist_perms sync_media_ensure_remote $HOST $CACHE/$folder sync_media_identity sync_media_add git annex sync sync_media_get $VOLUME/$MEDIA/$folder sync_media_getall $VOLUME/$MEDIA/$folder sync_media_fsck sync_media_dropunused #git annex drop --auto --numcopies=2 ) elif [ -d "$CACHE/$folder" ] && [ ! -d "$CACHE/$folder/.git" ]; then # Avoid those configured to be skipped if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then echo "Syncing $CACHE/$folder with $VOLUME/$MEDIA/$folder..." unison $CACHE/$folder $VOLUME/$MEDIA/$folder -auto -logfile /dev/null # Avoid empty source folders #if [ ! -z "`ls -1 $CACHE/$folder`" ]; then # echo "Syncing $CACHE/$folder into $VOLUME/$MEDIA/$folder..." # rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/ #elif [ ! -z "`ls -1 $VOLUME/$MEDIA/$folder`" ]; then # echo "Syncing $VOLUME/$MEDIA/$folder into $CACHE/$folder..." # rsync -av --delete --exclude=.sync-media $VOLUME/$MEDIA/$folder/ $CACHE/$folder/ #fi fi fi # Run a custom synchronizer if [ -x "$CACHE/$folder/.sync-media/custom" ]; then $CACHE/$folder/.sync-media/custom $VOLUME/$MEDIA/$folder fi # Ensure the local cache has each repository listed in the removable media if [ ! -d "$CACHE/$folder" ]; then if [ -d "$VOLUME/$MEDIA/$folder/.git" ]; then ( cd $CACHE echo "Initializing $CACHE/$folder..." git clone $VOLUME/$MEDIA/$folder && cd $folder && sync_media_identity && git remote rename origin $REMOTE cd $VOLUME/$MEDIA/$folder && git remote add $HOST $CACHE/$folder ) if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then ( cd $CACHE/$folder git annex init $HOST ) fi else echo "Syncing $VOLUME/$MEDIA/$folder into $CACHE/$folder..." rsync -av --delete --exclude=.sync-media $VOLUME/$MEDIA/$folder/ $CACHE/$folder/ fi fi done elif [ ! -z "$REMOTE" ]; then # Try to copy to a remote for folder in `ls $CACHE`; do if [ -d "$CACHE/$folder/.git/annex" ]; then if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then continue fi sync_media_ensure_remote $REMOTE $CACHE/$folder ( cd $CACHE/$folder git annex copy . --to $REMOTE git annex sync ) else echo "Syncing $CACHE/$folder with ssh://$REMOTE.$DOMAIN/$CACHE/$folder..." unison $CACHE/$folder ssh://$REMOTE.$DOMAIN/$CACHE/$folder/ -auto -logfile /dev/null # Avoid empty source folders or those configured to be skipped #if [ ! -e "$CACHE/$folder/.sync-media/skip" ] && [ ! -z "`ls -1 $CACHE/$folder`" ]; then # echo "Syncing $CACHE/$folder into $REMOTE.$DOMAIN:$CACHE/$folder..." # rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/ #fi fi done fi