path: root/share/hydractl/sync-media

#!/bin/bash
#
# Sync media assets.
#

# Load
source $APP_BASE/lib/hydra/functions || exit 1
hydra_config_load

# Parameters
REMOTE="$1"
REPOSITORY="$2"
VOLUME="/media/$REMOTE"
DOMAIN="`facter domain`"
HOST="`facter hostname`"
CACHE="/var/cache/$HOST/media"
MEDIA="media.$DOMAIN"
INCOMING="$CACHE/incoming"
WHOAMI="`whoami`"
LOCK=".sync-media.lock"
OPTIONS="$*"

# Dependencies
hydra_install_package rsync
hydra_install_package unison
hydra_install_package git-annex

# Fatal error
# Adapted from borger
function fatal {
  info [fatal] $*
  exit 1;
}

# Create lockfile
# Adapted from borger
function sync_media_set_lockfile {
  if [ ! -z "$LOCKFILE" ]; then
    mkdir -p `dirname $LOCKFILE`

    if ( set -o noclobber; echo "$$" > "$LOCKFILE" ) &> /dev/null; then
      trap 'sync_media_unset_lockfile' INT TERM EXIT
    else
      fatal "Could not create lockfile $LOCKFILE, exiting"
    fi
  fi
}

# Remove lockfile
# Adapted from borger
function sync_media_unset_lockfile {
  if [ ! -z "$LOCKFILE" ] && [ -e "$LOCKFILE" ]; then
    git annex unannex $LOCK &> /dev/null
    git rm -f $LOCKFILE &> /dev/null || \
      rm -f $LOCKFILE || echo "Could not remove lockfile $LOCKFILE"
  fi
}

# Check lockfile
# Adapted from borger
function sync_media_check_lockfile {
  local pid process

  if [ ! -e ".gitignore" ] || ! grep -q "$LOCK" .gitignore; then
    # With git-ignore from git-extras package
    #git ignore $LOCK

    # Without git-ignore from git-extras package
    echo "$LOCK" >> .gitignore
  fi

  if [ ! -z "$LOCKFILE" ] && [ -f "$LOCKFILE" ]; then
    pid="`cat $LOCKFILE`"
    process="`ps --no-headers -o comm $pid`"

    if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then
      echo "Another program is running for $LOCKFILE, skipping"
      return 1
    else
      echo "Found old lockfile $LOCKFILE, removing it"
      sync_media_unset_lockfile
    fi

    return 0
  fi
}

# Fix identity
function sync_media_identity {
  if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then
    repo="$(basename `pwd`)"
    git config user.name "${repo^} Archive"
    git config user.email "$repo@localhost"
  fi
}

# Add files into the annex
function sync_media_add {
  git annex add .

  # Handle playlists which are generally not managed by git-annex
  if [ -d "playlists" ]; then
    git add playlists
  fi

  # Adds hidden files and symlinks, git+while version
  git status --porcelain -u | grep '^?? ' | sed -e 's/?? //' | while read file; do
    if [ -h "$file" ]; then
      git add "$file"
    else
      git annex add "$file"
    fi
  done
}

# Add meta files, making sure they're handled directly by Git
function sync_media_add_metadata {
  # Metadata state on the annex
  #
  # * add: metadata is managed in the annex, normally.
  # * unlock: managed in the annex, but modifications are tracked as well.
  # * unannex: metadata is kept out of the annex, but still tracked by Git.
  #
  # Usually, they should be regular files trackes by Git, so `git diff`
  # works as expected.
  #
  #metadata_state="add"
  #metadata_state="unlock"
  metadata_state="unannex"

  # Gitignore
  if [ -e ".gitignore" ]; then
    git annex $metadata_state .gitignore
    git add .gitignore
  fi

  # Playlist files in the playlist folder
  if [ -d "playlists" ]; then
    find playlists -name '*.m3u'  -type l -exec git annex $metadata_state {} \;
    find playlists -name '*.m3u'          -exec git add {} \;
    find playlists -name '*.m3u8' -type l -exec git annex $metadata_state {} \;
    find playlists -name '*.m3u8'         -exec git add {} \;
  fi

  # Koreader metadata files
  find -name metadata.pdf.lua     -type l -exec git annex $metadata_state {} \;
  find -name metadata.pdf.lua     -type l -exec git annex $metadata_state {} \;
  find -name metadata.pdf.lua             -exec git add {} \;
  find -name metadata.pdf.lua.old -type l -exec git annex $metadata_state {} \;
  find -name metadata.pdf.lua.old         -exec git add {} \;

  # Darktable sidecar files
  find -name '*.xmp' -type l -not -path '*.git*' -exec git annex $metadata_state {} \;
  find -name '*.xmp'         -not -path '*.git*' -exec git add {} \;
}

# If there is a playlists folder, make sure mpd user can write to it
function sync_media_playlist_perms {
  if [ -d "playlists" ]; then
    $sudo chmod 775 playlists
    $sudo chown -R mpd:audio playlists
    find playlists -type f -exec sudo chmod 664 {} \;
    find playlists -type d -exec sudo chmod 775 {} \;
  fi
}

# Fix incoming permissions
function sync_media_incoming_perms {
  if [ -d "$INCOMING" ]; then
    echo "Fixing $INCOMING permissions..."
    $sudo find $INCOMING -type f -exec chmod 664 {} \;
    $sudo find $INCOMING -type d -exec chmod 775 {} \;
    $sudo chown -R $WHOAMI:incoming $INCOMING
  fi
}

# Run fsck
function sync_media_fsck {
  if [ "$FSCK" == "true" ]; then
    git annex fsck --fast
  fi
}

# Run dropunused
function sync_media_dropunused {
  if [ "$DROPUNUSED" == "true" ]; then
    git annex unused
    git annex dropunused 1-1000
  fi
}

# Get copies of annexed files
function sync_media_get {
  local repo="$1"
  local numcopies

  if [ "`git -C $repo config sync-media.get`" != "false" ]; then
    if git -C $repo config sync-media.numcopies &> /dev/null; then
      numcopies="`git -C $repo config sync-media.numcopies`"
    else
      numcopies="3"
    fi

    git annex get . --numcopies=$numcopies
  fi
}

# Control whether the repository should have a copy of everything
function sync_media_getall {
  local repo="$1"

  if [ "`git -C $repo config sync-media.getall`" == "true" ]; then
    git annex get .
  fi
}

# Ensure we have a reference to the remote repository
function sync_media_ensure_remote {
  local remote="$1"
  local path="$2"

  if [ -z "$remote" ] || [ "$remote" == "local" ] || [ "$remote" == "localhost" ] || [ -z "$path" ]; then
    return
  fi

  # Check for local or remote repo
  if [ -z "$DRIVE" ]; then
    path="$REMOTE.$DOMAIN:$path"
  elif [ ! -d "$path/.git/annex" ]; then
    return
  fi

  if ! git remote | grep -q "^$remote$"; then
    git remote add $remote $path
  fi
}

# Set sudo config
if [ "$WHOAMI" != 'root' ]; then
  sudo="sudo"
else
  echo "Sorry, cannot run as root, since archives are usually user-managed"
  exit 1
fi

# Set fsck config
if echo $OPTIONS | grep -q -- "--fsck"; then
  FSCK="true"
fi

# Set unused config
if echo $OPTIONS | grep -q -- "--dropunused"; then
  DROPUNUSED="true"
fi

# Set drive config
# Ignore drive/volume if it's set to "local"
if [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ] && [ "$REMOTE" != "localhost" ]; then
  # Check storage media
  MOUNT="`mount | grep $VOLUME`"

  if [ ! -z "$MOUNT" ]; then
    DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)"
  #else
  #  echo "$BASENAME: volume $MEDIA is not mounted"
  #  exit 1
  fi
fi

# Ensure cache exists
mkdir -p $CACHE

# Fix cache permissions
#echo "Fixing $CACHE permissions..."
#$sudo find $CACHE -type f -exec chmod 644 {} \;
#$sudo find $CACHE -type d -exec chmod 755 {} \;
$sudo chown $WHOAMI: $CACHE
$sudo chown $WHOAMI: $CACHE/*

# Check if a specific repository was passed via the command line
if [ ! -z "$REPOSITORY" ] && [ -d "$CACHE/$REPOSITORY" ] && ! echo "$REPOSITORY" | grep -q -- '--'; then
  REPOSITORIES="$REPOSITORY"
else
  REPOSITORIES="`ls $CACHE`"
fi

if echo $REPOSITORIES | grep -q "incoming"; then
  sync_media_incoming_perms
fi

# Iterate over existing repositories in the local cache
for folder in $REPOSITORIES; do
  LOCKFILE="$CACHE/$folder/$LOCK"

  # Sync each repository in the local cache
  if [ -d "$CACHE/$folder/.git/annex" ]; then
    if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
      continue
    fi

    (
    cd $CACHE/$folder
    echo ""
    echo "Syncing $CACHE/$folder..."
    echo ""

    # Lockfile handling
    if ! sync_media_check_lockfile; then
      continue
    else
      sync_media_set_lockfile
    fi

    # Ensure the removable volume is in the list of remotes
    sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder

    # Ensure the repository is identified
    sync_media_identity

    # Disable automatic syncing of content
    git config annex.synccontent false

    # Sync before changing anything, to make sure the history does not have
    # conflicts with other remotes
    git annex sync --no-content

    # Fix any playlist permissions
    sync_media_playlist_perms

    # Process metadata
    sync_media_add_metadata

    # Add new content, and catch up changes
    sync_media_add

    # Sync everything again
    git annex sync --no-content

    # Get all that needs to be got
    sync_media_getall $CACHE/$folder

    # Repository maintenance
    sync_media_fsck
    sync_media_dropunused
    git prune
    git gc

    # Unset the lockfile
    sync_media_unset_lockfile
    )
  fi

  # Ensure the removable media has each repository listed in the local cache
  if [ ! -z "$DRIVE" ]; then
    mkdir -p $VOLUME/$MEDIA

    if [ -d "$CACHE/$folder/.git" ]; then
      if [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
        (
        cd $VOLUME/$MEDIA
        echo "Initializing $VOLUME/$MEDIA/$folder..."
        git clone $CACHE/$folder && cd $folder && sync_media_identity && git remote rename origin $HOST
        cd $CACHE/$folder && git remote add $DRIVE $VOLUME/$MEDIA/$folder
        )

        if [ -d "$CACHE/$folder/.git/annex" ]; then
          (
          cd $VOLUME/$MEDIA/$folder
          git annex init $DRIVE && git config sync-media.getall true
          )
        fi
      fi
    elif [ ! -d "$VOLUME/$MEDIA/$folder" ]; then
      mkdir -p $VOLUME/$MEDIA/$folder

      # Set timestamp old enough to help unison guess which copy is newer
      touch -t 197001010000 $VOLUME/$MEDIA/$folder
    fi
  fi
done

# Process removable or remote media
if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
  # Check if a specific repository was passed via the command line
  if [ ! -z "$REPOSITORY" ] && [ -d "$VOLUME/$MEDIA/$REPOSITORY" ] && ! echo "$REPOSITORY" | grep -q -- '--'; then
    REPOSITORIES="$REPOSITORY"
  else
    REPOSITORIES="`ls $VOLUME/$MEDIA`"
  fi

  # Iterate over existing repositories in the removable media
  for folder in $REPOSITORIES; do
    LOCKFILE="$VOLUME/$MEDIA/$folder/$LOCK"

    # Sync each local repository in the removable media
    if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then
      if [ "`git -C $VOLUME/$MEDIA/$folder config sync-media.skip`" == "true" ]; then
        continue
      fi

      (
      cd $VOLUME/$MEDIA/$folder
      echo ""
      echo "Syncing $VOLUME/$MEDIA/$folder..."
      echo ""

      # Lockfile handling
      if ! sync_media_check_lockfile; then
        continue
      else
        sync_media_set_lockfile
      fi

      sync_media_playlist_perms
      sync_media_ensure_remote $HOST $CACHE/$folder
      sync_media_identity
      sync_media_add

      # Disable automatic syncing of content
      git config annex.synccontent false

      git annex sync --no-content
      sync_media_get    $VOLUME/$MEDIA/$folder
      sync_media_getall $VOLUME/$MEDIA/$folder
      sync_media_fsck
      sync_media_dropunused
      git gc
      git prune
      #git annex drop --auto --numcopies=2

      # Unset the lockfile
      sync_media_unset_lockfile
      )
    elif [ -d "$CACHE/$folder" ] && [ ! -d "$CACHE/$folder/.git" ]; then
      # Avoid those configured to be skipped
      if [ ! -e "$CACHE/$folder/.sync-media/skip" ]; then
        echo "Syncing $CACHE/$folder with $VOLUME/$MEDIA/$folder..."

        if [ -e "$CACHE/$folder/.sync-media/method" ]; then
          method="`cat $CACHE/$folder/.sync-media/method`"
        else
          method="unison"
        fi

        if [ "$method" == "unison" ]; then
          unison $CACHE/$folder $VOLUME/$MEDIA/$folder -auto -logfile /dev/null
        elif [ "$method" == "rsync-to-media-volume" ]; then
          echo "Syncing $CACHE/$folder into $VOLUME/$MEDIA/$folder..."
          rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
        elif [ "$method" == "rsync-from-media-volume" ]; then
          echo "Syncing $VOLUME/$MEDIA/$folder into $CACHE/$folder..."
          rsync -av --delete --exclude=.sync-media $VOLUME/$MEDIA/$folder/ $CACHE/$folder/
        elif [ "$method" == "rsync-if-empty-dest" ]; then
          # Ensure both endpoint folders exist
          mkdir -p $CACHE/folder
          mkdir -p $VOLUME/$MEDIA/$folder

          # One way rsync method, from the non-empty to the empty folder
          if [ ! -z "`ls -1 $CACHE/$folder`" ] && [ -z "`ls -1 $VOLUME/$MEDIA/$folder`" ]; then
            echo "Syncing $CACHE/$folder into $VOLUME/$MEDIA/$folder..."
            rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $VOLUME/$MEDIA/$folder/
          elif [ ! -z "`ls -1 $VOLUME/$MEDIA/$folder`" ] && [ -z "`ls $CACHE/$folder`" ]; then
            echo "Syncing $VOLUME/$MEDIA/$folder into $CACHE/$folder..."
            rsync -av --delete --exclude=.sync-media $VOLUME/$MEDIA/$folder/ $CACHE/$folder/
          else
            echo "Skipping rsyncing between $CACHE/$folder and $VOLUME/$MEDIA/$folder since both are non-empty"
          fi
        elif [ "$method" == "" ]; then
          true
        else
          echo "Skipping unknown sync method $method"
        fi
      fi
    fi

    # Run an additional custom synchronizer
    if [ -x "$CACHE/$folder/.sync-media/custom" ]; then
      $CACHE/$folder/.sync-media/custom $VOLUME/$MEDIA/$folder
    fi

    # Ensure the local cache has each repository listed in the removable media
    if [ ! -d "$CACHE/$folder" ]; then
      if [ -d "$VOLUME/$MEDIA/$folder/.git" ]; then
        (
        cd $CACHE
        echo "Initializing $CACHE/$folder..."
        git clone $VOLUME/$MEDIA/$folder && cd $folder && sync_media_identity && git remote rename origin $REMOTE
        cd $VOLUME/$MEDIA/$folder && git remote add $HOST $CACHE/$folder
        )

        if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then
          (
          cd $CACHE/$folder
          git annex init $HOST
          )
        fi
      else
        echo "Syncing $VOLUME/$MEDIA/$folder into $CACHE/$folder..."
        rsync -av --delete --exclude=.sync-media $VOLUME/$MEDIA/$folder/ $CACHE/$folder/
      fi
    fi
  done
elif [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ] && [ "$REMOTE" != "localhost" ]; then
  # Try to copy to a remote
  for folder in `ls $CACHE`; do
    if [ -d "$CACHE/$folder/.git/annex" ]; then
      if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then
        continue
      fi

      sync_media_ensure_remote $REMOTE $CACHE/$folder

      (
      cd $CACHE/$folder
      git annex copy . --to $REMOTE
      git annex sync --no-content
      )
    else
      echo "Syncing $CACHE/$folder with ssh://$REMOTE.$DOMAIN/$CACHE/$folder..."
      unison $CACHE/$folder ssh://$REMOTE.$DOMAIN/$CACHE/$folder/ -auto -logfile /dev/null

      # Avoid empty source folders or those configured to be skipped
      #if [ ! -e "$CACHE/$folder/.sync-media/skip" ] && [ ! -z "`ls -1 $CACHE/$folder`" ]; then
      #  echo "Syncing $CACHE/$folder into $REMOTE.$DOMAIN:$CACHE/$folder..."
      #  rsync -av --delete --exclude=.sync-media $CACHE/$folder/ $REMOTE.$DOMAIN:$CACHE/$folder/
      #fi
    fi
  done
fi