diff options
| -rw-r--r-- | docs/backups.md | 9 | ||||
| -rw-r--r-- | docs/changelog.md | 7 | ||||
| -rw-r--r-- | docs/index.md | 8 | ||||
| -rw-r--r-- | docs/todo.md | 10 | ||||
| -rwxr-xr-x | share/hydractl/sync-backups | 4 | ||||
| -rwxr-xr-x | share/hydractl/sync-media | 130 | ||||
| -rwxr-xr-x | share/hydractl/sync-media-initremotes | 2 |
7 files changed, 152 insertions, 18 deletions
diff --git a/docs/backups.md b/docs/backups.md index d39a85c..bbb6bb1 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -123,12 +123,13 @@ If so, proceed as follows with the appliance device connected in your TPC: ## Smartphone -Smartphones usually have their own way to be backed up. This is an example -based on the [android-backup][] utility: +Smartphones usually have their own way to be backed up. +The following examples uses scripts from the [utils-android][] repository: - android-backup <device-name> + android-backup-adb <device-name> # backups using adb + android-backup-mtp <device-name> <profile> # backups using MTP file transfer -[android-backup]: https://git.fluxo.info/scripts/tree/android-backup +[utils-android]: https://git.fluxo.info/utils-android/ ## Hardware rotation diff --git a/docs/changelog.md b/docs/changelog.md index 155459b..011c3bb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -19,3 +19,10 @@ * [x] Provision: increase default partition sizes from 20G to 40G, as nowadays distro size increase a lot and a 20G system partition can be filled in easily. +* [x] Sync media: + * [x] Now `sync-media` does a `git unannex` on sidecar and other metadata + files and manage them through Git, instead of keeping them as unlocked + files. + * [x] Added support for `.m3u8` files. + * [x] Sync lock for git-annex repositories, respected by other applications + such as `docshower` from [utils-doc][https://git.fluxo.info/utils-doc]. diff --git a/docs/index.md b/docs/index.md index 66915c2..8f1853c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,8 +2,10 @@ { align=left } -Hydra is a suite for orchestration and management of machines composed of -**lots of conventions and assumptions**, and the following basic commands: +Hydra is a suite for orchestration and management of machines, with **lots of +conventions and assumptions**. + +It's basically composed by the following commands: * `hydractl`: act in the current host. * `hydra`: act on a set of hosts. @@ -12,7 +14,7 @@ Hydra is a suite for orchestration and management of machines composed of An Hydra is not a "cloud computing" platform, it's something else: a set of systems that operate together, where any node can be used to spawn new nodes. -This is an ongoing experiment in how a person or a collective can manage +This is an ongoing experiment in how a single person or a collective can manage many computers in an unified way. It's not production ready, and it always struggles to pass the test of time. diff --git a/docs/todo.md b/docs/todo.md index 48cbf94..1346660 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -4,8 +4,9 @@ * [ ] Deploy: * [ ] Collect basic hardware information along with facts: cpuid, - dmidecode, hwinfo, lsblk, lscpu, lshw, lspci, lsusb etc. + dmidecode, hwinfo, lsblk, lscpu, lshw, lspci, lsusb, sfdisk etc. Store somewhere under the `config/hardware` folder? + Could also be a separate action, `hydra <hydra> collect <node>`. * [ ] Command line is broken for ansible when multiple nodes are provided. * [ ] Use console-based GnuPG agent when calling `keyringer`. * [ ] Mass: @@ -24,9 +25,14 @@ > [perhaps a '--transport=<tn>' or '--vendor=<vn>' option is needed] * [ ] Mount/umount system volume supporting split partiton scheme (`root`, `var`, `home` etc). - * [ ] Try to detected the device partition (`/dev/sdb1` etc) based + * [ ] Try to detect the device partition (`/dev/sdb1` etc) based on the LUKS2 label. + * [ ] Cryptdisks script needs fixing as `systemd-tty-ask-password-agent` is not + catching cryptsetup password prompts anymore. Otherwise, consider to + deprecate this script. * [ ] Syncing: + * [ ] Run `git submodule --update --init --recursive` after running + `git annex sync`. * [ ] Syncing packages: a frontend to `apt-offline` that uses `git-annex` repositories: getting, installing, cleaning. One node can request packages through an external drive, and another can fetch then. diff --git a/share/hydractl/sync-backups b/share/hydractl/sync-backups index 9cc79de..63e11b4 100755 --- a/share/hydractl/sync-backups +++ b/share/hydractl/sync-backups @@ -84,13 +84,13 @@ fi # Check volume name if [ "$VOLUME" == "`hostname -f`" ]; then - echo "volume is the hostname, cannot sync to myself" + echo "$BASENAME: volume is the hostname, cannot sync to myself" exit 1 fi # Check if it is mounted if ! mount | grep -q $MEDIA; then - echo "volume $MEDIA is not mounted" + echo "$BASENAME: volume $MEDIA is not mounted" exit 1 fi diff --git a/share/hydractl/sync-media b/share/hydractl/sync-media index 8132315..2373413 100755 --- a/share/hydractl/sync-media +++ b/share/hydractl/sync-media @@ -17,6 +17,7 @@ CACHE="/var/cache/$HOST/media" MEDIA="media.$DOMAIN" INCOMING="$CACHE/incoming" WHOAMI="`whoami`" +LOCK=".sync-media.lock" OPTIONS="$*" # Dependencies @@ -24,6 +25,66 @@ hydra_install_package rsync hydra_install_package unison hydra_install_package git-annex +# Fatal error +# Adapted from borger +function fatal { + info [fatal] $* + exit 1; +} + +# Create lockfile +# Adapted from borger +function sync_media_set_lockfile { + if [ ! -z "$LOCKFILE" ]; then + mkdir -p `dirname $LOCKFILE` + + if ( set -o noclobber; echo "$$" > "$LOCKFILE" ) &> /dev/null; then + trap 'sync_media_unset_lockfile' INT TERM EXIT + else + fatal "Could not create lockfile $LOCKFILE, exiting" + fi + fi +} + +# Remove lockfile +# Adapted from borger +function sync_media_unset_lockfile { + if [ ! -z "$LOCKFILE" ] && [ -e "$LOCKFILE" ]; then + git annex unannex $LOCK &> /dev/null + git rm -f $LOCKFILE &> /dev/null || \ + rm -f $LOCKFILE || echo "Could not remove lockfile $LOCKFILE" + fi +} + +# Check lockfile +# Adapted from borger +function sync_media_check_lockfile { + local pid process + + if [ ! -e ".gitignore" ] || ! grep -q "$LOCK" .gitignore; then + # With git-ignore from git-extras package + #git ignore $LOCK + + # Without git-ignore from git-extras package + echo "$LOCK" >> .gitignore + fi + + if [ ! -z "$LOCKFILE" ] && [ -f "$LOCKFILE" ]; then + pid="`cat $LOCKFILE`" + process="`ps --no-headers -o comm $pid`" + + if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then + echo "Another program is running for $LOCKFILE, skipping" + return 1 + else + echo "Found old lockfile $LOCKFILE, removing it" + sync_media_unset_lockfile + fi + + return 0 + fi +} + # Fix identity function sync_media_identity { if [ -z "`git config --local user.email`" ] || [ -z "`git config --local user.name`" ]; then @@ -54,20 +115,42 @@ function sync_media_add { # Add meta files, making sure they're handled directly by Git function sync_media_add_metadata { + # Metadata state on the annex + # + # * add: metadata is managed in the annex, normally. + # * unlock: managed in the annex, but modifications are tracked as well. + # * unannex: metadata is kept out of the annex, but still tracked by Git. + # + # Usually, they should be regular files trackes by Git, so `git diff` + # works as expected. + # + #metadata_state="add" + #metadata_state="unlock" + metadata_state="unannex" + + # Gitignore + if [ -e ".gitignore" ]; then + git annex $metadata_state .gitignore + git add .gitignore + fi + # Playlist files in the playlist folder if [ -d "playlists" ]; then - find playlists -name '*.m3u' -type l -exec git annex unlock {} \; - find playlists -name '*.m3u' -exec git add {} \; + find playlists -name '*.m3u' -type l -exec git annex $metadata_state {} \; + find playlists -name '*.m3u' -exec git add {} \; + find playlists -name '*.m3u8' -type l -exec git annex $metadata_state {} \; + find playlists -name '*.m3u8' -exec git add {} \; fi # Koreader metadata files - find -name metadata.pdf.lua -type l -exec git annex unlock {} \; + find -name metadata.pdf.lua -type l -exec git annex $metadata_state {} \; + find -name metadata.pdf.lua -type l -exec git annex $metadata_state {} \; find -name metadata.pdf.lua -exec git add {} \; - find -name metadata.pdf.lua.old -type l -exec git annex unlock {} \; + find -name metadata.pdf.lua.old -type l -exec git annex $metadata_state {} \; find -name metadata.pdf.lua.old -exec git add {} \; # Darktable sidecar files - find -name '*.xmp' -type l -not -path '*.git*' -exec git annex unlock {} \; + find -name '*.xmp' -type l -not -path '*.git*' -exec git annex $metadata_state {} \; find -name '*.xmp' -not -path '*.git*' -exec git add {} \; } @@ -171,12 +254,16 @@ if echo $OPTIONS | grep -q -- "--dropunused"; then fi # Set drive config -# Ingore drive/volume if it's set to "local" +# Ignore drive/volume if it's set to "local" if [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ] && [ "$REMOTE" != "localhost" ]; then # Check storage media MOUNT="`mount | grep $VOLUME`" + if [ ! -z "$MOUNT" ]; then DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)" + #else + # echo "$BASENAME: volume $MEDIA is not mounted" + # exit 1 fi fi @@ -203,6 +290,8 @@ fi # Iterate over existing repositories in the local cache for folder in $REPOSITORIES; do + LOCKFILE="$CACHE/$folder/$LOCK" + # Sync each repository in the local cache if [ -d "$CACHE/$folder/.git/annex" ]; then if [ "`git -C $CACHE/$folder config sync-media.skip`" == "true" ]; then @@ -211,7 +300,16 @@ for folder in $REPOSITORIES; do ( cd $CACHE/$folder + echo "" echo "Syncing $CACHE/$folder..." + echo "" + + # Lockfile handling + if ! sync_media_check_lockfile; then + continue + else + sync_media_set_lockfile + fi # Ensure the removable volume is in the list of remotes sync_media_ensure_remote $REMOTE $VOLUME/$MEDIA/$folder @@ -244,6 +342,11 @@ for folder in $REPOSITORIES; do # Repository maintenance sync_media_fsck sync_media_dropunused + git prune + git gc + + # Unset the lockfile + sync_media_unset_lockfile ) fi @@ -287,6 +390,7 @@ if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then # Iterate over existing repositories in the removable media for folder in $REPOSITORIES; do + LOCKFILE="$VOLUME/$MEDIA/$folder/$LOCK" # Sync each local repository in the removable media if [ -d "$VOLUME/$MEDIA/$folder/.git/annex" ]; then @@ -296,7 +400,16 @@ if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then ( cd $VOLUME/$MEDIA/$folder + echo "" echo "Syncing $VOLUME/$MEDIA/$folder..." + echo "" + + # Lockfile handling + if ! sync_media_check_lockfile; then + continue + else + sync_media_set_lockfile + fi sync_media_playlist_perms sync_media_ensure_remote $HOST $CACHE/$folder @@ -311,7 +424,12 @@ if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then sync_media_getall $VOLUME/$MEDIA/$folder sync_media_fsck sync_media_dropunused + git gc + git prune #git annex drop --auto --numcopies=2 + + # Unset the lockfile + sync_media_unset_lockfile ) elif [ -d "$CACHE/$folder" ] && [ ! -d "$CACHE/$folder/.git" ]; then # Avoid those configured to be skipped diff --git a/share/hydractl/sync-media-initremotes b/share/hydractl/sync-media-initremotes index 839deea..5b851f6 100755 --- a/share/hydractl/sync-media-initremotes +++ b/share/hydractl/sync-media-initremotes @@ -7,7 +7,7 @@ BASENAME="`basename $0`" VOLUME="$1" TYPES="$2" -DOMAIN="`facter DOMAIN`" +DOMAIN="`facter domain`" HOST="`facter hostname`" CACHES="" VOLUMES="" |