aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
l---------ChangeLog.md1
-rw-r--r--Makefile4
-rw-r--r--README.md5
l---------TODO.md1
-rw-r--r--docs/backups.md134
-rw-r--r--docs/changelog.md21
-rw-r--r--docs/ideas.md2
-rw-r--r--docs/index.md4
-rw-r--r--docs/todo.md55
-rw-r--r--docs/tpc.md113
-rw-r--r--lib/hydra/deploy4
-rw-r--r--lib/hydra/provision2
-rw-r--r--mkdocs.yml74
-rwxr-xr-xshare/hydra/deploy6
-rwxr-xr-xshare/hydra/import-keys8
-rwxr-xr-xshare/hydra/newkeys7
-rwxr-xr-xshare/hydractl/dist-upgrade7
-rwxr-xr-xshare/hydractl/provision8
-rwxr-xr-xshare/hydractl/provision-raspi2
-rwxr-xr-xshare/hydractl/remove-dep2
-rwxr-xr-xshare/hydractl/sync-backups17
-rwxr-xr-xshare/hydractl/sync-home2
-rwxr-xr-xshare/hydractl/sync-media35
-rwxr-xr-xshare/hydractl/sync-tpc2
25 files changed, 429 insertions, 88 deletions
diff --git a/.gitignore b/.gitignore
index c391806..ac84f83 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
*.swp
doc/_build
site
+.cache
diff --git a/ChangeLog.md b/ChangeLog.md
new file mode 120000
index 0000000..1bed66b
--- /dev/null
+++ b/ChangeLog.md
@@ -0,0 +1 @@
+docs/changelog.md \ No newline at end of file
diff --git a/Makefile b/Makefile
index 6e28c6c..ee069d6 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,11 @@
.PHONY: publish docs
-publish: docs
+web_deploy:
@rsync -avz --delete site/ hydra:/var/sites/hidra/www/
@rsync -avz --delete slides/ hydra:/var/sites/hidra/www/slides/
docs:
@mkdocs build
+
+publish: docs web_deploy
diff --git a/README.md b/README.md
index 39c4f96..aeeb620 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
# The Hydra Suite
-This is a highly opinionated set of scripts for managing computer configurations.
+This is a highly opinionated set of scripts for managing computer
+configurations.
-Se details at the [docs](doc/index.rst).
+Se details at the [docs](docs/index.md).
diff --git a/TODO.md b/TODO.md
new file mode 120000
index 0000000..338f3c3
--- /dev/null
+++ b/TODO.md
@@ -0,0 +1 @@
+docs/todo.md \ No newline at end of file
diff --git a/docs/backups.md b/docs/backups.md
index f6faf8a..d39a85c 100644
--- a/docs/backups.md
+++ b/docs/backups.md
@@ -11,33 +11,73 @@ This mostly has to do with `hydractl` commands such as
* But all this does not exempt a team from manually testing backups and to make
their own offline copies in external drives.
+## Parameters
+
+Start by running the following commands in your terminal, adjusting to your
+case:
+
+ export HYDRA="hydra-name"
+ export DISK="disk-name"
+ export DEVICE="/dev/sdc"
+ export USER="`whoami`"
+ export DOMAIN="`facter domain`"
+
+ if [ "$USER" != "root" ]; then
+ export SUDO="sudo"
+ fi
+
## New external drive
-Proceed as follows
+To add a new external drive into the pool, proceed as follows.
+
+First make sure to create a passphrase for the new disk into your hydra's keyring:
+
+ keyringer $HYDRA pwgen disks/$DISK.$DOMAIN/luks/root
+
+If you're using a USB drive, run this before plugging it on the computer:
+
+ hydractl usb-enable
+
+With the disk plugged in, make a layout with single `${DEVICE}1` partition
+(example: a disk with only one partition like `/dev/sdc1`):
+
+ $SUDO cfdisk ${DEVICE}
+
+Now create the LUKS encryption header in this new disk, using the passphrase
+generated above by keyringer:
- disk=new-disk-name
- device=/dev/sdc
- hydractl usb-enable # run this for USB drives, then connect the disk
- sudo cfdisk ${device} # layout with single ${device}1 partition
- sudo cryptsetup luksFormat ${device}1
- sudo cryptsetup luksOpen ${device}1 $disk
- sudo mkfs.ext4 /dev/mapper/$disk
- sudo mkdir /media/$disk
- sudo mount /dev/mapper/$disk /media/$disk
- sudo mkdir /media/$disk/media.`facter domain`
- sudo chown `whoami`: /media/$disk/media.`facter domain`
+ $SUDO cryptsetup luksFormat ${DEVICE}1
-## NAS
+If you want to make this volume to have a label (helpful for detecting the disk
+etc; downside: third parties can easily get the name), run this command:
+
+ $SUDO cryptsetup config ${DEVICE}1 --label ${DISK}
+
+Open the encrypted volume and create the basic filesystem structure needed
+to sync content:
+
+ $SUDO cryptsetup luksOpen ${DEVICE}1 $DISK
+ $SUDO mkfs.ext4 /dev/mapper/$DISK
+ $SUDO mkdir /media/$DISK
+ $SUDO mount /dev/mapper/$DISK /media/$DISK
+ $SUDO mkdir /media/$DISK/media.$DOMAIN
+ $SUDO chown ${USER}: /media/$DISK/media.$DOMAIN
+
+Finally, umount the drive:
+
+ hydractl umount-media $DISK
+
+## Regular sync
These commands should be enough to sync all media archives:
hydractl usb-enable # run this for USB drives, then connect the disk
- hydractl mount-media $MEDIA
- hydractl sync-media $MEDIA
+ hydractl mount-media $DISK
+ hydractl sync-media $DISK
As this should handle syncing all backups:
- hydractl sync-backups $MEDIA
+ hydractl sync-backups $DISK
## TPC
@@ -57,6 +97,11 @@ Then do the following:
hydractl mount-media $TPC
hydractl sync-tpc $TPC
+If you don't want to do a full TPC sync, but just want to sync the home folders
+(which is faster), use the following instead of the last command above:
+
+ hydractl sync-home $TPC
+
To sync archives and remote backups, proceed with as explained in the NAS
section.
@@ -71,10 +116,10 @@ homedir backups in the external archive/backup volume.
You might want to backup the whole SSD, M-SATA or microSD from your appliances.
If so, proceed as follows with the appliance device connected in your TPC:
- export appliance=appliance-name
- export dest="/var/backups/remote/$appliance.`facter domain`/image/`date +%Y%m%d`"
- sudo mkdir -p $dest
- dcfldd if=/dev/sdb | bzip2 > $dest/$appliance.img.bz2
+ export APPLIANCE="appliance-name"
+ export DEST="/var/backups/remote/${APPLIANCE}.${DOMAIN}/image/`date +%Y%m%d`"
+ $SUDO mkdir -p $dest
+ dcfldd if=/dev/sdb | bzip2 > $DEST/${APPLIANCE}.img.bz2
## Smartphone
@@ -100,10 +145,12 @@ A Backup Kit is a box with the following items:
* External encrypted archive/backup disk.
* Case for SSD transportation.
* Laptop power adapter and cables.
-* Dockstation SATA/USB/M-2/microSD/etc (with power adapter).
+* Dockstation SATA/USB/M.2/microSD/etc (with power adapter -- usually 12V).
+* Power adapter for external hard drives (usually 12V, and might be compatible
+ with the dockstationadapter, so you might carry just one).
* USB power adapter and cable (including USB 2, USB 3 and USB C).
* USB cables (USB 2, USB 3 and USB C) for the Dockstation and the external drive.
-* TPC laptop with battery and TPS (SSD, M-2 etc), optionally with a UltraBase/Dockstation.
+* TPC laptop with battery and TPS (SSD, M.2 etc), optionally with a UltraBase/Dockstation.
* Philips screwdriver and other tools.
* FCR-MG2 adaptor for microSD to USB.
* Anything else your need (like eyeglasses).
@@ -113,7 +160,13 @@ This may be the ultimate disaster recovery kit for your Hydra!
## Restore
-Examples according to the software used to make the backup.
+Having backup data leaked is a serious security issue, and that's why we
+encrypt backups.
+But losing access to the encrypted material is data loss, so it's important
+to make sure in advance that we can get back the material.
+
+Procedures vary, and the following examples are sorted according to the
+software used to make the backup.
### Duplicity
@@ -128,19 +181,50 @@ For [duplicity][]:
For [Borg][]:
+ export PATH_TO_RESTORE="path/to/be/restored"
mkdir ~/temp/misc/restore
cd ~/temp/misc/restore
borg list ssh://$USER@$SERVER:$PORT//var/backups/users/$USER/borg
- borg extract ssh://$USER@$SERVER:$PORT//var/backups/users/$USER/borg::$USER-2018-06-11T17:07:39 mnt/crypt/home/$USER/$FILE_OR_FOLDER
+ borg extract ssh://$USER@$SERVER:$PORT//var/backups/users/$USER/borg::$USER-2018-06-11T17:07:39 mnt/crypt/home/$USER/$PATH_TO_RESTORE
Make sure to cleanup `~/temp/misc/restore` after recovering what you need.
+Note on backup keys:
+
+* In the past (before 2024), the Hydra Suite and it's companion [Puppet][]
+ modules used pre-generated [Borg][] repository keys for the sake of automation.
+ This is [not possible anymore][].
+* As it's [important to keep copies of the borg repository key safely
+ elsewhere][], the managed configuration supports OpenPGP-encrypting the
+ repository key and uploading it to the remote repository.
+* This OpenPGP-encrypted key file is named as `keyfile.asc` and is uploaded
+ in the root folder of the remote repository.
+* This OpenPGP-encrypted key file is encrypted and signed with a provided
+ OpenPGP keypair and passphrase (convention is to use the machines's OpenPGP
+ general purpose key, or the machine's role key).
+* This allows the operators to fetch this encrypted keyfile and use their copy
+ of the machine's OpenPGP key to extract the passphrase _on their
+ encrypted-storage workstations_ (recommendation is to not do this on the remote
+ repository).
+
+Just to be sure, let's emphasize Borg's own recommendation:
+
+> IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!
+>
+> If you used a repokey mode, the key is stored in the repo, but you should
+> back it up separately.
+> Use "borg key export" to export the key, optionally in printable format.
+> Write down the passphrase. Store both at safe place(s).
+
[Borg]: https://www.borgbackup.org/
+[Puppet]: https://www.puppet.com/
+[not possible anymore]: https://github.com/borgbackup/borg/issues/7047
+[important to keep copies of the borg repository key safely elsewhere]: https://borgbackup.readthedocs.io/en/latest/faq.html#how-important-is-the-home-config-borg-directory
### eCryptfs
For [eCryptfs][]:
- sudo ecryptfs-recover-private /media/$MEDIA/home/.ecryptfs/$USER/.Private
+ $SUDO ecryptfs-recover-private /media/$DISK/home/.ecryptfs/$USER/.Private
[eCryptfs]: https://www.ecryptfs.org/
diff --git a/docs/changelog.md b/docs/changelog.md
new file mode 100644
index 0000000..155459b
--- /dev/null
+++ b/docs/changelog.md
@@ -0,0 +1,21 @@
+# ChangeLog
+
+## 0.3.0 - Unrelased
+
+### hydra
+
+* [x] Deploy:
+ * [x] Automatically fix permission of (or delete) puppet's `devices`
+ folder: https://github.com/puppetlabs/puppet-specifications/blob/master/file_paths.md
+* [x] Keys:
+ * [x] Deprecate generating and deploying/import borg keys, since
+ pre-generation is not a supported behavior right now:
+ https://github.com/borgbackup/borg/issues/7047
+ * [x] Document about how keys are encrypted and backed up in the server:
+ https://borgbackup.readthedocs.io/en/latest/faq.html#how-important-is-the-home-config-borg-directory
+
+### hydractl
+
+* [x] Provision: increase default partition sizes from 20G to 40G, as nowadays
+ distro size increase a lot and a 20G system partition can be filled in
+ easily.
diff --git a/docs/ideas.md b/docs/ideas.md
index 7c96301..dfd549b 100644
--- a/docs/ideas.md
+++ b/docs/ideas.md
@@ -1,5 +1,7 @@
# Ideas
+## Misc
+
* `hydractl`:
* `mount-kvmx-supervised`: open encrypted volumes from supervised [kvmx][] guests.
* `hydra`:
diff --git a/docs/index.md b/docs/index.md
index 58aead8..66915c2 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,5 +1,7 @@
# Hydra Suite - Command and Control
+![Hydra](assets/logo.png){ align=left }
+
Hydra is a suite for orchestration and management of machines composed of
**lots of conventions and assumptions**, and the following basic commands:
@@ -10,7 +12,7 @@ Hydra is a suite for orchestration and management of machines composed of
An Hydra is not a "cloud computing" platform, it's something else: a set of
systems that operate together, where any node can be used to spawn new nodes.
-This is an ongoing experiment is how a person or a collective can manage
+This is an ongoing experiment in how a person or a collective can manage
many computers in an unified way. It's not production ready, and it always
struggles to pass the test of time.
diff --git a/docs/todo.md b/docs/todo.md
new file mode 100644
index 0000000..e13ae1f
--- /dev/null
+++ b/docs/todo.md
@@ -0,0 +1,55 @@
+# TODO
+
+## hydra
+
+* [ ] Deploy:
+ * [ ] Collect basic hardware information along with facts: cpuid,
+ dmidecode, hwinfo, lsblk, lscpu, lshw, lspci, lsusb, sfdisk etc.
+ Store somewhere under the `config/hardware` folder?
+ Could also be a separate action, `hydra <hydra> collect <node>`.
+ * [ ] Command line is broken for ansible when multiple nodes are provided.
+ * [ ] Use console-based GnuPG agent when calling `keyringer`.
+* [ ] Mass:
+ * [ ] Support for [cumin][] ([Debian package](https://tracker.debian.org/pkg/cumin)).
+
+[cumin]: https://github.com/wikimedia/cumin
+
+## hydractl
+
+* [ ] External volume script (replacing new drive procedure from
+ `docs/backups.md`).
+* [ ] Mount/umount media:
+ * [ ] Fix STANDBY handling on external drives:
+ > Disabling STANDBY on drive...
+ > couldn't find field acronym: STANDBY
+ > [perhaps a '--transport=<tn>' or '--vendor=<vn>' option is needed]
+ * [ ] Mount/umount system volume supporting split partiton scheme (`root`,
+ `var`, `home` etc).
+ * [ ] Try to detect the device partition (`/dev/sdb1` etc) based
+ on the LUKS2 label.
+ * [ ] Cryptdisks script needs fixing as `systemd-tty-ask-password-agent` is not
+ catching cryptsetup password prompts anymore. Otherwise, consider to
+ deprecate this script.
+* [ ] Syncing:
+ * [ ] Sync lock.
+ * [ ] Run `git submodule --update --init --recursive` after running
+ `git annex sync`.
+ * [ ] Shall `sync-media` do a `git unannex` on sidecar and other metadata
+ files and manage them through Git instead, or keep them as unlocked
+ files?
+ * [ ] Syncing packages: a frontend to `apt-offline` that uses `git-annex`
+ repositories: getting, installing, cleaning. One node can request
+ packages through an external drive, and another can fetch then.
+ * [ ] Integration with [Baobáxia](https://baobaxia.mocambos.net)?
+ Maybe that will already happen if `git-annex` can run directly
+ on a mucua's repository.
+ * [ ] Merge `sync-home` into `sync-tpc`, or make `sync-tpc` call
+ `sync-home`, to avoid code duplication.
+* [ ] Provision:
+ * [ ] Fix booting issues detailed [here](tpc.md#booting).
+* [ ] Upgrade:
+ * [ ] Support for [fwupd](https://fwupd.org), at least for showing
+ available updates.
+ * [ ] Raspbian does not have the `non-free-firmware` component?
+ Maybe this is not an issue, as we may move away from
+ Raspbian/Raspberry Pi OS.
diff --git a/docs/tpc.md b/docs/tpc.md
index 54f707f..c2a6954 100644
--- a/docs/tpc.md
+++ b/docs/tpc.md
@@ -6,7 +6,10 @@ The Hydra Suite was made with the [Trusted Personal Computer (TPC)][]
This guide documents the lifecycle of a TPC storage device, or a "TPS
Cartridge":
- Cartridge = TPS = Trusted Personal Storage (like and SSD, microSD or M-2)
+ Cartridge = TPS = Trusted Personal Storage
+
+A TPS can usually be whatever removable block device like an SSD, microSD, M.2
+etc.
The idea is that a number of TPCs can have interchangeable cartridges, as long
as each cartridge has an unique name.
@@ -19,12 +22,12 @@ as each cartridge has an unique name.
Start by running the following commands in your terminal, adjusting to your
case:
- export HYDRA="nome-da-hydra"
- export VOLNAME="nome-do-disco"
- export DOMAIN="dominio.example.org"
+ export HYDRA="hydra-name"
+ export DISK="disk-name"
export DEVICE="/dev/sdb"
- export USER=`whoami`
- export DEST=/media/$VOLNAME
+ export DOMAIN="`facter domain`"
+ export USER="`whoami`"
+ export DEST="/media/$DISK"
export CONFIG_FOLDER="`hydra $HYDRA folder`"
if [ "$USER" != "root" ]; then
@@ -46,35 +49,52 @@ procedures from this document can be done from another TPC.
If non-existing, create entries and keys:
- hydra $HYDRA newnode $VOLNAME tpc
+ hydra $HYDRA newnode $DISK tpc
Customize:
- vim $CONFIG_FOLDER/config/provision/$VOLNAME.conf
- vim $CONFIG_FOLDER/puppet/config/node/$VOLNAME.$DOMAIN.yaml
- vim $CONFIG_FOLDER/puppet/config/secrets/node/$VOLNAME.$DOMAIN.yaml
+ vim $CONFIG_FOLDER/config/provision/$DISK.conf
+ vim $CONFIG_FOLDER/puppet/config/node/$DISK.$DOMAIN.yaml
+ vim $CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN.yaml
### Secrets
Proceed adding other passwords and secrets at
-`$CONFIG_FOLDER/puppet/config/secrets/node/$VOLNAME.$DOMAIN`, make sure to
+`$CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN`, make sure to
encode them with [hiera-yaml][]:
- hydra $HYDRA eyaml $VOLNAME encrypt -q -o block -p -l some::password
+ hydra $HYDRA eyaml $DISK encrypt -q -o block -p -l some::password
If you already have the secret somewhere, you can use a construction like this
which already adds the eyaml block into the hiera config file:
keyringer $HYDA decrypt /path/to/some/secret | \
- hydra $HYDRA eyaml $VOLNAME encrypt -q -o block --sdtin -l some::password >> \
- $CONFIG_FOLDER/puppet/config/secrets/node/$VOLNAME.$DOMAIN.yaml
+ tr -d '\n' | \
+ hydra $HYDRA eyaml $DISK encrypt -q -o block --stdin -l some::password >> \
+ $CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN.yaml
You can also ensure a fresh random passphrase is used, using your favourite
generator like this:
head -c ${1:-20} /dev/urandom | base64 | \
- hydra $HYDRA eyaml $VOLNAME encrypt -q -o block --sdtin -l some::random:password >> \
- $CONFIG_FOLDER/puppet/config/secrets/node/$VOLNAME.$DOMAIN.yaml
+ tr -d '\n' | \
+ hydra $HYDRA eyaml $DISK encrypt -q -o block --stdin -l some::random::password >> \
+ $CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN.yaml
+
+For `passwd(5)` and `shadow(5)` hashed passphrases, use something like this:
+
+ mkpasswd -m sha-512 | \
+ tr -d '\n' | \
+ hydra $HYDRA eyaml $DISK encrypt -q -o block --stdin -l some::random::password >> \
+ $CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN.yaml
+
+Or this, for a random passphrase:
+
+ head -c ${1:-20} /dev/urandom | base64 | \
+ mkpasswd -m sha-512 --stdin | \
+ tr -d '\n' | \
+ hydra $HYDRA eyaml $DISK encrypt -q -o block --stdin -l some::random::password >> \
+ $CONFIG_FOLDER/puppet/config/secrets/node/$DISK.$DOMAIN.yaml
[hiera-yaml]: https://github.com/voxpupuli/hiera-eyaml
@@ -83,7 +103,7 @@ generator like this:
The proceed with the system installation, which should create all volumes in
the TPS and setup the base Operating System:
- hydra $HYDRA provision $VOLNAME
+ hydra $HYDRA provision $DISK
### Metadata
@@ -95,23 +115,23 @@ can be useful during recovery procedures.
1. Write down UUIDS, models and serials from the new disk. Examples:
- lsblk $DEVICE -n -o +UUID | keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/lsblk
- udevadm info --query=all --name=$DEVICE | keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/udevadm
+ lsblk $DEVICE -n -o +UUID | keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/lsblk
+ udevadm info --query=all --name=$DEVICE | keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/udevadm
# For non-USB and non-microSD disks only
- $SUDO hdparm -I $DEVICE | keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/hdparm
- $SUDO smartctl -i $DEVICE | keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/smartctl
+ $SUDO hdparm -I $DEVICE | keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/hdparm
+ $SUDO smartctl -i $DEVICE | keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/smartctl
2. Save partition tables, the LUKS header and other relevant information:
- $SUDO LC_ALL=C sfdisk -d $DEVICE | keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/sfdisk
+ $SUDO LC_ALL=C sfdisk -d $DEVICE | keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/sfdisk
- $SUDO cryptsetup luksDump /dev/mapper/$VOLNAME-root | keyringer $HYDRA encrypt nodes/houdini.fluxo.info/luksDump-root
- $SUDO cryptsetup luksDump /dev/mapper/$VOLNAME-swap | keyringer $HYDRA encrypt nodes/houdini.fluxo.info/luksDump-swap
+ $SUDO cryptsetup luksDump /dev/mapper/$DISK-root | keyringer $HYDRA encrypt nodes/houdini.fluxo.info/luksDump-root
+ $SUDO cryptsetup luksDump /dev/mapper/$DISK-swap | keyringer $HYDRA encrypt nodes/houdini.fluxo.info/luksDump-swap
- $SUDO cryptsetup luksHeaderBackup /dev/mapper/$VOLNAME-root --header-backup-file luksHeaderBackup-root
+ $SUDO cryptsetup luksHeaderBackup /dev/mapper/$DISK-root --header-backup-file luksHeaderBackup-root
$SUDO chown $(whoami) luksHeaderBackup-root
- keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/luksHeaderBackup-root luksHeaderBackup-root
+ keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/luksHeaderBackup-root luksHeaderBackup-root
chmod +w luksHeaderBackup-root
wipe luksHeaderBackup-root
@@ -120,15 +140,15 @@ can be useful during recovery procedures.
* Optionally, configure ATA passphrases for the new disk. If so, make sure to
save the passphrase in the keyring:
- keyringer $HYDRA encrypt nodes/$VOLNAME.$DOMAIN/ata
+ keyringer $HYDRA encrypt nodes/$DISK.$DOMAIN/ata
## Mounting
Use the following whenever you need to mount the cartridge in another system:
- hydractl mount-media $VOLNAME
+ hydractl mount-media $DISK
-This will make the volume available at `/media/$VOLNAME`.
+This will make the volume available at `/media/$DISK`.
## Initial user
@@ -144,7 +164,7 @@ home folder encrypted, backup your data and then proceed running
### Syncing the user and encrypted home in the new cartridge
-Then, with the new TPS cartridge mounted (`hydractl mount-media $VOLNAME`), do
+Then, with the new TPS cartridge mounted (`hydractl mount-media $DISK`), do
the following to replicate your user and homedir there:
$SUDO chroot $DEST adduser $USER
@@ -166,7 +186,7 @@ and is logged in there as `root`.
From another system, and with the cartridge mounted:
- hydractl sync-tpc $VOLNAME
+ hydractl sync-tpc $DISK
This might involve a lot of data being transmitted. If you want `sync-tpc` to
ignore all your QEMU virtual machine images hosted at `/var/cache/quemu`, setup
@@ -184,7 +204,34 @@ Now that basic provisioning is completed, you can unmount the cartridge.
From another system, and with the cartridge mounted:
- hydractl umount-media $VOLNAME
+ hydractl umount-media $DISK
+
+## Booting
+
+The current provisioning procedure (as of 2024-02) may yield into
+a partially bootable system, and may need some additional manual
+steps right after installation:
+
+ hydractl mount-media $DISK
+ sudo chroot $DEST
+ update-initramfs -v -u
+ hydractl umount-media $DISK
+
+During boot, at the `(initramfs)` prompt, type this to successfully
+unlock the encrypted volumes:
+
+ cryptsetup luksOpen /dev/mapper/${DISK}-root root
+ cryptsetup luksOpen /dev/mapper/${DISK}-swap swap
+ ^Ctrl-D
+
+Then, in the running system, do:
+
+ sudo update-initramfs -v -u
+
+With these steps, the system's boot procedure was fixed.
+
+But there are fixes be done at `hydractl provision` to produce fully working
+installations.
## Deploy
@@ -218,7 +265,7 @@ collected data:
## Adding a new hardware in the TPC pool
-* Proceed with [homologatin a new unit](https://plano.autodefesa.org/specs/tpc.html).
+* Proceed with [homologating a new unit](https://plano.autodefesa.org/specs/tpc.html).
* Provision, deploy, sync-media, sync-vms etc.
* Add the devices' network MAC Addresses at `70-persistent-net.rules`, then make sure
to sync this in the other cartridges.
diff --git a/lib/hydra/deploy b/lib/hydra/deploy
index 6990339..9f1a0ba 100644
--- a/lib/hydra/deploy
+++ b/lib/hydra/deploy
@@ -5,8 +5,8 @@ function hydra_deploy_setup {
# Common parameters
# Exclude eventual keys and version control files
#DEPLOY_DEPENDENCIES="puppet-common ruby-sqlite3 ruby-activerecord ruby-activerecord-deprecated-finders augeas-tools"
- DEPLOY_DEPENDENCIES="rsync puppet hiera-eyaml python3"
- DEPLOY_RSYNC="rsync -CrltDvpq --no-owner --exclude=/ssl --exclude=keys --exclude=site_keys --exclude=config/secrets --delete --rsync-path"
+ DEPLOY_DEPENDENCIES="git rsync puppet hiera-eyaml python3"
+ DEPLOY_RSYNC="rsync -CrltDvpq --no-owner --exclude=/ssl --exclude=logs --exclude=keys --exclude=site_keys --exclude=config/secrets --delete --rsync-path"
RSYNC_PATH="rsync -q"
# Ensure keystore existence
diff --git a/lib/hydra/provision b/lib/hydra/provision
index 0b00682..54d9bcf 100644
--- a/lib/hydra/provision
+++ b/lib/hydra/provision
@@ -38,7 +38,7 @@ function hydra_provision_config_load {
# Networking.
function hydra_provision_networking {
echo "Doing a basic network config..."
- cat <<-EOF | $SUDO tee $WORK/etc/network/interfaces > /dev/null
+ cat <<-EOF | $SUDO tee $WORK/etc/network/interfaces/eth0 > /dev/null
auto eth0
allow-hotplug eth0
iface eth0 inet dhcp
diff --git a/mkdocs.yml b/mkdocs.yml
index 9929a04..3837e87 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -1,10 +1,80 @@
+# Site parameters
site_name: Hydra Suite
+# Theme configuration
+# See https://www.mkdocs.org/user-guide/choosing-your-theme/
+# https://squidfunk.github.io/mkdocs-material/
theme:
- name: readthedocs
+ # Theme name
+ name: material
+ # Logo
logo: assets/logo.png
+ # Do not use external fonts
+ # Useful for offline operation, GDPR compliance and privacy in general
+ # See https://squidfunk.github.io/mkdocs-material/setup/changing-the-fonts/#autoloading
+ #
+ # This option was superseded by the privacy plugin:
+ # https://squidfunk.github.io/mkdocs-material/setup/ensuring-data-privacy/#built-in-privacy-plugin
+ #font: false
+
+# Plugins
+# https://www.mkdocs.org/dev-guide/plugins/
+plugins:
+ # Note that this might not be available in the mkdocs-material package from
+ # Debian bookworm (as of 2024-08-23)
+ privacy: {}
+
+# Markdown extensions
+# See https://www.mkdocs.org/user-guide/configuration/#markdown_extensions
+# https://squidfunk.github.io/mkdocs-material/setup/extensions/
+markdown_extensions:
+ # Footnotes
+ # https://squidfunk.github.io/mkdocs-material/reference/footnotes/
+ footnotes: {}
+
+ # Tasklist handling
+ # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#tasklist
+ pymdownx.tasklist:
+ custom_checkbox: true
+
+ # SuperFences
+ # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#superfences
+ pymdownx.superfences:
+ custom_fences:
+ - name: mermaid
+ class: mermaid
+ format: !!python/name:pymdownx.superfences.fence_code_format
+
+ # Table of Contents configuration
+ # See https://www.mkdocs.org/user-guide/configuration/#markdown_extensions
+ # https://www.mkdocs.org/user-guide/writing-your-docs/#linking-to-pages
+ toc:
+ permalink: true
+
+ # Attribute Lists
+ # Needed by proper image handling
+ # https://squidfunk.github.io/mkdocs-material/reference/images/
+ # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#attribute-lists
+ attr_list: {}
+
+ # Markdown in HTML
+ # Needed by proper image handling
+ # https://squidfunk.github.io/mkdocs-material/reference/images/
+ # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#markdown-in-html
+ md_in_html: {}
+
+# Copyright notice
+copyright: Copyleft &copy; 2024 Fluxo Group. See LICENSE for details.
+
+# Extra template parameters
+# https://www.mkdocs.org/user-guide/configuration/#extra
+extra:
+ generator: false
+
+# Navigation
+# https://www.mkdocs.org/user-guide/writing-your-docs/#configure-pages-and-navigation
nav:
- index.md
- install.md
@@ -12,3 +82,5 @@ nav:
- tpc.md
- backups.md
- ideas.md
+ - todo.md
+ - changelog.md
diff --git a/share/hydra/deploy b/share/hydra/deploy
index b675577..5fabb6c 100755
--- a/share/hydra/deploy
+++ b/share/hydra/deploy
@@ -151,6 +151,9 @@ for node in $NODES; do
# Fix ssl folder ownership
$SUDO chown -R `whoami`: $HYDRA_FOLDER/puppet/ssl
+ # Fix devices folder ownership
+ $SUDO chown -R `whoami`: $HYDRA_FOLDER/puppet/devices
+
# Import keys if needed
if ! $SUDO test -f /root/.ssh/id_rsa || \
! $SUDO test -f /root/.config/borg/hydra/key || \
@@ -300,7 +303,8 @@ if [ -e "$HYDRA_FOLDER/ansible/ansible.cfg" ]; then
if [ "$NODES" == "all" ]; then
hydra $HYDRA ansible-playbook site.yml $ANSIBLE_ARGS
else
- hydra $HYDRA ansible-playbook site.yml $ANSIBLE_ARGS --limit $NODES
+ NODES="`echo $NODES | tr ' ' ','`"
+ hydra $HYDRA ansible-playbook site.yml $ANSIBLE_ARGS --limit "$NODES"
fi
fi
fi
diff --git a/share/hydra/import-keys b/share/hydra/import-keys
index 7339a0d..1811930 100755
--- a/share/hydra/import-keys
+++ b/share/hydra/import-keys
@@ -123,6 +123,14 @@ EOF
}
# Import Borg key
+#
+# Borg does not support using pre-generated keys anymore (as of 2024-05-16).
+#
+# This code is therefore deprecated, but will stay here for a while, as maybe
+# in the long term borg starts to support this again.
+#
+# Check also https://github.com/borgbackup/borg/issues/7047
+# https://borgbackup.readthedocs.io/en/latest/faq.html#how-important-is-the-home-config-borg-directory
function hydra_import_keys_borg {
if [ "`facter fqdn`" != "$hostname" ]; then
echo "-----------------------------------------------------"
diff --git a/share/hydra/newkeys b/share/hydra/newkeys
index deb9b6e..c53dfe7 100755
--- a/share/hydra/newkeys
+++ b/share/hydra/newkeys
@@ -69,6 +69,13 @@ function hydra_newkeys {
done
}
+# Borg does not support using pre-generated keys anymore (as of 2024-05-16).
+#
+# This code is therefore deprecated, but will stay here for a while, as maybe
+# in the long term borg starts to support this again.
+#
+# Check also https://github.com/borgbackup/borg/issues/7047
+# https://borgbackup.readthedocs.io/en/latest/faq.html#how-important-is-the-home-config-borg-directory
function hydra_newkeys_borg {
# Check for borg
#if ! which borg &> /dev/null; then
diff --git a/share/hydractl/dist-upgrade b/share/hydractl/dist-upgrade
index c59a1ae..b5720e2 100755
--- a/share/hydractl/dist-upgrade
+++ b/share/hydractl/dist-upgrade
@@ -105,10 +105,12 @@ function hydra_system_upgrade_prepare {
echo "Updating apt configuration..."
sed -i -e "s/$release/$nextrelease/g" /etc/apt/sources.list
+ # Update APT Preferences with the next debian release
if [ -e "/etc/apt/preferences" ]; then
sed -i -e "s/$release/$nextrelease/g" /etc/apt/preferences
fi
+ # Disable backports during the upgrade
sed -i -e 's|^deb http://backports.debian.org/debian-backports|#deb http://backports.debian.org/debian-backports|' /etc/apt/sources.list
# These will be generated by puppet and can be safely removed
@@ -121,6 +123,11 @@ function hydra_system_upgrade_prepare {
apt-get remove colord -y
fi
+ # Handles component migration
+ if [ "$nextrelease" == "bookworm" ]; then
+ sed -i -e 's|main contrib non-free$|main contrib non-free non-free-firmware|' /etc/apt/sources.list
+ fi
+
# Set next state
hydra_system_upgrade_stage download
}
diff --git a/share/hydractl/provision b/share/hydractl/provision
index d318aaf..7d224e3 100755
--- a/share/hydractl/provision
+++ b/share/hydractl/provision
@@ -118,16 +118,16 @@ function hydra_provision_config {
hydra_user_config interactive y "Interactive mode? (y/n)"
hydra_user_config device /dev/sdb "Physical device(s) (more than one auto sets RAID mode)"
hydra_user_config swap_size 2000 "Swap size (in MB, 0 to not create it)"
- hydra_user_config root_size 20G "Size of root partition (-1 for all free space)"
+ hydra_user_config root_size 40G "Size of root partition (-1 for all free space)"
if [ "$root_size" != "-1" ]; then
- hydra_user_config home_size 20G "Size of home partition (0 to not create it, -1 for all free space)"
+ hydra_user_config home_size 40G "Size of home partition (0 to not create it, -1 for all free space)"
else
home_size="0"
fi
if [ "$root_size" != "-1" ] && [ "$home_size" != "-1" ]; then
- hydra_user_config var_size 20G "Size of var partition (0 to not create it, -1 for all free space)"
+ hydra_user_config var_size 40G "Size of var partition (0 to not create it, -1 for all free space)"
else
var_size="0"
fi
@@ -633,7 +633,7 @@ $APT_INSTALL linux-image-$kernel_arch -y
echo "Creating initramfs..."
if [ "$initramfs" == "initramfs-tools" ]; then
- $APT_INSTALL initramfs-tools -y
+ $APT_INSTALL initramfs-tools cryptsetup-initramfs -y
cat <<-EOF | $SUDO tee $WORK/etc/initramfs-tools/modules > /dev/null
dm-mod
diff --git a/share/hydractl/provision-raspi b/share/hydractl/provision-raspi
index 8444186..d437ce0 100755
--- a/share/hydractl/provision-raspi
+++ b/share/hydractl/provision-raspi
@@ -26,7 +26,7 @@ function hydra_provision_config {
hydra_user_config device /dev/mmcblk0 "Destination device"
hydra_user_config hostname machine "Hostname"
hydra_user_config domain example.org "Domain"
- hydra_user_config arch armel "System arch"
+ hydra_user_config arch armhf "System arch"
hydra_user_config version stretch "Distro version"
hydra_user_config mirror https://deb.debian.org/debian/ "Debian mirror"
diff --git a/share/hydractl/remove-dep b/share/hydractl/remove-dep
index 5e1a732..709cda2 100755
--- a/share/hydractl/remove-dep
+++ b/share/hydractl/remove-dep
@@ -38,3 +38,5 @@ fi
$sudo aptitude markauto $(apt-cache showsrc "$1" \
| grep Build-Depends \
| perl -p -e 's/(?:[\[(].+?[\])]|Build-Depends:|,|\|)//g')
+
+$sudo apt autoremove --purge
diff --git a/share/hydractl/sync-backups b/share/hydractl/sync-backups
index 844d630..63e11b4 100755
--- a/share/hydractl/sync-backups
+++ b/share/hydractl/sync-backups
@@ -7,12 +7,21 @@
BASENAME="`basename $0`"
VOLUME="$1"
MEDIA="/media/$VOLUME"
-BWLIMIT=${BWLIMIT:=32000}
+BWLIMIT=${BWLIMIT:=0}
IMAGES="/var/data/crypt/"
-RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete --bwlimit=$BWLIMIT"
CP="ionice -c 3 nice -n 19 cp"
WHOAMI="`whoami`"
+# Set bandwidth limit
+if [ "$BWLIMIT" != "0" ]; then
+ BWLIMIT="--bwlimit=$BWLIMIT"
+else
+ BWLIMIT=""
+fi
+
+# Build rsync command line
+RSYNC="ionice -c 3 nice -n 19 rsync -avH --delete $BWLIMIT"
+
# Sync backups for a node.
function sync_backups_node {
if [ ! -z "$NODE" ]; then
@@ -75,13 +84,13 @@ fi
# Check volume name
if [ "$VOLUME" == "`hostname -f`" ]; then
- echo "volume is the hostname, cannot sync to myself"
+ echo "$BASENAME: volume is the hostname, cannot sync to myself"
exit 1
fi
# Check if it is mounted
if ! mount | grep -q $MEDIA; then
- echo "volume $MEDIA is not mounted"
+ echo "$BASENAME: volume $MEDIA is not mounted"
exit 1
fi
diff --git a/share/hydractl/sync-home b/share/hydractl/sync-home
index a0460e9..b2cd85f 100755
--- a/share/hydractl/sync-home
+++ b/share/hydractl/sync-home
@@ -4,7 +4,7 @@
#
# Parameters
-BASENAME="$0"
+BASENAME="`basename $0`"
TPC="$1"
DEST="/media/$1"
MEDIA="$DEST/var/cache/$TPC/media"
diff --git a/share/hydractl/sync-media b/share/hydractl/sync-media
index 88c8901..0976c68 100755
--- a/share/hydractl/sync-media
+++ b/share/hydractl/sync-media
@@ -67,8 +67,8 @@ function sync_media_add_metadata {
find -name metadata.pdf.lua.old -exec git add {} \;
# Darktable sidecar files
- find -name '*.xmp' -type l ! -path '*.git' -exec git annex unlock {} \;
- find -name '*.xmp' ! -path '*.git' -exec git add {} \;
+ find -name '*.xmp' -type l -not -path '*.git*' -exec git annex unlock {} \;
+ find -name '*.xmp' -not -path '*.git*' -exec git add {} \;
}
# If there is a playlists folder, make sure mpd user can write to it
@@ -136,7 +136,7 @@ function sync_media_ensure_remote {
local remote="$1"
local path="$2"
- if [ -z "$remote" ] || [ "$remote" == "local" ] || [ -z "$path" ]; then
+ if [ -z "$remote" ] || [ "$remote" == "local" ] || [ "$remote" == "localhost" ] || [ -z "$path" ]; then
return
fi
@@ -171,12 +171,16 @@ if echo $OPTIONS | grep -q -- "--dropunused"; then
fi
# Set drive config
-# Ingore drive/volume if it's set to "local"
-if [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ]; then
+# Ignore drive/volume if it's set to "local"
+if [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ] && [ "$REMOTE" != "localhost" ]; then
# Check storage media
MOUNT="`mount | grep $VOLUME`"
+
if [ ! -z "$MOUNT" ]; then
DRIVE="$(basename `echo $MOUNT | awk '{ print $1 }'`)"
+ #else
+ # echo "$BASENAME: volume $MEDIA is not mounted"
+ # exit 1
fi
fi
@@ -219,9 +223,12 @@ for folder in $REPOSITORIES; do
# Ensure the repository is identified
sync_media_identity
+ # Disable automatic syncing of content
+ git config annex.synccontent false
+
# Sync before changing anything, to make sure the history does not have
# conflicts with other remotes
- git annex sync
+ git annex sync --no-content
# Fix any playlist permissions
sync_media_playlist_perms
@@ -233,7 +240,7 @@ for folder in $REPOSITORIES; do
sync_media_add
# Sync everything again
- git annex sync
+ git annex sync --no-content
# Get all that needs to be got
sync_media_getall $CACHE/$folder
@@ -241,6 +248,8 @@ for folder in $REPOSITORIES; do
# Repository maintenance
sync_media_fsck
sync_media_dropunused
+ git prune
+ git gc
)
fi
@@ -299,11 +308,17 @@ if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
sync_media_ensure_remote $HOST $CACHE/$folder
sync_media_identity
sync_media_add
- git annex sync
+
+ # Disable automatic syncing of content
+ git config annex.synccontent false
+
+ git annex sync --no-content
sync_media_get $VOLUME/$MEDIA/$folder
sync_media_getall $VOLUME/$MEDIA/$folder
sync_media_fsck
sync_media_dropunused
+ git gc
+ git prune
#git annex drop --auto --numcopies=2
)
elif [ -d "$CACHE/$folder" ] && [ ! -d "$CACHE/$folder/.git" ]; then
@@ -375,7 +390,7 @@ if [ ! -z "$DRIVE" ] && [ -d "$VOLUME/$MEDIA" ]; then
fi
fi
done
-elif [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ]; then
+elif [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ] && [ "$REMOTE" != "localhost" ]; then
# Try to copy to a remote
for folder in `ls $CACHE`; do
if [ -d "$CACHE/$folder/.git/annex" ]; then
@@ -388,7 +403,7 @@ elif [ ! -z "$REMOTE" ] && [ "$REMOTE" != "local" ]; then
(
cd $CACHE/$folder
git annex copy . --to $REMOTE
- git annex sync
+ git annex sync --no-content
)
else
echo "Syncing $CACHE/$folder with ssh://$REMOTE.$DOMAIN/$CACHE/$folder..."
diff --git a/share/hydractl/sync-tpc b/share/hydractl/sync-tpc
index c739c5a..0a72866 100755
--- a/share/hydractl/sync-tpc
+++ b/share/hydractl/sync-tpc
@@ -4,7 +4,7 @@
#
# Parameters
-BASENAME="$0"
+BASENAME="`basename $0`"
TPC="$1"
DEST="/media/$1"
MEDIA="$DEST/var/cache/$TPC/media"