From cfe68b9c2da2ff48171e2bbdb8988b92f7f41ee1 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Sat, 19 May 2018 08:17:47 -0300 Subject: Adds borg action and misc changes --- manifests/borg.pp | 31 ++++++++++++ manifests/duplicity.pp | 2 +- manifests/init.pp | 1 + manifests/params.pp | 2 +- manifests/utils.pp | 5 +- templates/borg.sh.erb | 115 +++++++++++++++++++++++++++++++++++++++++++ templates/duplicity.conf.erb | 58 ---------------------- templates/duplicity.sh.erb | 58 ++++++++++++++++++++++ 8 files changed, 210 insertions(+), 62 deletions(-) create mode 100644 manifests/borg.pp create mode 100644 templates/borg.sh.erb delete mode 100644 templates/duplicity.conf.erb create mode 100644 templates/duplicity.sh.erb diff --git a/manifests/borg.pp b/manifests/borg.pp new file mode 100644 index 0000000..4d08908 --- /dev/null +++ b/manifests/borg.pp @@ -0,0 +1,31 @@ +define backup::borg( + $port = '22', + $ensure = present, + $keepdaily = '7', + $keepweekly = '4', + $keepmonthly = '3', + $user = $::hostname, + $host = "${title}.${::domain}", + $encryption = 'repokey', + $order = 95, + $periodic_check = absent, + $password, +) { + file { "${backupninja::configdir}/${order}_borg-${host}.sh": + ensure => $ensure, + content => template('backup/borg.sh.erb'), + owner => root, + group => root, + mode => '0600', + require => File["${backupninja::configdir}"], + } + + cron { "borg_check-$title.$domain": + command => "/bin/bash ${backupninja::configdir}/${order}_borg-${host}.sh --check", + user => root, + hour => "0", + minute => "0", + weekday => "0", + ensure => $periodic_check, + } +} diff --git a/manifests/duplicity.pp b/manifests/duplicity.pp index e84e050..c7801a3 100644 --- a/manifests/duplicity.pp +++ b/manifests/duplicity.pp @@ -32,7 +32,7 @@ define backup::duplicity( # the backupninja rule for this duplicity backup file { "${backupninja::configdir}/${order}_duplicity-${title}.sh": ensure => $ensure, - content => template('backup/duplicity.conf.erb'), + content => template('backup/duplicity.sh.erb'), owner => root, group => root, mode => '0600', diff --git a/manifests/init.pp b/manifests/init.pp index 421800f..a4492ca 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -46,6 +46,7 @@ class backup( } package { "duplicity": ensure => installed, } + package { "borgbackup": ensure => installed, } package { "debconf-utils": ensure => installed, } # See http://www.rfc3092.net/2013/09/missing-modules-for-paramiko-and-gio-in-duplicity-foo/ diff --git a/manifests/params.pp b/manifests/params.pp index caeff04..4efed5b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -6,7 +6,7 @@ class backup::params { # for data that's going to be encrypted and signed $include_unencrypted = [ "/etc", "/var", "/home", ] - $exclude_unencrypted = [ "$backupdir_remote", "$backupdir/duplicity", "$backupdir/restore", "/var/cache", "/var/log", "/var/vservers", "/var/chroot", "/root/.cache", "/var/lib/dpkg", "/var/lib/apt", "/var/lib/aptitude/", "/var/sites/backups", "/var/data/crypt", "/data/backups", "/data/cache" ] + $exclude_unencrypted = [ "$backupdir_remote", "$backupdir/duplicity", "$backupdir/borg", "$backupdir/restore", "/var/cache", "/var/log", "/var/vservers", "/var/chroot", "/root/.cache", "/var/lib/dpkg", "/var/lib/apt", "/var/lib/aptitude/", "/var/sites/backups", "/var/data/crypt", "/var/data/backups", "/var/data/cache" ] # for data that were previously encrypted and signed $include_encrypted = [ "$backupdir/duplicity", ] diff --git a/manifests/utils.pp b/manifests/utils.pp index b13c878..7cc358f 100644 --- a/manifests/utils.pp +++ b/manifests/utils.pp @@ -1,5 +1,6 @@ class backup::utils { - package { [ 'bup', 'attic', 'obnam' ]: - ensure => present, + # Not in use + package { [ 'bup', 'attic', 'obnam', 'restic' ]: + ensure => absent, } } diff --git a/templates/borg.sh.erb b/templates/borg.sh.erb new file mode 100644 index 0000000..3eddd87 --- /dev/null +++ b/templates/borg.sh.erb @@ -0,0 +1,115 @@ +#!/bin/sh +# Adapted from https://borgbackup.readthedocs.io/en/stable/quickstart.html#automating-backups + +export SSH_SERVER="<%= @user %>@<%= @host %>" +export SSH_PORT="<%= @port %>" + +export HOSTNAME=`cat /etc/hostname` + +# Setting this, so the repo does not need to be given on the commandline: +export BORG_REPO=ssh://$SSH_SERVER:$SSH_PORT//var/backups/remote/$HOSTNAME/borg + +# Setting this, so you won't be asked for your repository passphrase: +export BORG_PASSPHRASE='<%= @password %>' +# or this to ask an external program to supply the passphrase: +#export BORG_PASSCOMMAND='pass show backup' + +# some helpers and error handling: +#info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; } +trap 'info $( date ) Backup interrupted >&2; exit 2' INT TERM + +# Check +if [ "$1" == "--check" ]; then + borg list + exit $? +fi + +# Initialize +if ! ssh $SSH_SERVER -p $SSH_PORT test -f /var/backups/remote/$HOSTNAME/borg/config; then + info "Initializing borg repository at ssh://$SSH_SERVER:$SSH_PORT//var/backups/remote/$HOSTNAME/borg..." + borg init --encryption=<%= @encryption %> ssh://$SSH_SERVER:$SSH_PORT//var/backups/remote/$HOSTNAME/borg + + init_exit=$? + + if [ "$init_exit" != "0" ]; then + fatal "Error initializing repository" + fi +fi + +# Backup the most important directories into an archive named after +# the machine this script is currently running on: + +info "Starting backup..." + +borg create \ + --verbose \ + --filter AME \ + --list \ + --stats \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + --exclude '/home/*/.cache/*' \ + --exclude '/var/cache/*' \ + --exclude '/var/tmp/*' \ + --exclude '/var/backups/remote' \ + --exclude '/var/backups/duplicity' \ + --exclude '/var/backups/restore' \ + --exclude '/var/cache' \ + --exclude '/var/vservers' \ + --exclude '/var/chroot' \ + --exclude '/root/.cache' \ + --exclude '/var/lib/dpkg' \ + --exclude '/var/lib/apt' \ + --exclude '/var/lib/aptitude' \ + --exclude '/var/sites/backups' \ + --exclude '/var/data/crypt' \ + --exclude '/var/data/backups' \ + --exclude '/var/data/cache' \ + ::'{hostname}-{now}' \ + /etc \ + /home \ + /root \ + /var \ + + backup_exit=$? + +if [ "$backup_exit" != "0" ]; then + fatal "Error creating snapshot" +fi + +info "Pruning repository..." + +# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly +# archives of THIS machine. The '{hostname}-' prefix is very important to +# limit prune's operation to this machine's archives and not apply to +# other machines' archives also: + +borg prune \ + --list \ + --prefix '{hostname}-' \ + --show-rc \ + --keep-daily <%= @keepdaily %> \ + --keep-weekly <%= @keepweekly %> \ + --keep-monthly <%= @keepmonthly %> \ + + prune_exit=$? + +if [ "$prune_exit" != "0" ]; then + fatal "Error pruning repository" +fi + +# use highest exit code as global exit code +#global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit )) +# +#if [ ${global_exit} -eq 1 ]; +#then +# info "Backup and/or Prune finished with a warning" +#fi +# +#if [ ${global_exit} -gt 1 ]; +#then +# info "Backup and/or Prune finished with an error" +#fi +# +#exit ${global_exit} diff --git a/templates/duplicity.conf.erb b/templates/duplicity.conf.erb deleted file mode 100644 index 846f9de..0000000 --- a/templates/duplicity.conf.erb +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# This configuration file was auto-generated by the Puppet configuration -# management system. Any changes you make to this file will be overwritten -# the next time Puppet runs. Please make configuration changes to this -# service in Puppet. -# -# TODO: custom log file - -BACKUP_FOLDER="<%= @directory %>" -FULL_IF_OLDER_THAN="<%= @full_if_older_than %>" -REMOVE_OLDER_THAN="<%= @remove_older_than %>" -REMOVE_ALL_BUT_N_FULL="<%= @remove_all_but_n_full %>" -ENCRYPT_KEY="<%= @encryptkey %>" -SIGN_KEY="<%= @encryptkey %>" -<% @exclude_unencrypted.each do |del| -%> -EXCLUDE="$EXCLUDE --exclude <%= del %>" -<% end -%> -<% @include_unencrypted.each do |add| -%> -INCLUDE="$INCLUDE --include <%= add %>" -<% end -%> - -# some systems have a limited /tmp folder -TMP="/var/tmp" -mkdir -p $TMP - -export PASSPHRASE='<%= @password %>' -mkdir -p $BACKUP_FOLDER - -# adds support for checking the existing backup -if [ "$1" == "--check" ]; then - duplicity collection-status file:////$BACKUP_FOLDER -else - # cleanup any previous broken backups - duplicity cleanup file:////$BACKUP_FOLDER --force - - # remove dangling lockfiles - if [ -z "`pidof duplicity`" ]; then - rm /root/.cache/duplicity/*/lockfile.lock - fi - - # it's important to let $EXCLUDE come before $INCLUDE to - # have greater precedence; see duplicity(1) for more info - duplicity -v6 --full-if-older-than $FULL_IF_OLDER_THAN \ - --tempdir $TMP --encrypt-key $ENCRYPT_KEY --sign-key $SIGN_KEY \ - --exclude $TMP $EXCLUDE $INCLUDE --exclude '**' / file:////$BACKUP_FOLDER - - if [ "$?" != "0" ]; then - fatal "Duplicity backup failed." - fi - - duplicity remove-older-than $REMOVE_OLDER_THAN file:////$BACKUP_FOLDER --force || exit 1 - duplicity remove-all-but-n-full $REMOVE_ALL_BUT_N_FULL file:////$BACKUP_FOLDER --force || exit 1 - - if [ "$?" != "0" ]; then - fatal "Removal of old backups failed." - fi -fi diff --git a/templates/duplicity.sh.erb b/templates/duplicity.sh.erb new file mode 100644 index 0000000..846f9de --- /dev/null +++ b/templates/duplicity.sh.erb @@ -0,0 +1,58 @@ +#!/bin/bash +# +# This configuration file was auto-generated by the Puppet configuration +# management system. Any changes you make to this file will be overwritten +# the next time Puppet runs. Please make configuration changes to this +# service in Puppet. +# +# TODO: custom log file + +BACKUP_FOLDER="<%= @directory %>" +FULL_IF_OLDER_THAN="<%= @full_if_older_than %>" +REMOVE_OLDER_THAN="<%= @remove_older_than %>" +REMOVE_ALL_BUT_N_FULL="<%= @remove_all_but_n_full %>" +ENCRYPT_KEY="<%= @encryptkey %>" +SIGN_KEY="<%= @encryptkey %>" +<% @exclude_unencrypted.each do |del| -%> +EXCLUDE="$EXCLUDE --exclude <%= del %>" +<% end -%> +<% @include_unencrypted.each do |add| -%> +INCLUDE="$INCLUDE --include <%= add %>" +<% end -%> + +# some systems have a limited /tmp folder +TMP="/var/tmp" +mkdir -p $TMP + +export PASSPHRASE='<%= @password %>' +mkdir -p $BACKUP_FOLDER + +# adds support for checking the existing backup +if [ "$1" == "--check" ]; then + duplicity collection-status file:////$BACKUP_FOLDER +else + # cleanup any previous broken backups + duplicity cleanup file:////$BACKUP_FOLDER --force + + # remove dangling lockfiles + if [ -z "`pidof duplicity`" ]; then + rm /root/.cache/duplicity/*/lockfile.lock + fi + + # it's important to let $EXCLUDE come before $INCLUDE to + # have greater precedence; see duplicity(1) for more info + duplicity -v6 --full-if-older-than $FULL_IF_OLDER_THAN \ + --tempdir $TMP --encrypt-key $ENCRYPT_KEY --sign-key $SIGN_KEY \ + --exclude $TMP $EXCLUDE $INCLUDE --exclude '**' / file:////$BACKUP_FOLDER + + if [ "$?" != "0" ]; then + fatal "Duplicity backup failed." + fi + + duplicity remove-older-than $REMOVE_OLDER_THAN file:////$BACKUP_FOLDER --force || exit 1 + duplicity remove-all-but-n-full $REMOVE_ALL_BUT_N_FULL file:////$BACKUP_FOLDER --force || exit 1 + + if [ "$?" != "0" ]; then + fatal "Removal of old backups failed." + fi +fi -- cgit v1.2.3