From 7355535ee18efc472f5a598b8474e8d05765d6f1 Mon Sep 17 00:00:00 2001 From: Jacob Anawalt Date: Mon, 11 Oct 2010 18:53:00 -0600 Subject: Enable pg_dump format option. The format option of pg_dump enables tar and custom archive file formats in addition to the default plain-text file containing SQL commands. When either the tar or custom format are selected the behaviour of database=all is changed to no longer dump a single file via pg_dumpall. Instead pg_dumpall is called once to export the "global" data (roles & tablespaces) and then pg_dump is called once for each non-template table in the database. To support the GZIP and GZIP_OPTS variables in backupninja and to give the default --rsyncable gzip compression flag a chance at working on a PostgreSQL backup, the custom output is forced to not use compression. Instead compression is done via a pipe to gzip. Hopefully this benefits rsync and rdiff-backup style backups for reduced backup and storage costs that outweigh the restoration ones. --- handlers/pgsql.in | 100 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 11 deletions(-) (limited to 'handlers/pgsql.in') diff --git a/handlers/pgsql.in b/handlers/pgsql.in index 0b7badf..0564f5a 100644 --- a/handlers/pgsql.in +++ b/handlers/pgsql.in @@ -8,6 +8,8 @@ getconf backupdir /var/backups/postgres getconf databases all getconf compress yes getconf vsname +# format maps to pg_dump --format= option, old/default was plain +getconf format plain localhost=`hostname` @@ -35,17 +37,31 @@ fi # Make sure that the system to backup has the needed executables if [ $usevserver = yes ]; then debug "Examining vserver '$vsname'." - if [ "$databases" == "all" ]; then + if [ "$databases" == "all" ] && [ "$format" = "plain" ]; then [ -x "$vroot`$VSERVER $vsname exec which $PGSQLDUMPALL`" ] || \ fatal "Can't find $PGSQLDUMPALL in vserver $vsname." + elif [ "$format" != "plain" ]; then + [ -x "$vroot`$VSERVER $vsname exec which $PGSQLDUMPALL`" ] || \ + fatal "Can't find $PGSQLDUMPALL in vserver $vsname." + [ -x "$vroot`$VSERVER $vsname exec which $PGSQLDUMP`" ] || \ + fatal "Can't find $PGSQLDUMP in vserver $vsname." + [ -x "$vroot`$VSERVER $vsname exec which $PSQL`" ] || \ + fatal "Can't find $PSQL in vserver $vsname." else [ -x "$vroot`$VSERVER $vsname exec which $PGSQLDUMP`" ] || \ fatal "Can't find $PGSQLDUMP in vserver $vsname." fi else - if [ "$databases" == "all" ]; then + if [ "$databases" == "all" ] && [ "$format" = "plain" ]; then + [ -x "`which $PGSQLDUMPALL`" ] || \ + fatal "Can't find $PGSQLDUMPALL." + elif [ "$format" != "plain" ]; then [ -x "`which $PGSQLDUMPALL`" ] || \ fatal "Can't find $PGSQLDUMPALL." + [ -x "`which $PGSQLDUMP`" ] || \ + fatal "Can't find $PGSQLDUMP." + [ -x "`which $PSQL`" ] || \ + fatal "Can't find $PSQL." else [ -x "`which $PGSQLDUMP`" ] || \ fatal "Can't find $PGSQLDUMP." @@ -71,6 +87,30 @@ chown $pguid $vroot$backupdir debug "chmod 700 $vroot$backupdir" chmod 700 $vroot$backupdir + +# If we are using the custom (best) or tar pg_dump format, and +# dumping "all" databases, we will substitute "all" for a list +# of all non-template databases to avoid the use of pg_dumpall. +dumpglobals="no" +if [ "$databases" = "all" ] && [ "$format" != "plain" ]; then + cmdprefix="" + if [ "$usevserver" = "yes" ]; then + cmdprefix="$VSERVER $vsname exec " + fi + execstr="${cmdprefix} su - $PGSQLUSER -c 'psql -AtU $PGSQLUSER -c \"SELECT datname FROM pg_database WHERE NOT datistemplate\"'" + debug execstr + dblist="" + for db in $(eval $execstr 2>&1); do + dblist="$dblist $db" + done + if [ "$dblist" != "" ]; then + databases="$dblist" + fi + # Dump globals (pg_dumpall -g) for roles and tablespaces + dumpglobals="yes" +fi + + # if $databases = all, use pg_dumpall if [ "$databases" == "all" ]; then if [ $usevserver = yes ]; then @@ -101,20 +141,58 @@ if [ "$databases" == "all" ]; then # else use pg_dump on each specified database else - for db in $databases; do + # If we're not doing plain format, database=all may now be database=list + # so we track the database=all selection in dumpglobals which tells us + # to also dump the roles and tablespaces via pg_dumpall -g + if [ "$dumpglobals" = "yes" ]; then + globalscmd="" + if [ "$compress" == "yes" ]; then + globalscmd="set -o pipefail ; $PGSQLDUMPALL -g | $GZIP $GZIP_OPTS > '$backupdir/globals.sql.gz'" + else + globalscmd="$PGSQLDUMPALL -g > '$backupdir/globals.sql'" + fi if [ $usevserver = yes ]; then - if [ "$compress" == "yes" ]; then - execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"set -o pipefail ; $PGSQLDUMP $db | $GZIP $GZIP_OPTS > '$backupdir/${db}.sql.gz'\"" - else - execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$PGSQLDUMP $db | > '$backupdir/${db}.sql'\"" - fi + execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$globalscmd\"" else - if [ "$compress" == "yes" ]; then - execstr="su - $PGSQLUSER -c \"set -o pipefail ; $PGSQLDUMP $db | $GZIP $GZIP_OPTS > '$backupdir/${db}.sql.gz'\"" + execstr="su - $PGSQLUSER -c \"$globalscmd\"" + fi + debug "$execstr" + if [ ! $test ]; then + output=`eval $execstr 2>&1` + code=$? + if [ "$code" == "0" ]; then + debug $output + info "Successfully finished pgsql globals (roles and tablespaces) dump" else - execstr="su - $PGSQLUSER -c \"$PGSQLDUMP $db > '$backupdir/${db}.sql'\"" + warning $output + warning "Failed to dump pgsql globals (roles and tablespaces)" fi fi + fi + for db in $databases; do + dumpext=".sql" + if [ "$format" != "plain" ]; then + dumpext="pg_dump" + fi + # To better support the backupninja global GZIP and rsync-friendly GZIP_OPTS + # the custom archive format is told to disable compression. The plain format + # is uncompressed by default and the tar format doesn't support pg_dump compression. + disablecustomcompress="" + if [ "$format" = "custom" ]; then + disablecustomcompress="--compress=0" + fi + dumpcmd="" + globalscmd="" + if [ "$compress" == "yes" ]; then + dumpcmd="set -o pipefail ; $PGSQLDUMP --format=$format ${disablecustomcompress} $db | $GZIP $GZIP_OPTS > '$backupdir/${db}.${dumpext}.gz'" + else + dumpcmd="$PGSQLDUMP --format=$format ${disablecustomcompress} $db | > '$backupdir/${db}.${dumpext}'" + fi + if [ $usevserver = yes ]; then + execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$dumpcmd\"" + else + execstr="su - $PGSQLUSER -c \"$dumpcmd\"" + fi debug "$execstr" if [ ! $test ]; then output=`eval $execstr 2>&1` -- cgit v1.2.3 From 0b60b58153f352e6f0c7e8f52b0824720c2a4c05 Mon Sep 17 00:00:00 2001 From: Jacob Anawalt Date: Tue, 19 Oct 2010 13:37:10 -0600 Subject: Removed extra '.' in plain/sql dumps Removed an extra '.' in file names introduced by the pg_dump options patch, resulting in names like db..sql or db..sql.gz instead of just db.sql or db.sql.gz. --- handlers/pgsql.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'handlers/pgsql.in') diff --git a/handlers/pgsql.in b/handlers/pgsql.in index 0564f5a..f68ec2f 100644 --- a/handlers/pgsql.in +++ b/handlers/pgsql.in @@ -170,7 +170,7 @@ else fi fi for db in $databases; do - dumpext=".sql" + dumpext="sql" if [ "$format" != "plain" ]; then dumpext="pg_dump" fi -- cgit v1.2.3 From 13f247afebde199ffb03d23e8662d362d5681ca6 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Sun, 7 Nov 2010 13:07:26 +0100 Subject: Fix reliance on bash for pipefail. (Closes: #602374) Thanks to Sergio Talens-Oliag for the bug report and patch. --- AUTHORS | 1 + ChangeLog | 7 +++++++ handlers/ldap.in | 2 +- handlers/mysql.in | 2 +- handlers/pgsql.in | 12 ++++++------ 5 files changed, 16 insertions(+), 8 deletions(-) (limited to 'handlers/pgsql.in') diff --git a/AUTHORS b/AUTHORS index 5ca3087..90fa918 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,3 +37,4 @@ Chris Nolan -- maildir subdirectory expansion Dan Carley -- mysql bugfix Jordi Mallach -- do not error when no jobs are configured Jacob Anawalt -- pg_dump format option +Sergio Talens-Oliag -- pipefail fixes diff --git a/ChangeLog b/ChangeLog index f7bf585..8bcdc6b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,10 +8,17 @@ version 0.9.9 -- UNRELEASED (Closes Roundup bug #598) . Remove support for duplicity < 0.4.4. Even etch-backports has a newer one. + ldap: + . Fix reliance on bash for pipefail. + mysql: + . Fix reliance on bash for pipefail. + Thanks to Sergio Talens-Oliag for the patch. + (Closes: #602374) postgresql: . Support various pg_dump formats in addition to pg_dumpall. Thanks to Jacob Anawalt for the patch. (Closes Roundup bug #2534) + . Fix reliance on bash for pipefail. rdiff: . Support reading include/exclude patterns from files using the "include @/etc/backup_includes" syntax (Closes Roundup bug diff --git a/handlers/ldap.in b/handlers/ldap.in index 83307ee..600f172 100644 --- a/handlers/ldap.in +++ b/handlers/ldap.in @@ -91,7 +91,7 @@ if [ "$ldif" == "yes" ]; then execstr="$execstr > $dumpdir/$dbsuffix.ldif" fi debug "$execstr" - output=`su root -c "set -o pipefail ; $execstr" 2>&1` + output=`su root -s /bin/bash -c "set -o pipefail ; $execstr" 2>&1` code=$? if [ "$code" == "0" ]; then debug $output diff --git a/handlers/mysql.in b/handlers/mysql.in index 0282046..05ea396 100644 --- a/handlers/mysql.in +++ b/handlers/mysql.in @@ -303,7 +303,7 @@ then debug "su $user -c \"$execstr\"" if [ ! $test ] then - output=`su $user -c "set -o pipefail ; $execstr" 2>&1` + output=`su $user -s /bin/bash -c "set -o pipefail ; $execstr" 2>&1` code=$? if [ "$code" == "0" ] then diff --git a/handlers/pgsql.in b/handlers/pgsql.in index f68ec2f..ff71ebc 100644 --- a/handlers/pgsql.in +++ b/handlers/pgsql.in @@ -115,13 +115,13 @@ fi if [ "$databases" == "all" ]; then if [ $usevserver = yes ]; then if [ "$compress" == "yes" ]; then - execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"set -o pipefail ; $PGSQLDUMPALL | $GZIP $GZIP_OPTS > '$backupdir/${vsname}.sql.gz'\"" + execstr="$VSERVER $vsname exec su - $PGSQLUSER -s /bin/bash -c \"set -o pipefail ; $PGSQLDUMPALL | $GZIP $GZIP_OPTS > '$backupdir/${vsname}.sql.gz'\"" else execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$PGSQLDUMPALL > '$backupdir/${vsname}.sql'\"" fi else if [ "$compress" == "yes" ]; then - execstr="su - $PGSQLUSER -c \"set -o pipefail ; $PGSQLDUMPALL | $GZIP $GZIP_OPTS > '$backupdir/${localhost}-all.sql.gz'\"" + execstr="su - $PGSQLUSER -s /bin/bash -c \"set -o pipefail ; $PGSQLDUMPALL | $GZIP $GZIP_OPTS > '$backupdir/${localhost}-all.sql.gz'\"" else execstr="su - $PGSQLUSER -c \"$PGSQLDUMPALL > '$backupdir/${localhost}-all.sql'\"" fi @@ -152,9 +152,9 @@ else globalscmd="$PGSQLDUMPALL -g > '$backupdir/globals.sql'" fi if [ $usevserver = yes ]; then - execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$globalscmd\"" + execstr="$VSERVER $vsname exec su - $PGSQLUSER -s /bin/bash -c \"$globalscmd\"" else - execstr="su - $PGSQLUSER -c \"$globalscmd\"" + execstr="su - $PGSQLUSER -s /bin/bash -c \"$globalscmd\"" fi debug "$execstr" if [ ! $test ]; then @@ -189,9 +189,9 @@ else dumpcmd="$PGSQLDUMP --format=$format ${disablecustomcompress} $db | > '$backupdir/${db}.${dumpext}'" fi if [ $usevserver = yes ]; then - execstr="$VSERVER $vsname exec su - $PGSQLUSER -c \"$dumpcmd\"" + execstr="$VSERVER $vsname exec su - $PGSQLUSER -s /bin/bash -c \"$dumpcmd\"" else - execstr="su - $PGSQLUSER -c \"$dumpcmd\"" + execstr="su - $PGSQLUSER -s /bin/bash -c \"$dumpcmd\"" fi debug "$execstr" if [ ! $test ]; then -- cgit v1.2.3 From 6aca9c822b640372b0fad24d2a3a288ef2cd6a0f Mon Sep 17 00:00:00 2001 From: intrigeri Date: Mon, 25 Apr 2011 18:55:58 +0200 Subject: All handlers: stop using "local VAR" outside functions. (Closes: #530647) --- ChangeLog | 2 ++ handlers/dup.in | 2 +- handlers/mysql.in | 4 ++-- handlers/pgsql.in | 4 ++-- handlers/rdiff.in | 2 +- handlers/svn.in | 4 ++-- handlers/sys.in | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) (limited to 'handlers/pgsql.in') diff --git a/ChangeLog b/ChangeLog index 2a94593..29ce38d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ version 0.9.9 -- UNRELEASED action. Thanks to Olivier Berger for the patch. (Closes: #511300) handler changes + all handlers: + . Stop using "local VAR" outside functions. (Closes: #530647) dup: . Use --tempdir option rather than TMPDIR environment variable. (Closes Roundup bug #598) diff --git a/handlers/dup.in b/handlers/dup.in index e3475b8..1d345a3 100644 --- a/handlers/dup.in +++ b/handlers/dup.in @@ -51,7 +51,7 @@ fi ### VServers # If vservers are configured, check that the ones listed in $vsnames do exist. -local usevserver=no +usevserver=no if [ $vservers_are_available = yes ]; then if [ "$vsnames" = all ]; then vsnames="$found_vservers" diff --git a/handlers/mysql.in b/handlers/mysql.in index 05ea396..185a98a 100644 --- a/handlers/mysql.in +++ b/handlers/mysql.in @@ -24,8 +24,8 @@ getconf configfile /etc/mysql/debian.cnf # Decide if the handler should operate on a vserver or on the host. # In the former case, check that $vsname exists and is running. -local usevserver=no -local vroot +usevserver=no +vroot='' if [ $vservers_are_available = yes ]; then if [ -n "$vsname" ]; then # does it exist ? diff --git a/handlers/pgsql.in b/handlers/pgsql.in index ff71ebc..a50d3c7 100644 --- a/handlers/pgsql.in +++ b/handlers/pgsql.in @@ -15,8 +15,8 @@ localhost=`hostname` # Decide if the handler should operate on a vserver or on the host. # In the former case, check that $vsname exists and is running. -local usevserver=no -local vroot +usevserver=no +vroot='' if [ $vservers_are_available = yes ]; then if [ -n "$vsname" ]; then # does it exist ? diff --git a/handlers/rdiff.in b/handlers/rdiff.in index e391edd..c2f5aa0 100644 --- a/handlers/rdiff.in +++ b/handlers/rdiff.in @@ -115,7 +115,7 @@ fi ### CHECK CONFIG ### # If vservers are configured, check that the ones listed in $vsnames do exist. -local usevserver=no +usevserver=no if [ $vservers_are_available = yes ]; then if [ "$vsnames" = all ]; then vsnames="$found_vservers" diff --git a/handlers/svn.in b/handlers/svn.in index 5e5531a..bb70eee 100644 --- a/handlers/svn.in +++ b/handlers/svn.in @@ -14,8 +14,8 @@ error=0 # Decide if the handler should operate on a vserver or on the host. # In the former case, check that $vsname exists and is running. -local usevserver=no -local vroot +usevserver=no +vroot='' if [ $vservers_are_available = yes ]; then if [ -n "$vsname" ]; then # does it exist ? diff --git a/handlers/sys.in b/handlers/sys.in index fcf3e31..101745c 100755 --- a/handlers/sys.in +++ b/handlers/sys.in @@ -103,7 +103,7 @@ getconf lvm no getconf vsnames all # If vservers are configured, check that the ones listed in $vsnames are running. -local usevserver=no +usevserver=no if [ $vservers_are_available = yes ]; then if [ "$vsnames" = all ]; then vsnames="$found_vservers" -- cgit v1.2.3