Deploy: do not globally abort on failures to apply in a node

2 files changed, 56 insertions, 26 deletions

diff --git a/lib/hydra/deploy b/lib/hydra/deploy
index 84e31a2..96dd67b 100644
--- a/lib/hydra/deploy
+++ b/lib/hydra/deploy
@@ -27,7 +27,7 @@ function hydra_deploy_setup {
       # Check for config/node name
       if [ "$FQDN" == ".yaml" ]; then
         echo "Cannot find config for $NODE."
-        exit 1
+        return 1
       fi
 
       DEPLOY_BASE="/etc"
@@ -36,10 +36,10 @@ function hydra_deploy_setup {
       DEPLOY_DEST="$FQDN:"
       RSYNC_PATH="sudo rsync -q"
       REMOTE_ENV="LC_ALL=C"
-      hydra_deploy_set_manifest
+      hydra_deploy_set_manifest || return 1
     else
       echo "No folder specified."
-      exit 1
+      return 1
     fi
   elif [ "$1" == "folder" ]; then
     # Deploy in a local folder
@@ -56,15 +56,15 @@ function hydra_deploy_setup {
         FQDN="`cat $FOLDER/etc/hostname`"
       fi
 
-      hydra_deploy_set_manifest
+      hydra_deploy_set_manifest || return 1
 
       if [ ! -d "$FOLDER" ]; then
         echo "folder not found: $FOLDER"
-        exit 1
+        return 1
       fi
     else
       echo "No node specified."
-      exit 1
+      return 1
     fi
   else
     # Deploy on the localhost
@@ -73,7 +73,7 @@ function hydra_deploy_setup {
     DEPLOY_RSYNC=""
     FQDN="`cat /etc/hostname`"
     PUPPET_OPTS="--confdir=$HYDRA_FOLDER/puppet --modulepath=$HYDRA_FOLDER/puppet/modules"
-    hydra_deploy_set_manifest $HYDRA_FOLDER
+    hydra_deploy_set_manifest $HYDRA_FOLDER || return 1
 
     # Remove old eyaml symlinks if exists
     if [ -h "$HYDRA_FOLDER/puppet/keys/private_key.pkcs7.pem" ]; then
@@ -118,7 +118,7 @@ function hydra_deploy_set_manifest {
     PUPPET_MANIFEST="$prefix/puppet/manifests/nodes/default.pp"
   else
     echo "no manifest found for $FQDN"
-    exit 1
+    return 1
   fi
 }
 
@@ -151,18 +151,30 @@ function hydra_deploy_copy_keys {
 
   # Test for multi-keys setup
   if [ -e "$HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/private_key.pkcs7.pem" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/private_key.pkcs7.pem $DEPLOY_DEST/etc/puppet/keys/private_key.pkcs7.pem
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/private_key.pkcs7.pem \
+      $DEPLOY_DEST/etc/puppet/keys/private_key.pkcs7.pem \
+      || return 1
   # Then try single-key setup
   elif [ -e "$HYDRA_FOLDER/puppet/keys/private_key.pkcs7.pem" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/keys/private_key.pkcs7.pem $DEPLOY_DEST/etc/puppet/keys/private_key.pkcs7.pem
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/keys/private_key.pkcs7.pem \
+      $DEPLOY_DEST/etc/puppet/keys/private_key.pkcs7.pem \
+      || return 1
   fi
 
   # Test for multi-keys setup
   if [ -e "$HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/public_key.pkcs7.pem" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/public_key.pkcs7.pem $DEPLOY_DEST/etc/puppet/keys/public_key.pkcs7.pem
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/keys/$FQDN/eyaml/public_key.pkcs7.pem \
+      $DEPLOY_DEST/etc/puppet/keys/public_key.pkcs7.pem \
+      || return 1
   # Then try single-key setup
   elif [ -e "$HYDRA_FOLDER/puppet/keys/public_key.pkcs7.pem" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/keys/public_key.pkcs7.pem $DEPLOY_DEST/etc/puppet/keys/public_key.pkcs7.pem
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/keys/public_key.pkcs7.pem \
+      $DEPLOY_DEST/etc/puppet/keys/public_key.pkcs7.pem \
+      || return 1
   fi
 }
 
@@ -174,22 +186,34 @@ function hydra_deploy_copy_secrets {
     location="remote"
   fi
 
-  $DEPLOY_COMMAND mkdir -p $FOLDER/etc/puppet/config/secrets/{domain,location,node,role}
+  $DEPLOY_COMMAND mkdir -p $FOLDER/etc/puppet/config/secrets/{domain,location,node,role} || return 1
 
   if [ ! -z "$DOMAIN" ] && [ -e "$HYDRA_FOLDER/puppet/config/secrets/domain/$DOMAIN.yaml" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/config/secrets/domain/$DOMAIN.yaml $DEPLOY_DEST/etc/puppet/config/secrets/domain/
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/config/secrets/domain/$DOMAIN.yaml \
+      $DEPLOY_DEST/etc/puppet/config/secrets/domain/ \
+      || return 1
   fi
 
   if [ ! -z "$LOCATION" ] && [ -e "$HYDRA_FOLDER/puppet/config/secrets/location/$LOCATION.yaml" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/config/secrets/location/$LOCATION.yaml $DEPLOY_DEST/etc/puppet/config/secrets/location/
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/config/secrets/location/$LOCATION.yaml \
+      $DEPLOY_DEST/etc/puppet/config/secrets/location/ \
+      || return 1
   fi
 
   if [ ! -z "$ROLE" ] && [ -e "$HYDRA_FOLDER/puppet/config/secrets/role/$ROLE.yaml" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/config/secrets/role/$ROLE.yaml $DEPLOY_DEST/etc/puppet/config/secrets/role/
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/config/secrets/role/$ROLE.yaml \
+      $DEPLOY_DEST/etc/puppet/config/secrets/role/ \
+      || return 1
   fi
 
   if [ ! -z "$FQDN" ] && [ -e "$HYDRA_FOLDER/puppet/config/secrets/node/$FQDN.yaml" ]; then
-    hydra_deploy_copy $location $HYDRA_FOLDER/puppet/config/secrets/node/$FQDN.yaml $DEPLOY_DEST/etc/puppet/config/secrets/node/
+    hydra_deploy_copy $location \
+      $HYDRA_FOLDER/puppet/config/secrets/node/$FQDN.yaml \
+      $DEPLOY_DEST/etc/puppet/config/secrets/node/ \
+      || return 1
   fi
 }
 
@@ -208,7 +232,7 @@ function hydra_deploy_copy {
     # Check result
     if [ "$?" != "0" ]; then
       echo "Error copying data to $FQDN."
-      exit 1
+      return 1
     fi
   fi
 }
@@ -225,7 +249,7 @@ function hydra_deploy_facts_collect {
     # Check result
     if [ "$?" != "0" ]; then
       echo "Error retrieving facts from $FQDN."
-      exit 1
+      return 1
     fi
   fi
 }
diff --git a/share/hydra/deploy b/share/hydra/deploy
index e55e545..38d8bcb 100755
--- a/share/hydra/deploy
+++ b/share/hydra/deploy
@@ -49,7 +49,7 @@ for node in $NODES; do
 
     # Setup deploy environment
     FOLDER=$node
-    hydra_deploy_setup folder $FOLDER
+    hydra_deploy_setup folder $FOLDER || continue
 
     # Check if puppet is installed
     if [ ! -e "$FOLDER/usr/bin/puppet" ]; then
@@ -78,7 +78,7 @@ for node in $NODES; do
     echo "Deploying to localhost..."
 
     # Setup deploy environment
-    hydra_deploy_setup
+    hydra_deploy_setup || continue
 
     # Ensure key availability
     hydra $HYDRA eyaml $FQDN
@@ -99,7 +99,7 @@ for node in $NODES; do
     echo "Deploying to $node..."
 
     # Setup deploy environment
-    hydra_deploy_setup remote $node
+    hydra_deploy_setup remote $node || continue
 
     # Check if puppet is installed
     $HYDRA_CONNECT $FQDN <<EOF
@@ -115,24 +115,30 @@ EOF
     # Check connection
     if [ "$?" != "0" ]; then
       echo "Error connecting or setting up $FQDN."
-      exit 1
+      continue
     fi
 
     # Create folders
     hydra_deploy_mkdirs
 
     # Collect facts
-    hydra_deploy_facts_collect remote
+    hydra_deploy_facts_collect remote || continue
 
     # Sync repository to server
     echo "Syncing configuration..."
     $DEPLOY_RSYNC "$RSYNC_PATH" $DEPLOY_OPTS
 
+    # Check connection
+    if [ "$?" != "0" ]; then
+      echo "Error syncing to $FQDN."
+      continue
+    fi
+
     # Copy keys
-    hydra_deploy_copy_keys remote
+    hydra_deploy_copy_keys remote || continue
 
     # Copy configuration
-    hydra_deploy_copy_secrets remote
+    hydra_deploy_copy_secrets remote || continue
 
     # Run puppet
     echo "Applying configuration..."