add support for `leap facts`. includes some fun new helpers, like run_with_progress(), capture(), and replace_file!().

13 files changed, 330 insertions, 48 deletions

diff --git a/lib/core_ext/json.rb b/lib/core_ext/json.rb
new file mode 100644
index 0000000..3b08a04
--- /dev/null
+++ b/lib/core_ext/json.rb
@@ -0,0 +1,39 @@
+module JSON
+  #
+  # Output JSON from ruby objects in such a manner that all the hashes and arrays are output in alphanumeric sorted order.
+  # This is required so that our generated configs don't throw puppet or git for a tizzy fit.
+  #
+  # Beware: some hacky stuff ahead.
+  #
+  # This relies on the pure ruby implementation of JSON.generate (i.e. require 'json/pure')
+  # see https://github.com/flori/json/blob/master/lib/json/pure/generator.rb
+  #
+  # The Oj way that we are not using: Oj.dump(obj, :mode => :compat, :indent => 2)
+  #
+  def self.sorted_generate(obj)
+    # modify hash and array
+    Hash.class_eval do
+      alias_method :each_without_sort, :each
+      def each(&block)
+        keys.sort {|a,b| a.to_s <=> b.to_s }.each do |key|
+          yield key, self[key]
+        end
+      end
+    end
+    Array.class_eval do
+      alias_method :each_without_sort, :each
+      def each(&block)
+        sort {|a,b| a.to_s <=> b.to_s }.each_without_sort &block
+      end
+    end
+
+    # generate json
+    json_str = JSON.pretty_generate(obj)
+
+    # restore hash and array
+    Hash.class_eval  {alias_method :each, :each_without_sort}
+    Array.class_eval {alias_method :each, :each_without_sort}
+
+    return json_str
+  end
+end
diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb
index 5d74813..259c00f 100644
--- a/lib/leap_cli.rb
+++ b/lib/leap_cli.rb
@@ -11,6 +11,7 @@ require 'core_ext/hash'
 require 'core_ext/boolean'
 require 'core_ext/nil'
 require 'core_ext/string'
+require 'core_ext/json'
 
 require 'leap_cli/log'
 require 'leap_cli/path'
diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb
new file mode 100644
index 0000000..3653c46
--- /dev/null
+++ b/lib/leap_cli/commands/facts.rb
@@ -0,0 +1,93 @@
+#
+# Gather facter facts
+#
+
+module LeapCli; module Commands
+
+  desc 'Gather information on nodes.'
+  command :facts do |facts|
+    facts.desc 'Query servers to update facts.json.'
+    facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json"
+    facts.arg_name 'FILTER'
+    facts.command :update do |update|
+      update.action do |global_options,options,args|
+        update_facts(global_options, options, args)
+      end
+    end
+  end
+
+  protected
+
+  def facter_cmd
+    'facter --json ' + Leap::Platform.facts.join(' ')
+  end
+
+  def remove_node_facts(name)
+    if file_exists?(:facts)
+      update_facts_file({name => nil})
+    end
+  end
+
+  def update_node_facts(name, facts)
+    update_facts_file({name => facts})
+  end
+
+  def rename_node_facts(old_name, new_name)
+    if file_exists?(:facts)
+      facts = JSON.parse(read_file(:facts) || {})
+      facts[new_name] = facts[old_name]
+      facts[old_name] = nil
+      update_facts_file(facts, true)
+    end
+  end
+
+  #
+  # if overwrite = true, then ignore existing facts.json.
+  #
+  def update_facts_file(new_facts, overwrite=false)
+    replace_file!(:facts) do |content|
+      if overwrite || content.nil? || content.empty?
+        old_facts = {}
+      else
+        old_facts = JSON.parse(content)
+      end
+      facts = old_facts.merge(new_facts)
+      facts.each do |name, value|
+        if value.is_a? String
+          if value == ""
+            value = nil
+          else
+            value = JSON.parse(value)
+          end
+        end
+        if value.is_a? Hash
+          value.delete_if {|key,v| v.nil?}
+        end
+        facts[name] = value
+      end
+      facts.delete_if do |name, value|
+        value.nil? || value.empty?
+      end
+      if facts.empty?
+        nil
+      else
+        JSON.sorted_generate(facts) + "\n"
+      end
+    end
+  end
+
+  private
+
+  def update_facts(global_options, options, args)
+    nodes = manager.filter(args)
+    new_facts = {}
+    ssh_connect(nodes) do |ssh|
+      ssh.leap.run_with_progress(facter_cmd) do |response|
+        new_facts[response[:host]] = response[:data].strip
+      end
+    end
+    overwrite_existing = args.empty?
+    update_facts_file(new_facts, overwrite_existing)
+  end
+
+end; end
\ No newline at end of file
diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb
index bf552d3..12c9500 100644
--- a/lib/leap_cli/commands/node.rb
+++ b/lib/leap_cli/commands/node.rb
@@ -45,7 +45,7 @@ module LeapCli; module Commands
 
     node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages'
     node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " +
-                   "copying the authorized_keys file, and installing packages that are required for deploying. " +
+                   "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " +
                    "Node init must be run before deploying to a server, and the server must be running and available via the network. " +
                    "This command only needs to be run once, but there is no harm in running it multiple times."
     node.arg_name 'FILTER' #, :optional => false, :multiple => false
@@ -61,6 +61,13 @@ module LeapCli; module Commands
           ssh_connect(node, :bootstrap => true, :echo => options[:echo]) do |ssh|
             ssh.install_authorized_keys
             ssh.install_prerequisites
+            ssh.leap.capture(facter_cmd) do |response|
+              if response[:exitcode] == 0
+                update_node_facts(node.name, response[:data])
+              else
+                log :failed, "to run facter on #{node.name}"
+              end
+            end
           end
           finished << node.name
         end
@@ -79,6 +86,7 @@ module LeapCli; module Commands
           rename_file! [path, node.name], [path, new_name]
         end
         remove_directory! [:node_files_dir, node.name]
+        rename_node_facts(node.name, new_name)
       end
     end
 
@@ -93,6 +101,7 @@ module LeapCli; module Commands
         if node.vagrant?
           vagrant_command("destroy --force", [node.name])
         end
+        remote_node_facts(node.name)
       end
     end
   end
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
index 714cd6a..d2bc1f3 100644
--- a/lib/leap_cli/config/manager.rb
+++ b/lib/leap_cli/config/manager.rb
@@ -8,8 +8,16 @@ module LeapCli
     #
     class Manager
 
+      ##
+      ## ATTRIBUTES
+      ##
+
       attr_reader :services, :tags, :nodes, :provider, :common, :secrets
 
+      def facts
+        @facts ||= JSON.parse(Util.read_file(:facts) || {})
+      end
+
       ##
       ## IMPORT EXPORT
       ##
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
index 4f348b3..b88c7b4 100644
--- a/lib/leap_cli/config/object.rb
+++ b/lib/leap_cli/config/object.rb
@@ -43,7 +43,7 @@ module LeapCli
 
       def dump_json
         evaluate
-        generate_json(self)
+        JSON.sorted_generate(self)
       end
 
       def evaluate
@@ -278,44 +278,6 @@ module LeapCli
       end
 
       #
-      # Output json from ruby objects in such a manner that all the hashes and arrays are output in alphanumeric sorted order.
-      # This is required so that our generated configs don't throw puppet or git for a tizzy fit.
-      #
-      # Beware: some hacky stuff ahead.
-      #
-      # This relies on the pure ruby implementation of JSON.generate (i.e. require 'json/pure')
-      # see https://github.com/flori/json/blob/master/lib/json/pure/generator.rb
-      #
-      # The Oj way that we are not using: Oj.dump(obj, :mode => :compat, :indent => 2)
-      #
-      def generate_json(obj)
-        # modify hash and array
-        Hash.class_eval do
-          alias_method :each_without_sort, :each
-          def each(&block)
-            keys.sort {|a,b| a.to_s <=> b.to_s }.each do |key|
-              yield key, self[key]
-            end
-          end
-        end
-        Array.class_eval do
-          alias_method :each_without_sort, :each
-          def each(&block)
-            sort {|a,b| a.to_s <=> b.to_s }.each_without_sort &block
-          end
-        end
-
-        # generate json
-        return_value = JSON.pretty_generate(obj)
-
-        # restore hash and array
-        Hash.class_eval  {alias_method :each, :each_without_sort}
-        Array.class_eval {alias_method :each, :each_without_sort}
-
-        return return_value
-      end
-
-      #
       # when merging, we raise an error if this method returns true for the two values.
       #
       def type_mismatch?(old_value, new_value)
diff --git a/lib/leap_cli/remote/leap_plugin.rb b/lib/leap_cli/remote/leap_plugin.rb
index 2c427e9..8cc96d4 100644
--- a/lib/leap_cli/remote/leap_plugin.rb
+++ b/lib/leap_cli/remote/leap_plugin.rb
@@ -39,6 +39,122 @@ module LeapCli; module Remote; module LeapPlugin
     run "touch #{INITIALIZED_FILE}"
   end
 
+  #
+  # This is a hairy ugly hack, exactly the kind of stuff that makes ruby
+  # dangerous and too much fun for its own good.
+  #
+  # In most places, we run remote ssh without a current 'task'. This works fine,
+  # except that in a few places, the behavior of capistrano ssh is controlled by
+  # the options of the current task.
+  #
+  # We don't want to create an actual current task, because tasks are no fun
+  # and can't take arguments or return values. So, when we need to configure
+  # things that can only be configured in a task, we use this handy hack to
+  # fake the current task.
+  #
+  # This is NOT thread safe, but could be made to be so with some extra work.
+  #
+  def with_task(name)
+    task = @config.tasks[name]
+    @config.class.send(:alias_method, :original_current_task, :current_task)
+    @config.class.send(:define_method, :current_task, Proc.new(){ task })
+    begin
+      yield
+    ensure
+      @config.class.send(:remove_method, :current_task)
+      @config.class.send(:alias_method, :current_task, :original_current_task)
+    end
+  end
+
+  #
+  # similar to run(cmd, &block), but with:
+  #
+  # * exit codes
+  # * stdout and stderr are combined
+  #
+  def stream(cmd, &block)
+    command = '%s 2>&1; echo "exitcode=$?"' % cmd
+    run(command) do |channel, stream, data|
+      exitcode = nil
+      if data =~ /exitcode=(\d+)\n/
+        exitcode = $1.to_i
+        data.sub!(/exitcode=(\d+)\n/,'')
+      end
+      yield({:host => channel[:host], :data => data, :exitcode => exitcode})
+    end
+  end
+
+  #
+  # like stream, but capture all the output before returning
+  #
+  def capture(cmd, &block)
+    command = '%s 2>&1; echo "exitcode=$?" 2>&1;' % cmd
+    host_data = {}
+    run(command) do |channel, stream, data|
+      host_data[channel[:host]] ||= ""
+      if data =~ /exitcode=(\d+)\n/
+        exitcode = $1.to_i
+        data.sub!(/exitcode=(\d+)\n/,'')
+        host_data[channel[:host]] += data
+        yield({:host => channel[:host], :data => host_data[channel[:host]], :exitcode => exitcode})
+      else
+        host_data[channel[:host]] += data
+      end
+    end
+  end
+
+  #
+  # Run a command, with a nice status report and progress indicator.
+  # Only successful results are returned, errors are printed.
+  #
+  # For each successful run on each host, block is yielded with a hash like so:
+  #
+  # {:host => 'bluejay', :exitcode => 0, :data => 'shell output'}
+  #
+  def run_with_progress(cmd, &block)
+    ssh_failures = []
+    exitcode_failures = []
+    succeeded = []
+    task = LeapCli.log_level > 1 ? :standard_task : :skip_errors_task
+    with_task(task) do
+      log :querying, 'facts' do
+        progress "   "
+        call_on_failure do |host|
+          ssh_failures << host
+          progress 'F'
+        end
+        capture(cmd) do |response|
+          if response[:exitcode] == 0
+            progress '.'
+            yield response
+          else
+            exitcode_failures << response
+            progress 'F'
+          end
+        end
+      end
+    end
+    puts "done"
+    if ssh_failures.any?
+      log :failed, 'to connect to nodes: ' + ssh_failures.join(' ')
+    end
+    if exitcode_failures.any?
+      log :failed, 'to run successfully:' do
+        exitcode_failures.each do |response|
+          log "[%s] exit %s - %s" % [response[:host], response[:exitcode], response[:data].strip]
+        end
+      end
+    end
+  rescue Capistrano::RemoteError => err
+    log :error, err.to_s
+  end
+
+  private
+
+  def progress(str='.')
+    $stdout.print str; $stdout.flush;
+  end
+
   #def mkdir(dir)
   #  run "mkdir -p #{dir}"
   #end
diff --git a/lib/leap_cli/remote/tasks.rb b/lib/leap_cli/remote/tasks.rb
index f967db1..0721c34 100644
--- a/lib/leap_cli/remote/tasks.rb
+++ b/lib/leap_cli/remote/tasks.rb
@@ -25,9 +25,12 @@ task :install_prerequisites, :max_hosts => MAX_HOSTS do
   leap.mark_initialized
 end
 
-#task :apply_puppet, :max_hosts => MAX_HOSTS do
-#  raise "now such directory #{puppet_source}" unless File.directory?(puppet_source)
-#  leap.log :applying, "puppet" do
-#    puppet.apply
-#  end
-#end
+#
+# just dummies, used to capture task options
+#
+
+task :skip_errors_task, :on_error => :continue, :max_hosts => MAX_HOSTS do
+end
+
+task :standard_task, :max_hosts => MAX_HOSTS do
+end
\ No newline at end of file
diff --git a/lib/leap_cli/util.rb b/lib/leap_cli/util.rb
index b2a1dcf..116c212 100644
--- a/lib/leap_cli/util.rb
+++ b/lib/leap_cli/util.rb
@@ -203,6 +203,40 @@ module LeapCli
       end
     end
 
+    #
+    # replace contents of a file, with an exclusive lock.
+    #
+    # 1. locks file
+    # 2. reads contents
+    # 3. yields contents
+    # 4. replaces file with return value of the block
+    #
+    def replace_file!(filepath, &block)
+      filepath = Path.named_path(filepath)
+      if !File.exists?(filepath)
+        content = yield(nil)
+        unless content.nil?
+          write_file!(filepath, content)
+          log :created, filepath
+        end
+      else
+        File.open(filepath, File::RDWR|File::CREAT, 0644) do |f|
+          f.flock(File::LOCK_EX)
+          old_content = f.read
+          new_content = yield(old_content)
+          if old_content == new_content
+            log :nochange, filepath, 2
+          else
+            f.rewind
+            f.write(new_content)
+            f.flush
+            f.truncate(f.pos)
+            log :updated, filepath
+          end
+        end
+      end
+    end
+
     def remove_file!(filepath)
       filepath = Path.named_path(filepath)
       if File.exists?(filepath)
diff --git a/lib/leap_cli/util/remote_command.rb b/lib/leap_cli/util/remote_command.rb
index 57234eb..db02037 100644
--- a/lib/leap_cli/util/remote_command.rb
+++ b/lib/leap_cli/util/remote_command.rb
@@ -73,6 +73,7 @@ module LeapCli; module Util; module RemoteCommand
     @capistrano_enabled ||= begin
       require 'capistrano'
       require 'capistrano/cli'
+      require 'lib_ext/capistrano_connections'
       require 'leap_cli/remote/leap_plugin'
       require 'leap_cli/remote/puppet_plugin'
       require 'leap_cli/remote/rsync_plugin'
diff --git a/lib/leap_cli/version.rb b/lib/leap_cli/version.rb
index 45c5df2..bbec03a 100644
--- a/lib/leap_cli/version.rb
+++ b/lib/leap_cli/version.rb
@@ -1,6 +1,6 @@
 module LeapCli
   unless defined?(LeapCli::VERSION)
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
     SUMMARY = 'Command line interface to the LEAP platform'
     DESCRIPTION = 'The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home.'
     LOAD_PATHS = ['lib', 'vendor/certificate_authority/lib', 'vendor/rsync_command/lib']
diff --git a/lib/lib_ext/capistrano_connections.rb b/lib/lib_ext/capistrano_connections.rb
new file mode 100644
index 0000000..c46455f
--- /dev/null
+++ b/lib/lib_ext/capistrano_connections.rb
@@ -0,0 +1,16 @@
+module Capistrano
+  class Configuration
+    module Connections
+      def failed!(server)
+        @failure_callback.call(server) if @failure_callback
+        Thread.current[:failed_sessions] << server
+      end
+
+      def call_on_failure(&block)
+        @failure_callback = block
+      end
+    end
+  end
+end
+
+
diff --git a/test/provider/files/service-definitions/provider.json.erb b/test/provider/files/service-definitions/provider.json.erb
index 742b88f..96953c5 100644
--- a/test/provider/files/service-definitions/provider.json.erb
+++ b/test/provider/files/service-definitions/provider.json.erb
@@ -35,5 +35,5 @@
   #   "ca_cert_uri": "https://springbok/ca.crt"
   # }
 
-  generate_json hsh
+  JSON.sorted_generate hsh
 %>
\ No newline at end of file