From 8f79b632aeeee1111087dee6ebb6302aca700bbd Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 4 Jun 2013 23:06:10 -0700 Subject: add support for `leap facts`. includes some fun new helpers, like run_with_progress(), capture(), and replace_file!(). --- lib/core_ext/json.rb | 39 +++++++ lib/leap_cli.rb | 1 + lib/leap_cli/commands/facts.rb | 93 +++++++++++++++++ lib/leap_cli/commands/node.rb | 11 +- lib/leap_cli/config/manager.rb | 8 ++ lib/leap_cli/config/object.rb | 40 +------ lib/leap_cli/remote/leap_plugin.rb | 116 +++++++++++++++++++++ lib/leap_cli/remote/tasks.rb | 15 +-- lib/leap_cli/util.rb | 34 ++++++ lib/leap_cli/util/remote_command.rb | 1 + lib/leap_cli/version.rb | 2 +- lib/lib_ext/capistrano_connections.rb | 16 +++ .../files/service-definitions/provider.json.erb | 2 +- 13 files changed, 330 insertions(+), 48 deletions(-) create mode 100644 lib/core_ext/json.rb create mode 100644 lib/leap_cli/commands/facts.rb create mode 100644 lib/lib_ext/capistrano_connections.rb diff --git a/lib/core_ext/json.rb b/lib/core_ext/json.rb new file mode 100644 index 0000000..3b08a04 --- /dev/null +++ b/lib/core_ext/json.rb @@ -0,0 +1,39 @@ +module JSON + # + # Output JSON from ruby objects in such a manner that all the hashes and arrays are output in alphanumeric sorted order. + # This is required so that our generated configs don't throw puppet or git for a tizzy fit. + # + # Beware: some hacky stuff ahead. + # + # This relies on the pure ruby implementation of JSON.generate (i.e. require 'json/pure') + # see https://github.com/flori/json/blob/master/lib/json/pure/generator.rb + # + # The Oj way that we are not using: Oj.dump(obj, :mode => :compat, :indent => 2) + # + def self.sorted_generate(obj) + # modify hash and array + Hash.class_eval do + alias_method :each_without_sort, :each + def each(&block) + keys.sort {|a,b| a.to_s <=> b.to_s }.each do |key| + yield key, self[key] + end + end + end + Array.class_eval do + alias_method :each_without_sort, :each + def each(&block) + sort {|a,b| a.to_s <=> b.to_s }.each_without_sort &block + end + end + + # generate json + json_str = JSON.pretty_generate(obj) + + # restore hash and array + Hash.class_eval {alias_method :each, :each_without_sort} + Array.class_eval {alias_method :each, :each_without_sort} + + return json_str + end +end diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb index 5d74813..259c00f 100644 --- a/lib/leap_cli.rb +++ b/lib/leap_cli.rb @@ -11,6 +11,7 @@ require 'core_ext/hash' require 'core_ext/boolean' require 'core_ext/nil' require 'core_ext/string' +require 'core_ext/json' require 'leap_cli/log' require 'leap_cli/path' diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb new file mode 100644 index 0000000..3653c46 --- /dev/null +++ b/lib/leap_cli/commands/facts.rb @@ -0,0 +1,93 @@ +# +# Gather facter facts +# + +module LeapCli; module Commands + + desc 'Gather information on nodes.' + command :facts do |facts| + facts.desc 'Query servers to update facts.json.' + facts.long_desc "Queries every node included in FILTER and saves the important information to facts.json" + facts.arg_name 'FILTER' + facts.command :update do |update| + update.action do |global_options,options,args| + update_facts(global_options, options, args) + end + end + end + + protected + + def facter_cmd + 'facter --json ' + Leap::Platform.facts.join(' ') + end + + def remove_node_facts(name) + if file_exists?(:facts) + update_facts_file({name => nil}) + end + end + + def update_node_facts(name, facts) + update_facts_file({name => facts}) + end + + def rename_node_facts(old_name, new_name) + if file_exists?(:facts) + facts = JSON.parse(read_file(:facts) || {}) + facts[new_name] = facts[old_name] + facts[old_name] = nil + update_facts_file(facts, true) + end + end + + # + # if overwrite = true, then ignore existing facts.json. + # + def update_facts_file(new_facts, overwrite=false) + replace_file!(:facts) do |content| + if overwrite || content.nil? || content.empty? + old_facts = {} + else + old_facts = JSON.parse(content) + end + facts = old_facts.merge(new_facts) + facts.each do |name, value| + if value.is_a? String + if value == "" + value = nil + else + value = JSON.parse(value) + end + end + if value.is_a? Hash + value.delete_if {|key,v| v.nil?} + end + facts[name] = value + end + facts.delete_if do |name, value| + value.nil? || value.empty? + end + if facts.empty? + nil + else + JSON.sorted_generate(facts) + "\n" + end + end + end + + private + + def update_facts(global_options, options, args) + nodes = manager.filter(args) + new_facts = {} + ssh_connect(nodes) do |ssh| + ssh.leap.run_with_progress(facter_cmd) do |response| + new_facts[response[:host]] = response[:data].strip + end + end + overwrite_existing = args.empty? + update_facts_file(new_facts, overwrite_existing) + end + +end; end \ No newline at end of file diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb index bf552d3..12c9500 100644 --- a/lib/leap_cli/commands/node.rb +++ b/lib/leap_cli/commands/node.rb @@ -45,7 +45,7 @@ module LeapCli; module Commands node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages' node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " + - "copying the authorized_keys file, and installing packages that are required for deploying. " + + "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " + "Node init must be run before deploying to a server, and the server must be running and available via the network. " + "This command only needs to be run once, but there is no harm in running it multiple times." node.arg_name 'FILTER' #, :optional => false, :multiple => false @@ -61,6 +61,13 @@ module LeapCli; module Commands ssh_connect(node, :bootstrap => true, :echo => options[:echo]) do |ssh| ssh.install_authorized_keys ssh.install_prerequisites + ssh.leap.capture(facter_cmd) do |response| + if response[:exitcode] == 0 + update_node_facts(node.name, response[:data]) + else + log :failed, "to run facter on #{node.name}" + end + end end finished << node.name end @@ -79,6 +86,7 @@ module LeapCli; module Commands rename_file! [path, node.name], [path, new_name] end remove_directory! [:node_files_dir, node.name] + rename_node_facts(node.name, new_name) end end @@ -93,6 +101,7 @@ module LeapCli; module Commands if node.vagrant? vagrant_command("destroy --force", [node.name]) end + remote_node_facts(node.name) end end end diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb index 714cd6a..d2bc1f3 100644 --- a/lib/leap_cli/config/manager.rb +++ b/lib/leap_cli/config/manager.rb @@ -8,8 +8,16 @@ module LeapCli # class Manager + ## + ## ATTRIBUTES + ## + attr_reader :services, :tags, :nodes, :provider, :common, :secrets + def facts + @facts ||= JSON.parse(Util.read_file(:facts) || {}) + end + ## ## IMPORT EXPORT ## diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb index 4f348b3..b88c7b4 100644 --- a/lib/leap_cli/config/object.rb +++ b/lib/leap_cli/config/object.rb @@ -43,7 +43,7 @@ module LeapCli def dump_json evaluate - generate_json(self) + JSON.sorted_generate(self) end def evaluate @@ -277,44 +277,6 @@ module LeapCli return result end - # - # Output json from ruby objects in such a manner that all the hashes and arrays are output in alphanumeric sorted order. - # This is required so that our generated configs don't throw puppet or git for a tizzy fit. - # - # Beware: some hacky stuff ahead. - # - # This relies on the pure ruby implementation of JSON.generate (i.e. require 'json/pure') - # see https://github.com/flori/json/blob/master/lib/json/pure/generator.rb - # - # The Oj way that we are not using: Oj.dump(obj, :mode => :compat, :indent => 2) - # - def generate_json(obj) - # modify hash and array - Hash.class_eval do - alias_method :each_without_sort, :each - def each(&block) - keys.sort {|a,b| a.to_s <=> b.to_s }.each do |key| - yield key, self[key] - end - end - end - Array.class_eval do - alias_method :each_without_sort, :each - def each(&block) - sort {|a,b| a.to_s <=> b.to_s }.each_without_sort &block - end - end - - # generate json - return_value = JSON.pretty_generate(obj) - - # restore hash and array - Hash.class_eval {alias_method :each, :each_without_sort} - Array.class_eval {alias_method :each, :each_without_sort} - - return return_value - end - # # when merging, we raise an error if this method returns true for the two values. # diff --git a/lib/leap_cli/remote/leap_plugin.rb b/lib/leap_cli/remote/leap_plugin.rb index 2c427e9..8cc96d4 100644 --- a/lib/leap_cli/remote/leap_plugin.rb +++ b/lib/leap_cli/remote/leap_plugin.rb @@ -39,6 +39,122 @@ module LeapCli; module Remote; module LeapPlugin run "touch #{INITIALIZED_FILE}" end + # + # This is a hairy ugly hack, exactly the kind of stuff that makes ruby + # dangerous and too much fun for its own good. + # + # In most places, we run remote ssh without a current 'task'. This works fine, + # except that in a few places, the behavior of capistrano ssh is controlled by + # the options of the current task. + # + # We don't want to create an actual current task, because tasks are no fun + # and can't take arguments or return values. So, when we need to configure + # things that can only be configured in a task, we use this handy hack to + # fake the current task. + # + # This is NOT thread safe, but could be made to be so with some extra work. + # + def with_task(name) + task = @config.tasks[name] + @config.class.send(:alias_method, :original_current_task, :current_task) + @config.class.send(:define_method, :current_task, Proc.new(){ task }) + begin + yield + ensure + @config.class.send(:remove_method, :current_task) + @config.class.send(:alias_method, :current_task, :original_current_task) + end + end + + # + # similar to run(cmd, &block), but with: + # + # * exit codes + # * stdout and stderr are combined + # + def stream(cmd, &block) + command = '%s 2>&1; echo "exitcode=$?"' % cmd + run(command) do |channel, stream, data| + exitcode = nil + if data =~ /exitcode=(\d+)\n/ + exitcode = $1.to_i + data.sub!(/exitcode=(\d+)\n/,'') + end + yield({:host => channel[:host], :data => data, :exitcode => exitcode}) + end + end + + # + # like stream, but capture all the output before returning + # + def capture(cmd, &block) + command = '%s 2>&1; echo "exitcode=$?" 2>&1;' % cmd + host_data = {} + run(command) do |channel, stream, data| + host_data[channel[:host]] ||= "" + if data =~ /exitcode=(\d+)\n/ + exitcode = $1.to_i + data.sub!(/exitcode=(\d+)\n/,'') + host_data[channel[:host]] += data + yield({:host => channel[:host], :data => host_data[channel[:host]], :exitcode => exitcode}) + else + host_data[channel[:host]] += data + end + end + end + + # + # Run a command, with a nice status report and progress indicator. + # Only successful results are returned, errors are printed. + # + # For each successful run on each host, block is yielded with a hash like so: + # + # {:host => 'bluejay', :exitcode => 0, :data => 'shell output'} + # + def run_with_progress(cmd, &block) + ssh_failures = [] + exitcode_failures = [] + succeeded = [] + task = LeapCli.log_level > 1 ? :standard_task : :skip_errors_task + with_task(task) do + log :querying, 'facts' do + progress " " + call_on_failure do |host| + ssh_failures << host + progress 'F' + end + capture(cmd) do |response| + if response[:exitcode] == 0 + progress '.' + yield response + else + exitcode_failures << response + progress 'F' + end + end + end + end + puts "done" + if ssh_failures.any? + log :failed, 'to connect to nodes: ' + ssh_failures.join(' ') + end + if exitcode_failures.any? + log :failed, 'to run successfully:' do + exitcode_failures.each do |response| + log "[%s] exit %s - %s" % [response[:host], response[:exitcode], response[:data].strip] + end + end + end + rescue Capistrano::RemoteError => err + log :error, err.to_s + end + + private + + def progress(str='.') + $stdout.print str; $stdout.flush; + end + #def mkdir(dir) # run "mkdir -p #{dir}" #end diff --git a/lib/leap_cli/remote/tasks.rb b/lib/leap_cli/remote/tasks.rb index f967db1..0721c34 100644 --- a/lib/leap_cli/remote/tasks.rb +++ b/lib/leap_cli/remote/tasks.rb @@ -25,9 +25,12 @@ task :install_prerequisites, :max_hosts => MAX_HOSTS do leap.mark_initialized end -#task :apply_puppet, :max_hosts => MAX_HOSTS do -# raise "now such directory #{puppet_source}" unless File.directory?(puppet_source) -# leap.log :applying, "puppet" do -# puppet.apply -# end -#end +# +# just dummies, used to capture task options +# + +task :skip_errors_task, :on_error => :continue, :max_hosts => MAX_HOSTS do +end + +task :standard_task, :max_hosts => MAX_HOSTS do +end \ No newline at end of file diff --git a/lib/leap_cli/util.rb b/lib/leap_cli/util.rb index b2a1dcf..116c212 100644 --- a/lib/leap_cli/util.rb +++ b/lib/leap_cli/util.rb @@ -203,6 +203,40 @@ module LeapCli end end + # + # replace contents of a file, with an exclusive lock. + # + # 1. locks file + # 2. reads contents + # 3. yields contents + # 4. replaces file with return value of the block + # + def replace_file!(filepath, &block) + filepath = Path.named_path(filepath) + if !File.exists?(filepath) + content = yield(nil) + unless content.nil? + write_file!(filepath, content) + log :created, filepath + end + else + File.open(filepath, File::RDWR|File::CREAT, 0644) do |f| + f.flock(File::LOCK_EX) + old_content = f.read + new_content = yield(old_content) + if old_content == new_content + log :nochange, filepath, 2 + else + f.rewind + f.write(new_content) + f.flush + f.truncate(f.pos) + log :updated, filepath + end + end + end + end + def remove_file!(filepath) filepath = Path.named_path(filepath) if File.exists?(filepath) diff --git a/lib/leap_cli/util/remote_command.rb b/lib/leap_cli/util/remote_command.rb index 57234eb..db02037 100644 --- a/lib/leap_cli/util/remote_command.rb +++ b/lib/leap_cli/util/remote_command.rb @@ -73,6 +73,7 @@ module LeapCli; module Util; module RemoteCommand @capistrano_enabled ||= begin require 'capistrano' require 'capistrano/cli' + require 'lib_ext/capistrano_connections' require 'leap_cli/remote/leap_plugin' require 'leap_cli/remote/puppet_plugin' require 'leap_cli/remote/rsync_plugin' diff --git a/lib/leap_cli/version.rb b/lib/leap_cli/version.rb index 45c5df2..bbec03a 100644 --- a/lib/leap_cli/version.rb +++ b/lib/leap_cli/version.rb @@ -1,6 +1,6 @@ module LeapCli unless defined?(LeapCli::VERSION) - VERSION = '1.0.0' + VERSION = '1.1.0' SUMMARY = 'Command line interface to the LEAP platform' DESCRIPTION = 'The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home.' LOAD_PATHS = ['lib', 'vendor/certificate_authority/lib', 'vendor/rsync_command/lib'] diff --git a/lib/lib_ext/capistrano_connections.rb b/lib/lib_ext/capistrano_connections.rb new file mode 100644 index 0000000..c46455f --- /dev/null +++ b/lib/lib_ext/capistrano_connections.rb @@ -0,0 +1,16 @@ +module Capistrano + class Configuration + module Connections + def failed!(server) + @failure_callback.call(server) if @failure_callback + Thread.current[:failed_sessions] << server + end + + def call_on_failure(&block) + @failure_callback = block + end + end + end +end + + diff --git a/test/provider/files/service-definitions/provider.json.erb b/test/provider/files/service-definitions/provider.json.erb index 742b88f..96953c5 100644 --- a/test/provider/files/service-definitions/provider.json.erb +++ b/test/provider/files/service-definitions/provider.json.erb @@ -35,5 +35,5 @@ # "ca_cert_uri": "https://springbok/ca.crt" # } - generate_json hsh + JSON.sorted_generate hsh %> \ No newline at end of file -- cgit v1.2.3