diff options
| -rw-r--r-- | vendor/rsync_command/.gitignore | 17 | ||||
| -rw-r--r-- | vendor/rsync_command/LICENSE.txt | 22 | ||||
| -rw-r--r-- | vendor/rsync_command/README.md | 25 | ||||
| -rw-r--r-- | vendor/rsync_command/Rakefile | 6 | ||||
| -rw-r--r-- | vendor/rsync_command/lib/rsync_command.rb | 96 | ||||
| -rw-r--r-- | vendor/rsync_command/lib/rsync_command/ssh_options.rb | 159 | ||||
| -rw-r--r-- | vendor/rsync_command/lib/rsync_command/thread_pool.rb | 36 | ||||
| -rw-r--r-- | vendor/rsync_command/lib/rsync_command/version.rb | 3 | ||||
| -rw-r--r-- | vendor/rsync_command/rsync_command.gemspec | 23 | ||||
| -rw-r--r-- | vendor/rsync_command/test/rsync_test.rb | 74 | ||||
| -rw-r--r-- | vendor/rsync_command/test/ssh_options_test.rb | 61 |
11 files changed, 522 insertions, 0 deletions
diff --git a/vendor/rsync_command/.gitignore b/vendor/rsync_command/.gitignore new file mode 100644 index 0000000..d87d4be --- /dev/null +++ b/vendor/rsync_command/.gitignore @@ -0,0 +1,17 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp diff --git a/vendor/rsync_command/LICENSE.txt b/vendor/rsync_command/LICENSE.txt new file mode 100644 index 0000000..f94ecdd --- /dev/null +++ b/vendor/rsync_command/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2013 LEAP Encryption Access Project + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/rsync_command/README.md b/vendor/rsync_command/README.md new file mode 100644 index 0000000..4b53a5c --- /dev/null +++ b/vendor/rsync_command/README.md @@ -0,0 +1,25 @@ +RsyncCommand +================================== + +The gem rsync_command provides a library wrapper around the rsync command line program, with additional support for parallel execution of rsync and configuration of OpenSSH options in the format understood by Capistrano (and Net::SSH). + +Installation +------------------------------------ + + gem install rsync_command + +Usage +------------------------------------ + + rsync = RsyncCommand.new(:logger => logger, :ssh => {:auth_methods => 'publickey'}, :flags => '-a') + source = '/source/path' + servers = ['red', 'green', 'blue'] + + rsync.asynchronously(servers) do |server| + dest = {:user => 'root', :host => server, :path => '/dest/path'} + rsync.exec(source, dest) + end + + if rsync.failed? + puts rsync.failures.join + end
\ No newline at end of file diff --git a/vendor/rsync_command/Rakefile b/vendor/rsync_command/Rakefile new file mode 100644 index 0000000..03bba68 --- /dev/null +++ b/vendor/rsync_command/Rakefile @@ -0,0 +1,6 @@ +require "rake/testtask" + +Rake::TestTask.new do |t| + t.pattern = "test/*_test.rb" +end +task :default => :test diff --git a/vendor/rsync_command/lib/rsync_command.rb b/vendor/rsync_command/lib/rsync_command.rb new file mode 100644 index 0000000..39e5945 --- /dev/null +++ b/vendor/rsync_command/lib/rsync_command.rb @@ -0,0 +1,96 @@ +require "rsync_command/version" +require "rsync_command/ssh_options" +require "rsync_command/thread_pool" + +require 'monitor' + +class RsyncCommand + attr_accessor :failures, :logger + + def initialize(options={}) + @options = options.dup + @logger = @options.delete(:logger) + @flags = @options.delete(:flags) + @failures = [] + @failures.extend(MonitorMixin) + end + + # + # takes an Enumerable and iterates each item in the list in parallel. + # + def asynchronously(array, &block) + pool = ThreadPool.new + array.each do |item| + pool.schedule(item, &block) + end + pool.shutdown + end + + # + # runs rsync, recording failures + # + def exec(src, dest, options={}) + @failures.synchronize do + @failures.clear + end + rsync_cmd = command(src, dest, options) + if options[:chdir] + rsync_cmd = "cd '#{options[:chdir]}'; #{rsync_cmd}" + end + @logger.debug rsync_cmd if @logger + ok = system(rsync_cmd) + unless ok + @failures.synchronize do + @failures << {:source => src, :dest => dest, :options => options.dup} + end + end + end + + # + # returns true if last exec returned a failure + # + def failed? + @failures && @failures.any? + end + + # + # build rsync command + # + def command(src, dest, options={}) + src = remote_address(src) + dest = remote_address(dest) + options = @options.merge(options) + flags = [] + flags << @flags if @flags + flags << options[:flags] if options.has_key?(:flags) + flags << '--delete' if options[:delete] + flags << includes(options[:includes]) if options.has_key?(:includes) + flags << excludes(options[:excludes]) if options.has_key?(:excludes) + flags << SshOptions.new(options[:ssh]).to_flags if options.has_key?(:ssh) + "rsync #{flags.compact.join(' ')} #{src} #{dest}" + end + + private + + # + # Creates an rsync location if the +address+ is a hash with keys :user, :host, and :path + # (each component is optional). If +address+ is a string, we just pass it through. + # + def remote_address(address) + if address.is_a? String + address # assume it is already formatted. + elsif address.is_a? Hash + [[address[:user], address[:host]].compact.join('@'), address[:path]].compact.join(':') + end + end + + def excludes(patterns) + [patterns].flatten.compact.map { |p| "--exclude='#{p}'" } + end + + def includes(patterns) + [patterns].flatten.compact.map { |p| "--include='#{p}'" } + end + +end + diff --git a/vendor/rsync_command/lib/rsync_command/ssh_options.rb b/vendor/rsync_command/lib/rsync_command/ssh_options.rb new file mode 100644 index 0000000..494ec9d --- /dev/null +++ b/vendor/rsync_command/lib/rsync_command/ssh_options.rb @@ -0,0 +1,159 @@ +# +# Converts capistrano-style ssh configuration (which uses Net::SSH) into a OpenSSH command line flags suitable for rsync. +# +# For a list of the options normally support by Net::SSH (and thus Capistrano), see +# http://net-ssh.github.com/net-ssh/classes/Net/SSH.html#method-c-start +# +# Also, to see how Net::SSH does the opposite of the conversion we are doing here, check out: +# https://github.com/net-ssh/net-ssh/blob/master/lib/net/ssh/config.rb +# +# API mismatch: +# +# * many OpenSSH options not supported +# * some options only make sense for Net::SSH +# * compression: for Net::SSH, this option is supposed to accept true, false, or algorithm. OpenSSH accepts 'yes' or 'no' +# +class RsyncCommand + class SshOptions + + def initialize(options={}) + @options = parse_options(options) + end + + def to_flags + if @options.empty? + nil + else + %[-e "ssh #{@options.join(' ')}"] + end + end + + private + + def parse_options(options) + options.map do |key, value| + next unless value + # Convert Net::SSH options into OpenSSH options. + case key + when :auth_methods then opt_auth_methods(value) + when :bind_address then opt('BindAddress', value) + when :compression then opt('Compression', value ? 'yes' : 'no') + when :compression_level then opt('CompressionLevel', value.to_i) + when :config then "-F '#{value}'" + when :encryption then opt('Ciphers', [value].flatten.join(',')) + when :forward_agent then opt('ForwardAgent', value) + when :global_known_hosts_file then opt('GlobalKnownHostsFile', value) + when :hmac then opt('MACs', [value].flatten.join(',')) + when :host_key then opt('HostKeyAlgorithms', [value].flatten.join(',')) + when :host_key_alias then opt('HostKeyAlias', value) + when :host_name then opt('HostName', value) + when :kex then opt('KexAlgorithms', [value].flatten.join(',')) + when :key_data then nil # not supported + when :keys then [value].flatten.select { |k| File.exist?(k) }.map { |k| "-i '#{k}'" } + when :keys_only then opt('IdentitiesOnly', value ? 'yes' : 'no') + when :languages then nil # not applicable + when :logger then nil # not applicable + when :paranoid then opt('StrictHostKeyChecking', value ? 'yes' : 'no') + when :passphrase then nil # not supported + when :password then nil # not supported + when :port then "-p #{value.to_i}" + when :properties then nil # not applicable + when :proxy then nil # not applicable + when :rekey_blocks_limit then nil # not supported + when :rekey_limit then opt('RekeyLimit', reverse_interpret_size(value)) + when :rekey_packet_limit then nil # not supported + when :timeout then opt('ConnectTimeout', value.to_i) + when :user then "-l #{value}" + when :user_known_hosts_file then multi_opt('UserKnownHostsFile', value) + when :verbose then opt('LogLevel', interpret_log_level(value)) + end + end.compact + end + + private + + def opt(option_name, option_value) + "-o #{option_name}='#{option_value}'" + end + + def multi_opt(option_name, option_values) + [option_values].flatten.map do |value| + opt(option_name, value) + end.join(' ') + end + + # + # In OpenSSH, password and pubkey default to 'yes', hostbased defaults to 'no'. + # Regardless, if :auth_method is configured, then we explicitly set the auth method. + # + def opt_auth_methods(value) + value = [value].flatten + opts = [] + if value.any? + if value.include? 'password' + opts << opt('PasswordAuthentication', 'yes') + else + opts << opt('PasswordAuthentication', 'no') + end + if value.include? 'publickey' + opts << opt('PubkeyAuthentication', 'yes') + else + opts << opt('PubkeyAuthentication', 'no') + end + if value.include? 'hostbased' + opts << opt('HostbasedAuthentication', 'yes') + else + opts << opt('HostbasedAuthentication', 'no') + end + end + if opts.any? + return opts.join(' ') + else + nil + end + end + + # + # Converts the given integer size in bytes into a string with 'K', 'M', 'G' suffix, as appropriate. + # + # reverse of interpret_size in https://github.com/net-ssh/net-ssh/blob/master/lib/net/ssh/config.rb + # + def reverse_interpret_size(size) + size = size.to_i + if size < 1024 + "#{size}" + elsif size < 1024 * 1024 + "#{size/1024}K" + elsif size < 1024 * 1024 * 1024 + "#{size/(1024*1024)}M" + else + "#{size/(1024*1024*1024)}G" + end + end + + def interpret_log_level(level) + if level.is_a? Symbol + case level + when :debug then "DEBUG" + when :info then "INFO" + when :warn then "ERROR" + when :error then "ERROR" + when :fatal then "FATAL" + else "INFO" + end + elsif level.is_a?(Integer) && defined?(Logger) + case level + when Logger::DEBUG then "DEBUG" + when Logger::INFO then "INFO" + when Logger::WARN then "ERROR" + when Logger::ERROR then "ERROR" + when Logger::FATAL then "FATAL" + else "INFO" + end + else + "INFO" + end + end + + end +end diff --git a/vendor/rsync_command/lib/rsync_command/thread_pool.rb b/vendor/rsync_command/lib/rsync_command/thread_pool.rb new file mode 100644 index 0000000..c788ee2 --- /dev/null +++ b/vendor/rsync_command/lib/rsync_command/thread_pool.rb @@ -0,0 +1,36 @@ +require 'thread' + +class RsyncCommand + class ThreadPool + class << self + attr_accessor :default_size + end + + def initialize(size=nil) + @size = size || ThreadPool.default_size || 10 + @jobs = Queue.new + @retvals = [] + @pool = Array.new(@size) do |i| + Thread.new do + Thread.current[:id] = i + catch(:exit) do + loop do + job, args = @jobs.pop + @retvals << job.call(*args) + end + end + end + end + end + def schedule(*args, &block) + @jobs << [block, args] + end + def shutdown + @size.times do + schedule { throw :exit } + end + @pool.map(&:join) + @retvals + end + end +end diff --git a/vendor/rsync_command/lib/rsync_command/version.rb b/vendor/rsync_command/lib/rsync_command/version.rb new file mode 100644 index 0000000..654a308 --- /dev/null +++ b/vendor/rsync_command/lib/rsync_command/version.rb @@ -0,0 +1,3 @@ +class RsyncCommand + VERSION = "0.0.1" +end diff --git a/vendor/rsync_command/rsync_command.gemspec b/vendor/rsync_command/rsync_command.gemspec new file mode 100644 index 0000000..5690490 --- /dev/null +++ b/vendor/rsync_command/rsync_command.gemspec @@ -0,0 +1,23 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'rsync_command/version' + +Gem::Specification.new do |spec| + spec.name = "rsync_command" + spec.version = RsyncCommand::VERSION + spec.authors = ["elijah"] + spec.email = ["elijah@leap.se"] + spec.description = %q{A library wrapper for the the rsync command.} + spec.summary = %q{Includes support for Net::SSH-like configuration and asynchronous execution.} + spec.homepage = "https://github.com/leapcode/rsync_command" + spec.license = "MIT" + + spec.files = `git ls-files`.split($/) + spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + spec.require_paths = ["lib"] + + spec.add_development_dependency "bundler", "~> 1.3" + spec.add_development_dependency "rake" +end diff --git a/vendor/rsync_command/test/rsync_test.rb b/vendor/rsync_command/test/rsync_test.rb new file mode 100644 index 0000000..7860f73 --- /dev/null +++ b/vendor/rsync_command/test/rsync_test.rb @@ -0,0 +1,74 @@ +require 'test/unit' +require File.expand_path('../../lib/rsync_command', __FILE__) + +if RUBY_VERSION >= '1.9' + SimpleOrderedHash = ::Hash +else + class SimpleOrderedHash < Hash + def each; self.keys.map(&:to_s).sort.each {|key| yield [key.to_sym, self[key.to_sym]]}; end + end +end + +class RsyncTest < Test::Unit::TestCase + + def test_build_simple_command + command = rsync_command('bar', 'foo') + assert_equal 'rsync -az bar foo', command + end + + def test_allows_passing_delete + command = rsync_command('bar', 'foo', :delete => true) + assert_equal 'rsync -az --delete bar foo', command + end + + def test_allows_specifying_an_exclude + command = rsync_command('bar', 'foo', :excludes => '.git') + assert_equal "rsync -az --exclude='.git' bar foo", command + end + + def test_ssh_options_keys_only_lists_existing_files + command = rsync_command('.', 'foo', :ssh => { :keys => [__FILE__, "#{__FILE__}dadijofs"] }) + assert_match /-i '#{__FILE__}'/, command + end + + def test_ssh_options_ignores_keys_if_nil + command = rsync_command('.', 'foo', :ssh => { :keys => nil }) + assert_equal 'rsync -az . foo', command + command = rsync_command('bar', 'foo') + assert_equal 'rsync -az bar foo', command + end + + def test_ssh_options_config_adds_flag + command = rsync_command('.', 'foo', :ssh => { :config => __FILE__ }) + assert_equal %Q[rsync -az -e "ssh -F '#{__FILE__}'" . foo], command + end + + def test_ssh_options_port_adds_port + command = rsync_command('.', 'foo', :ssh => { :port => '30022' }) + assert_equal %Q[rsync -az -e "ssh -p 30022" . foo], command + end + + def test_ssh_options_ignores_config_if_nil_or_false + command = rsync_command('.', 'foo', :ssh => { :config => nil }) + assert_equal 'rsync -az . foo', command + command = rsync_command('.', 'foo', :ssh => { :config => false }) + assert_equal 'rsync -az . foo', command + end + + def test_remote_address + cmd = rsync_command('.', {:user => 'user', :host => 'box.local', :path => '/tmp'}) + assert_equal "rsync -az . user@box.local:/tmp", cmd + end + + #def test_remote_address_drops_at_when_user_is_nil + # assert_equal 'box.local:/tmp', SupplyDrop::Rsync.remote_address(nil, 'box.local', '/tmp') + #end + + protected + + def rsync_command(src, dest, options={}) + rsync = RsyncCommand.new(:flags => '-az') + rsync.command(src, dest, options) + end + +end
\ No newline at end of file diff --git a/vendor/rsync_command/test/ssh_options_test.rb b/vendor/rsync_command/test/ssh_options_test.rb new file mode 100644 index 0000000..1bffa00 --- /dev/null +++ b/vendor/rsync_command/test/ssh_options_test.rb @@ -0,0 +1,61 @@ +require 'test/unit' +require File.expand_path('../../lib/rsync_command', __FILE__) + +class SshOptionsTest < Test::Unit::TestCase + + def test_simple_ssh_options + options = ssh_options(Hash[ + :bind_address, '0.0.0.0', + :compression, true, + :compression_level, 1, + :config, '/etc/ssh/ssh_config', + :global_known_hosts_file, '/etc/ssh/known_hosts', + :host_name, 'myhost', + :keys_only, false, + :paranoid, true, + :port, 2222, + :timeout, 10000, + :user, 'root', + :user_known_hosts_file, '~/.ssh/known_hosts' + ]) + assert_match /-o BindAddress='0.0.0.0'/, options + assert_match /-o Compression='yes'/, options + assert_match %r{-o CompressionLevel='1' -F '/etc/ssh/ssh_config'}, options + assert_match %r{-o GlobalKnownHostsFile='/etc/ssh/known_hosts'}, options + assert_match /-o HostName='myhost'/, options + assert_match /-o StrictHostKeyChecking='yes' -p 2222/, options + assert_match /-o ConnectTimeout='10000' -l root/, options + assert_match %r{-o UserKnownHostsFile='~/.ssh/known_hosts'}, options + end + + def test_complex_ssh_options + options = ssh_options(Hash[ + :auth_methods, 'publickey', + :encryption, ['aes256-cbc', 'aes192-cbc'], + :hmac, 'hmac-sha2-256', + :host_key, 'ecdsa-sha2-nistp256-cert-v01@openssh.com', + :rekey_limit, 2*1024*1024, + :verbose, :debug, + :user_known_hosts_file, ['~/.ssh/known_hosts', '~/.ssh/production_known_hosts'] + ]) + assert_match /PasswordAuthentication='no'/, options + assert_match /PubkeyAuthentication='yes'/, options + assert_match /HostbasedAuthentication='no'/, options + assert_match /-o PasswordAuthentication='no'/, options + assert_match /-o PubkeyAuthentication='yes'/, options + assert_match /-o HostbasedAuthentication='no'/, options + assert_match /-o Ciphers='aes256-cbc,aes192-cbc'/, options + assert_match /-o MACs='hmac-sha2-256'/, options + assert_match /-o HostKeyAlgorithms='ecdsa-sha2-nistp256-cert-v01@openssh.com'/, options + assert_match /-o RekeyLimit='2M'/, options + assert_match %r{-o UserKnownHostsFile='~/.ssh/known_hosts'}, options + assert_match %r{-o UserKnownHostsFile='~/.ssh/production_known_hosts'}, options + assert_match /-o LogLevel='DEBUG'/, options + end + + protected + + def ssh_options(options) + RsyncCommand::SshOptions.new(options).to_flags + end +end |