diff options
67 files changed, 2355 insertions, 1058 deletions
diff --git a/RELEASES.md b/RELEASES.md index f33599f..f8407c0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,14 @@ +Version 1.6.1 + - add environment pinning, see `leap help env` + - support both rsa and ecdsa host keys + - custom puppet modules: drop modules in files/puppet/modules + - all json macros are now moved to the platform + - allow "+key" and "-key" json properties for adding and subtracting + arrays during inheritence + - bugfix: better CSR creation + - bugfix: always sort arrays in exported json. + - bugfix: improved cert updating + Version 1.5.6 - Added try{} macro function that quietly swallows exceptions. @@ -19,7 +30,7 @@ Version 1.5.3 Version 1.5.0 - Added ability to scope provider.json by environment - + Version 1.2.5 - Added initial support for remote tests. @@ -87,33 +87,6 @@ end task :default => :test ## -## CODE GENERATION -## - -desc "Updates the list of required configuration options for this version of LEAP CLI" -task 'update-requirements' do - Dir.chdir($base_dir) do - required_configs = `find -name '*.rb' | xargs grep -R 'assert_config!'`.split("\n").collect{|line| - if line =~ /def/ || line =~ /pre\.rb/ - nil - else - line.sub(/.*assert_config! ["'](.*?)["'].*/,'"\1"') - end - }.compact - File.open("#{$base_dir}/lib/leap_cli/requirements.rb", 'w') do |f| - f.puts "# run 'rake update-requirements' to generate this file." - f.puts "module LeapCli" - f.puts " REQUIREMENTS = [" - f.puts " " + required_configs.join(",\n ") - f.puts " ]" - f.puts "end" - end - puts "updated #{$base_dir}/lib/leap_cli/requirements.rb" - #puts `cat '#{$base_dir}/lib/leap_cli/requirements.rb'` - end -end - -## ## DOCUMENTATION ## @@ -1,7 +1,10 @@ #!/usr/bin/env ruby -if ARGV.include?('--debug') +if ARGV.include?('--debug') || ARGV.include?('-d') + DEBUG=true require 'debugger' +else + DEBUG=false end begin @@ -18,7 +21,6 @@ rescue LoadError # This allows you to run the command directly while developing the gem, and also lets you # run from anywhere (I like to link 'bin/leap' to /usr/local/bin/leap). # - require 'rubygems' base_dir = File.expand_path('..', File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__)) require File.join(base_dir, 'lib','leap_cli','load_paths') require 'leap_cli' @@ -78,9 +80,27 @@ module LeapCli::Commands exit(0) end + # disable GLI error catching + ENV['GLI_DEBUG'] = "true" + def error_message(msg) + end + # load commands and run commands_from('leap_cli/commands') ORIGINAL_ARGV = ARGV.dup - exit_status = run(ARGV) - exit(LeapCli::Util.exit_status || exit_status) + begin + exit_status = run(ARGV) + exit(LeapCli::Util.exit_status || exit_status) + rescue StandardError => exc + if exc.respond_to? :log + exc.log + else + puts + LeapCli.log :error, "%s: %s" % [exc.class, exc.message] + puts + end + if DEBUG + raise exc + end + end end diff --git a/contrib/leap.bash-completion b/contrib/leap.bash-completion new file mode 100644 index 0000000..b33c1b9 --- /dev/null +++ b/contrib/leap.bash-completion @@ -0,0 +1,112 @@ +# LEAP CLI BASH COMPLETION +# +# Usage: Add the following line to your ~/.bashrc file. +# +# source /path/to/leap.bash-completion +# + +# Config +# Where are your leap-cli gems stored? +versions="/var/lib/gems/2.1.0/gems/" + +_leap_complete_nodes () { + nodes="${nodes/}" + suffix=".json" + + local items=($(compgen -f $nodes$cur)) + for item in ${items[@]}; do + item="${item%$suffix}" + COMPREPLY+=("${item#$nodes}") + done +} + +_leap_complete_tags () { + tags="${tags/}" + suffix=".json" + local items=($(compgen plusdirs -f $tags$cur)) + for item in ${items[@]}; do + item="${item%$suffix}" + COMPREPLY+=("${item#$tags}") + done +} + +_leap_global_options() { + COMPREPLY+=($(compgen -W "--color --no-color --debug --help --log= --verbose= -v1 -v2 -v3 -v4 -v5 --version" -- ${cur})) +} + +_leap_complete_versions () { + prefix="leap_cli-" + + local items=($(compgen -d $versions$prefix)) + for item in ${items[@]}; do + item="${item#$versions}" + # COMPREPLY+=("_${item#$prefix}_") + COMPREPLY+=($(compgen -W "_${item#$prefix}_" -- ${cur})) + done +} + +_leap() +{ + COMPREPLY=() + local cur="${COMP_WORDS[COMP_CWORD]}" + local commands="add-user clean deploy help inspect list mosh new ssh cert compile db facts local node test" + + if [[ $COMP_CWORD -gt 1 ]] && [[ "$cur" != -* ]] && [[ "$cur" != _* ]]; then + local lastarg="${COMP_WORDS[$COMP_CWORD-1]}" + + case "${COMP_WORDS[$COMP_CWORD-1]}" in + deploy) + _leap_complete_nodes + _leap_complete_tags + ;; + mosh) + _leap_complete_nodes + ;; + ssh) + _leap_complete_nodes + ;; + cert) + COMPREPLY+=($(compgen -W "ca csr dh update" -- ${cur})) + ;; + compile) + COMPREPLY+=($(compgen -W "all zone" -- ${cur})) + ;; + facts) + COMPREPLY+=($(compgen -W "update" -- ${cur})) + ;; + local) + COMPREPLY+=($(compgen -W "start stop status save" -- ${cur})) + ;; + start|stop|status|save) + _leap_complete_nodes + ;; + node) + COMPREPLY+=($(compgen -W "add init rm mv" -- ${cur})) + ;; + add|rm|mv) + _leap_complete_nodes + ;; + test) + COMPREPLY+=($(compgen -W "init run" -- ${cur})) + ;; + init|run) + _leap_complete_nodes + ;; + *) + COMPREPLY+=($(compgen -W "${commands}" -- ${cur})) + ;; + esac + + else + if [[ "$cur" == -* ]]; then + _leap_global_options + elif [[ "$cur" == _* ]]; then + _leap_complete_versions + else + COMPREPLY+=($(compgen -W "${commands}" -- ${cur})) + fi + + fi +} + +complete -F _leap leap
\ No newline at end of file diff --git a/doc/leap.md b/doc/leap.md index b176541..d735fef 100644 --- a/doc/leap.md +++ b/doc/leap.md @@ -5,69 +5,83 @@ The command "leap" can be used to manage a bevy of servers running the LEAP plat # Global Options -* `--log FILE` -Override default log file -Default Value: None +* `--log FILE` +Override default log file +Default Value: None -* `-v|--verbose LEVEL` -Verbosity level 0..2 -Default Value: 1 +* `-v|--verbose LEVEL` +Verbosity level 0..5 +Default Value: 1 -* `--help` -Show this message +* `--[no-]color` +Disable colors in output -* `--version` -Display version number and exit +* `--debug` +Enable debugging library (leap_cli development only) -* `--yes` -Skip prompts and assume "yes" +* `--help` +Show this message + +* `--version` +Display version number and exit + +* `--yes` +Skip prompts and assume "yes" # leap add-user USERNAME -Adds a new trusted sysadmin +Adds a new trusted sysadmin by adding public keys to the "users" directory. **Options** -* `--pgp-pub-key arg` -OpenPGP public key file for this new user -Default Value: None +* `--pgp-pub-key arg` +OpenPGP public key file for this new user +Default Value: None -* `--ssh-pub-key arg` -SSH public key file for this new user -Default Value: None +* `--ssh-pub-key arg` +SSH public key file for this new user +Default Value: None -* `--self` -lets you choose among your public keys +* `--self` +Add yourself as a trusted sysadin by choosing among the public keys available for the current user. -# leap cert +# leap cert Manage X.509 certificates -## leap cert ca +## leap cert ca Creates two Certificate Authorities (one for validating servers and one for validating clients). See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`. -## leap cert csr +## leap cert csr Creates a CSR for use in buying a commercial X.509 certificate. -The CSR created is for the for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. + +**Options** + +* `--domain DOMAIN` +Specify what domain to create the CSR for. +Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`. +Default Value: None -## leap cert dh + +## leap cert dh Creates a Diffie-Hellman parameter file. -## leap cert update <node-filter> +## leap cert update FILTER Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed. @@ -75,22 +89,47 @@ This command will a generate new certificate for a node if some value in the nod **Options** -* `--force` -Always generate new certificates +* `--force` +Always generate new certificates -# leap clean +# leap clean Removes all files generated with the "compile" command. -# leap compile +# leap compile + +Compile generated files. + + + +## leap compile all [ENVIRONMENT] Compiles node configuration files into hiera files used for deployment. +## leap compile zone + +Compile a DNS zone file for your provider. + + +Default Command: all + +# leap db + +Database commands. + + + +## leap db destroy [FILTER] + +Destroy all the databases. If present, limit to FILTER nodes. + + + # leap deploy FILTER Apply recipes to a node or set of nodes. @@ -99,13 +138,67 @@ The FILTER can be the name of a node, service, or tag. **Options** -* `--tags TAG[,TAG]` -Specify tags to pass through to puppet (overriding the default). -Default Value: leap_base,leap_service +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--tags TAG[,TAG]` +Specify tags to pass through to puppet (overriding the default). +Default Value: leap_base,leap_service + +* `--dev` +Development mode: don't run 'git submodule update' before deploy. + +* `--fast` +Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. + +* `--force` +Deploy even if there is a lockfile. + +* `--[no-]sync` +Sync files, but don't actually apply recipes. + + +# leap env + +Manipulate and query environment information. + +The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. A node in one environment will never interact with a node from another environment. Environment pinning works by modifying your ~/.leaprc file and is dependent on the absolute file path of your provider directory (pins don't apply if you move the directory) + +## leap env ls + +List the available environments. The pinned environment, if any, will be marked with '*'. + -* `--fast` -Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy. +## leap env pin ENVIRONMENT + +Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment. + + + +## leap env unpin + +Unpin the environment. All subsequent commands will apply to all nodes. + + +Default Command: ls + +# leap facts + +Gather information on nodes. + + + +## leap facts update FILTER + +Query servers to update facts.json. + +Queries every node included in FILTER and saves the important information to facts.json # leap help command @@ -115,8 +208,8 @@ Gets help for the application or its commands. Can also list the commands in a w **Options** -* `-c` -List commands one per line, to assist with shell completion +* `-c` +List commands one per line, to assist with shell completion # leap inspect FILE @@ -125,6 +218,12 @@ Prints details about a file. Alternately, the argument FILE can be the name of a +**Options** + +* `--base` +Inspect the FILE from the provider_base (i.e. without local inheritance). + + # leap list [FILTER] List nodes and their classifications @@ -137,12 +236,15 @@ Prints out a listing of nodes, services, or tags. If present, the FILTER can be **Options** -* `--print arg` -What attributes to print (optional) -Default Value: None +* `--print arg` +What attributes to print (optional) +Default Value: None + +* `--disabled` +Include disabled nodes in the list. -# leap local +# leap local Manage local virtual machines. @@ -184,6 +286,12 @@ Shuts down the virtual machine(s) +# leap mosh NAME + +Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true). + + + # leap new DIRECTORY Creates a new provider instance in the specified directory, creating it if necessary. @@ -192,24 +300,24 @@ Creates a new provider instance in the specified directory, creating it if neces **Options** -* `--contacts arg` -Default email address contacts. -Default Value: None +* `--contacts arg` +Default email address contacts. +Default Value: None -* `--domain arg` -The primary domain of the provider. -Default Value: None +* `--domain arg` +The primary domain of the provider. +Default Value: None -* `--name arg` -The name of the provider. -Default Value: None +* `--name arg` +The name of the provider. +Default Value: None -* `--platform arg` -File path of the leap_platform directory. -Default Value: None +* `--platform arg` +File path of the leap_platform directory. +Default Value: None -# leap node +# leap node Node management @@ -231,20 +339,28 @@ Separeate multiple values for a single property with a comma, like so: `leap nod **Options** -* `--local` -Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer. +* `--local` +Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer. ## leap node init FILTER Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages -This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, and installing packages that are required for deploying. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times. +This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times. **Options** -* `--echo` -If set, passwords are visible as you type them (default is hidden) +* `--ip IPADDRESS` +Override the default SSH IP address. +Default Value: None + +* `--port PORT` +Override the default SSH port. +Default Value: None + +* `--echo` +If set, passwords are visible as you type them (default is hidden) ## leap node mv OLD_NAME NEW_NAME @@ -265,21 +381,38 @@ Log in to the specified node with an interactive shell. -# leap test +**Options** + +* `--port arg` +Override ssh port for remote host +Default Value: None + +* `--ssh arg` +Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig') +Default Value: None + + +# leap test Run tests. -## leap test init +## leap test init Creates files needed to run tests. -## leap test run +## leap test run Run tests. + +**Options** + +* `--[no-]continue` +Continue over errors and failures (default is --no-continue). + Default Command: run diff --git a/leap_cli.gemspec b/leap_cli.gemspec index d0b9a99..10ed371 100644 --- a/leap_cli.gemspec +++ b/leap_cli.gemspec @@ -56,14 +56,14 @@ spec = Gem::Specification.new do |s| s.add_runtime_dependency('tee') # network gems + s.add_runtime_dependency('net-ssh', '~> 2.7.0') + # ^^ we can upgrade once we get off broken capistrano + # https://github.com/net-ssh/net-ssh/issues/145 s.add_runtime_dependency('capistrano', '~> 2.15.5') - #s.add_runtime_dependency('supply_drop') - # ^^ currently vendored # crypto gems #s.add_runtime_dependency('certificate_authority', '>= 0.2.0') # ^^ currently vendored - s.add_runtime_dependency('net-ssh') s.add_runtime_dependency('gpgme') # not essential, but used for some minor stuff in adding sysadmins # misc gems diff --git a/lib/core_ext/yaml.rb b/lib/core_ext/yaml.rb new file mode 100644 index 0000000..bb0b5c9 --- /dev/null +++ b/lib/core_ext/yaml.rb @@ -0,0 +1,29 @@ +class Object + # + # ya2yaml will output hash keys in sorted order, but it outputs arrays + # in natural order. This new method, sorted_ya2yaml(), is the same as + # ya2yaml but ensures that arrays are sorted. + # + # This is important so that the .yaml files don't change each time you recompile. + # + # see https://github.com/afunai/ya2yaml/blob/master/lib/ya2yaml.rb + # + def sorted_ya2yaml(options = {}) + # modify array + Array.class_eval do + alias_method :collect_without_sort, :collect + def collect(&block) + sorted = sort {|a,b| a.to_s <=> b.to_s} + sorted.collect_without_sort(&block) + end + end + + # generate yaml + yaml_str = self.ya2yaml(options) + + # restore array + Array.class_eval {alias_method :collect, :collect_without_sort} + + return yaml_str + end +end diff --git a/lib/leap/platform.rb b/lib/leap/platform.rb index d97650b..6938fb3 100644 --- a/lib/leap/platform.rb +++ b/lib/leap/platform.rb @@ -16,10 +16,25 @@ module Leap attr_accessor :monitor_username attr_accessor :reserved_usernames + attr_accessor :hiera_path + attr_accessor :files_dir + attr_accessor :leap_dir + attr_accessor :init_path + + attr_accessor :default_puppet_tags + def define(&block) - # some sanity defaults: + # some defaults: @reserved_usernames = [] + @hiera_path = '/etc/leap/hiera.yaml' + @leap_dir = '/srv/leap' + @files_dir = '/srv/leap/files' + @init_path = '/srv/leap/initialized' + @default_puppet_tags = [] + self.instance_eval(&block) + + @version ||= Versionomy.parse("0.0") end def version=(version) @@ -44,10 +59,22 @@ module Leap # return true if the platform version is within the specified range. # def version_in_range?(range) + if range.is_a? String + range = range.split('..') + end minimum_platform_version = Versionomy.parse(range.first) maximum_platform_version = Versionomy.parse(range.last) @version >= minimum_platform_version && @version <= maximum_platform_version end + + def major_version + if @version.major == 0 + "#{@version.major}.#{@version.minor}" + else + @version.major + end + end + end end diff --git a/lib/leap_cli.rb b/lib/leap_cli.rb index 70727b7..2ca68c4 100644 --- a/lib/leap_cli.rb +++ b/lib/leap_cli.rb @@ -1,20 +1,28 @@ -module LeapCli; end +module LeapCli + module Commands; end # for commands in leap_cli/commands + module Macro; end # for macros in leap_platform/provider_base/lib/macros +end $ruby_version = RUBY_VERSION.split('.').collect{ |i| i.to_i }.extend(Comparable) -require 'leap/platform.rb' +# ensure leap_cli/lib/overrides has the highest priority +$:.unshift(File.expand_path('../override',__FILE__)) + +require 'rubygems' +gem 'net-ssh', '~> 2.7.0' -require 'leap_cli/version.rb' -require 'leap_cli/constants.rb' -require 'leap_cli/requirements.rb' -require 'leap_cli/exceptions.rb' +require 'leap/platform' -require 'leap_cli/leapfile.rb' +require 'leap_cli/version' +require 'leap_cli/exceptions' + +require 'leap_cli/leapfile' require 'core_ext/hash' require 'core_ext/boolean' require 'core_ext/nil' require 'core_ext/string' require 'core_ext/json' +require 'core_ext/yaml' require 'leap_cli/log' require 'leap_cli/path' @@ -31,12 +39,11 @@ require 'leap_cli/config/tag' require 'leap_cli/config/provider' require 'leap_cli/config/secrets' require 'leap_cli/config/object_list' +require 'leap_cli/config/filter' require 'leap_cli/config/manager' require 'leap_cli/markdown_document_listener' -module LeapCli::Commands; end - # # allow everyone easy access to log() command. # diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/ca.rb index 46e3494..579e305 100644 --- a/lib/leap_cli/commands/ca.rb +++ b/lib/leap_cli/commands/ca.rb @@ -1,6 +1,6 @@ -require 'openssl' -require 'certificate_authority' -require 'date' +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' +autoload :Date, 'date' require 'digest/md5' module LeapCli; module Commands @@ -36,6 +36,7 @@ module LeapCli; module Commands nodes = manager.filter!(args) nodes.each_node do |node| + warn_if_commercial_cert_will_soon_expire(node) if !node.x509.use remove_file!([:node_x509_key, node.name]) remove_file!([:node_x509_cert, node.name]) @@ -81,9 +82,19 @@ module LeapCli; module Commands # http://www.redkestrel.co.uk/Articles/CSR.html # cert.desc "Creates a CSR for use in buying a commercial X.509 certificate." - cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`." + cert.long_desc "Unless specified, the CSR is created for the provider's primary domain. "+ + "The properties used for this CSR come from `provider.ca.server_certificates`, "+ + "but may be overridden here." cert.command :csr do |csr| csr.flag 'domain', :arg_name => 'DOMAIN', :desc => 'Specify what domain to create the CSR for.' + csr.flag ['organization', 'O'], :arg_name => 'ORGANIZATION', :desc => "Override default O in distinguished name." + csr.flag ['unit', 'OU'], :arg_name => 'UNIT', :desc => "Set OU in distinguished name." + csr.flag 'email', :arg_name => 'EMAIL', :desc => "Set emailAddress in distinguished name." + csr.flag ['locality', 'L'], :arg_name => 'LOCALITY', :desc => "Set L in distinguished name." + csr.flag ['state', 'ST'], :arg_name => 'STATE', :desc => "Set ST in distinguished name." + csr.flag ['country', 'C'], :arg_name => 'COUNTRY', :desc => "Set C in distinguished name." + csr.flag :bits, :arg_name => 'BITS', :desc => "Override default certificate bit length" + csr.flag :digest, :arg_name => 'DIGEST', :desc => "Override default signature digest" csr.action do |global_options,options,args| assert_config! 'provider.domain' assert_config! 'provider.name' @@ -97,24 +108,25 @@ module LeapCli; module Commands # RSA key keypair = CertificateAuthority::MemoryKeyMaterial.new - log :generating, "%s bit RSA key" % server_certificates.bit_size do - keypair.generate_key(server_certificates.bit_size) + bit_size = (options[:bits] || server_certificates.bit_size).to_i + log :generating, "%s bit RSA key" % bit_size do + keypair.generate_key(bit_size) write_file! [:commercial_key, domain], keypair.private_key.to_pem end # CSR dn = CertificateAuthority::DistinguishedName.new - csr = CertificateAuthority::SigningRequest.new - dn.common_name = domain - dn.organization = provider.name[provider.default_language] - dn.country = server_certificates['country'] # optional - dn.state = server_certificates['state'] # optional - dn.locality = server_certificates['locality'] # optional - - log :generating, "CSR with commonName => '%s', organization => '%s'" % [dn.common_name, dn.organization] do - csr.distinguished_name = dn - csr.key_material = keypair - csr.digest = server_certificates.digest + dn.common_name = domain + dn.organization = options[:organization] || provider.name[provider.default_language] + dn.ou = options[:organizational_unit] # optional + dn.email_address = options[:email] # optional + dn.country = options[:country] || server_certificates['country'] # optional + dn.state = options[:state] || server_certificates['state'] # optional + dn.locality = options[:locality] || server_certificates['locality'] # optional + + digest = options[:digest] || server_certificates.digest + log :generating, "CSR with #{digest} digest and #{print_dn(dn)}" do + csr = create_csr(dn, keypair, digest) request = csr.to_x509_csr write_file! [:commercial_csr, domain], csr.to_pem end @@ -191,7 +203,7 @@ module LeapCli; module Commands return true else cert = load_certificate_file([:node_x509_cert, node.name]) - if cert.not_after < months_from_yesterday(1) + if cert.not_after < months_from_yesterday(2) log :updating, "cert for node '#{node.name}' because it will expire soon" return true end @@ -222,6 +234,18 @@ module LeapCli; module Commands return false end + def warn_if_commercial_cert_will_soon_expire(node) + dns_names_for_node(node).each do |domain| + if file_exists?([:commercial_cert, domain]) + cert = load_certificate_file([:commercial_cert, domain]) + if cert.not_after < months_from_yesterday(2) + log :warning, "the commercial certificate '#{Path.relative_path([:commercial_cert, domain])}' will expire soon. "+ + "You should renew it with `leap cert csr --domain #{domain}`." + end + end + end + end + def generate_cert_for_node(node) return if node.x509.use == false @@ -262,6 +286,43 @@ module LeapCli; module Commands yield cert.key_material.private_key.to_pem, cert.to_pem end + # + # creates a CSR and returns it. + # with the correct extReq attribute so that the CA + # doens't generate certs with extensions we don't want. + # + def create_csr(dn, keypair, digest) + csr = CertificateAuthority::SigningRequest.new + csr.distinguished_name = dn + csr.key_material = keypair + csr.digest = digest + + # define extensions manually (library doesn't support setting these on CSRs) + extensions = [] + extensions << CertificateAuthority::Extensions::BasicConstraints.new.tap {|basic| + basic.ca = false + } + extensions << CertificateAuthority::Extensions::KeyUsage.new.tap {|keyusage| + keyusage.usage = ["digitalSignature", "keyEncipherment"] + } + extensions << CertificateAuthority::Extensions::ExtendedKeyUsage.new.tap {|extkeyusage| + extkeyusage.usage = [ "serverAuth"] + } + + # convert extensions to attribute 'extReq' + # aka "Requested Extensions" + factory = OpenSSL::X509::ExtensionFactory.new + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence( + extensions.map{|e| factory.create_ext(e.openssl_identifier, e.to_s, e.critical)} + )]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + ] + csr.attributes = attrs + + return csr + end + def ca_root @ca_root ||= begin load_certificate_file(:ca_cert, :ca_key) @@ -406,6 +467,15 @@ module LeapCli; module Commands cert_serial_number(domain_name).to_s(36) end + # prints CertificateAuthority::DistinguishedName fields + def print_dn(dn) + fields = {} + [:common_name, :locality, :state, :country, :organization, :organizational_unit, :email_address].each do |attr| + fields[attr] = dn.send(attr) if dn.send(attr) + end + fields.inspect + end + ## ## TIME HELPERS ## diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb index 63c2047..644ce2a 100644 --- a/lib/leap_cli/commands/compile.rb +++ b/lib/leap_cli/commands/compile.rb @@ -5,9 +5,18 @@ module LeapCli desc "Compile generated files." command :compile do |c| c.desc 'Compiles node configuration files into hiera files used for deployment.' + c.arg_name 'ENVIRONMENT', :optional => true c.command :all do |all| all.action do |global_options,options,args| - compile_hiera_files + environment = args.first + if !LeapCli.leapfile.environment.nil? && !environment.nil? && environment != LeapCli.leapfile.environment + bail! "You cannot specify an ENVIRONMENT argument while the environment is pinned." + end + if environment && manager.environment_names.include?(environment) + compile_hiera_files(manager.filter([environment])) + else + compile_hiera_files(manager.filter) + end end end diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb index 814407f..6589837 100644 --- a/lib/leap_cli/commands/deploy.rb +++ b/lib/leap_cli/commands/deploy.rb @@ -17,9 +17,12 @@ module LeapCli # --force c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false + # --dev + c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false + # --tags c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).', - :default_value => DEFAULT_TAGS.join(','), :arg_name => 'TAG[,TAG]' + :arg_name => 'TAG[,TAG]' c.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT' @@ -28,9 +31,12 @@ module LeapCli :arg_name => 'IPADDRESS' c.action do |global,options,args| - init_submodules - nodes = filter_deploy_nodes(args) + if options[:dev] != true + init_submodules + end + + nodes = manager.filter!(args) if nodes.size > 1 say "Deploying to these nodes: #{nodes.keys.join(', ')}" if !global[:yes] && !agree("Continue? ") @@ -38,7 +44,16 @@ module LeapCli end end - compile_hiera_files + environments = nodes.field('environment').uniq + if environments.empty? + environments = [nil] + end + environments.each do |env| + check_platform_pinning(env) + end + # compile hiera files for all the nodes in every environment that is + # being deployed and only those environments. + compile_hiera_files(manager.filter(environments)) ssh_connect(nodes, connect_options(options)) do |ssh| ssh.leap.log :checking, 'node' do @@ -58,39 +73,128 @@ module LeapCli end end end + end end private + # + # The currently activated provider.json could have loaded some pinning + # information for the platform. If this is the case, refuse to deploy + # if there is a mismatch. + # + # For example: + # + # "platform": { + # "branch": "develop" + # "version": "1.0..99" + # "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD" + # } + # + def check_platform_pinning(environment) + provider = manager.env(environment).provider + return unless provider['platform'] + + if environment.nil? || environment == 'default' + provider_json = 'provider.json' + else + provider_json = 'provider.' + environment + '.json' + end + + # can we have json schema verification already? + unless provider.platform.is_a? Hash + bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect) + end + + # check version + if provider.platform['version'] + if !Leap::Platform.version_in_range?(provider.platform.version) + say("The platform is pinned to a version range of '#{provider.platform.version}' "+ + "by the `platform.version` property in #{provider_json}, but the platform "+ + "(#{Path.platform}) has version #{Leap::Platform.version}.") + quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong version? ") + end + end + + # check branch + if provider.platform['branch'] + if !is_git_directory?(Path.platform) + say("The platform is pinned to a particular branch by the `platform.branch` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.") + quit!("OK. Bye.") unless agree("Do you really want to deploy anyway? ") + end + unless provider.platform.branch == current_git_branch(Path.platform) + say("The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+ + "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " + + "(for directory '#{Path.platform}')") + quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong branch? ") + end + end + + # check commit + if provider.platform['commit'] + if !is_git_directory?(Path.platform) + say("The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.") + quit!("OK. Bye.") unless agree("Do you really want to deploy anyway? ") + end + current_commit = current_git_commit(Path.platform) + Dir.chdir(Path.platform) do + commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'", + "The platform is pinned to a particular commit range by the `platform.commit` property "+ + "in #{provider_json}, but git was not able to find commits in the range specified "+ + "(#{provider.platform.commit}).") + commit_range = commit_range.split("\n") + if !commit_range.include?(current_commit) && + provider.platform.commit.split('..').first != current_commit + say("The platform is pinned via the `platform.commit` property in #{provider_json} " + + "to a commit in the range #{provider.platform.commit}, but the current HEAD " + + "(#{current_commit}) is not in that range.") + quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong commit? ") + end + end + end + end + def sync_hiera_config(ssh) - dest_dir = provider.hiera_sync_destination ssh.rsync.update do |server| node = manager.node(server.host) hiera_file = Path.relative_path([:hiera, node.name]) - ssh.leap.log hiera_file + ' -> ' + node.name + ':' + dest_dir + '/hiera.yaml' + ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path { :source => hiera_file, - :dest => dest_dir + '/hiera.yaml', + :dest => Leap::Platform.hiera_path, :flags => "-rltp --chmod=u+rX,go-rwx" } end end + # + # sync various support files. + # def sync_support_files(ssh) - dest_dir = provider.hiera_sync_destination + dest_dir = Leap::Platform.files_dir + source_files = [] + if Path.defined?(:custom_puppet_dir) && file_exists?(:custom_puppet_dir) + source_files += [:custom_puppet_dir, :custom_puppet_modules_dir, :custom_puppet_manifests_dir].collect{|path| + Path.relative_path(path, Path.provider) + '/' # rsync needs trailing slash + } + ensure_dir :custom_puppet_modules_dir + end ssh.rsync.update do |server| node = manager.node(server.host) files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) } + files_to_sync += source_files if files_to_sync.any? ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir) { - :chdir => Path.provider, + :chdir => Path.named_path(:files_dir), :source => ".", :dest => dest_dir, :excludes => "*", - :includes => calculate_includes_from_files(files_to_sync), - :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --filter='protect hiera.yaml' --copy-links" + :includes => calculate_includes_from_files(files_to_sync, '/files'), + :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links" } else nil @@ -100,9 +204,9 @@ module LeapCli def sync_puppet_files(ssh) ssh.rsync.update do |server| - ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + LeapCli::PUPPET_DESTINATION) + ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir) { - :dest => LeapCli::PUPPET_DESTINATION, + :dest => Leap::Platform.leap_dir, :source => '.', :chdir => Path.platform, :excludes => '*', @@ -112,7 +216,12 @@ module LeapCli end end + # + # ensure submodules are up to date, if the platform is a git + # repository. + # def init_submodules + return unless is_git_directory?(Path.platform) Dir.chdir Path.platform do assert_run! "git submodule sync" statuses = assert_run! "git submodule status" @@ -126,11 +235,17 @@ module LeapCli end end - def calculate_includes_from_files(files) + # + # converts an array of file paths into an array + # suitable for --include of rsync + # + # if set, `prefix` is stripped off. + # + def calculate_includes_from_files(files, prefix=nil) return nil unless files and files.any? # prepend '/' (kind of like ^ for rsync) - includes = files.collect {|file| '/' + file} + includes = files.collect {|file| file =~ /^\// ? file : '/' + file } # include all sub files of specified directories includes.size.times do |i| @@ -148,6 +263,10 @@ module LeapCli end end + if prefix + includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')} + end + return includes end @@ -155,23 +274,11 @@ module LeapCli if options[:tags] tags = options[:tags].split(',') else - tags = LeapCli::DEFAULT_TAGS.dup + tags = Leap::Platform.default_puppet_tags.dup end tags << 'leap_slow' unless options[:fast] tags.join(',') end - # - # for safety, we allow production deploys to be turned off in the Leapfile. - # - def filter_deploy_nodes(filter) - nodes = manager.filter!(filter) - if !leapfile.allow_production_deploy - nodes = nodes[:environment => "!production"] - assert! nodes.any?, "Skipping deploy because @allow_production_deploy is disabled." - end - nodes - end - end end diff --git a/lib/leap_cli/commands/env.rb b/lib/leap_cli/commands/env.rb new file mode 100644 index 0000000..b2f585d --- /dev/null +++ b/lib/leap_cli/commands/env.rb @@ -0,0 +1,53 @@ +module LeapCli + module Commands + + desc "Manipulate and query environment information." + long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+ + "A node in one environment will never interact with a node from another environment. "+ + "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+ + "absolute file path of your provider directory (pins don't apply if you move the directory)" + command :env do |c| + c.desc "List the available environments. The pinned environment, if any, will be marked with '*'." + c.command :ls do |ls| + ls.action do |global_options, options, args| + envs = ["default"] + manager.environment_names.compact.sort + envs.each do |env| + if env + if LeapCli.leapfile.environment == env + puts "* #{env}" + else + puts " #{env}" + end + end + end + end + end + + c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.' + c.arg_name 'ENVIRONMENT' + c.command :pin do |pin| + pin.action do |global_options,options,args| + environment = args.first + if environment == 'default' || + (environment && manager.environment_names.include?(environment)) + LeapCli.leapfile.set('environment', environment) + log 0, :saved, "~/.leaprc with environment set to #{environment}." + end + end + end + + c.desc "Unpin the environment. All subsequent commands will apply to all nodes." + c.command :unpin do |unpin| + unpin.action do |global_options, options, args| + LeapCli.leapfile.unset('environment') + log 0, :saved, "~/.leaprc, removing environment property." + end + end + + c.default_command :ls + end + + protected + + end +end
\ No newline at end of file diff --git a/lib/leap_cli/commands/facts.rb b/lib/leap_cli/commands/facts.rb index d607086..65eda61 100644 --- a/lib/leap_cli/commands/facts.rb +++ b/lib/leap_cli/commands/facts.rb @@ -91,7 +91,9 @@ module LeapCli; module Commands end end end - overwrite_existing = args.empty? + # only overwrite the entire facts file if and only if we are gathering facts + # for all nodes in all environments. + overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil? update_facts_file(new_facts, overwrite_existing) end diff --git a/lib/leap_cli/commands/inspect.rb b/lib/leap_cli/commands/inspect.rb index 746a80c..e8f5caf 100644 --- a/lib/leap_cli/commands/inspect.rb +++ b/lib/leap_cli/commands/inspect.rb @@ -109,7 +109,7 @@ module LeapCli; module Commands if options[:base] inspect_json manager.base_provider elsif arg =~ /provider\.(.*)\.json/ - inspect_json manager.providers[$1] + inspect_json manager.env($1).provider else inspect_json manager.provider end diff --git a/lib/leap_cli/commands/list.rb b/lib/leap_cli/commands/list.rb index 5b84113..b8d7739 100644 --- a/lib/leap_cli/commands/list.rb +++ b/lib/leap_cli/commands/list.rb @@ -15,24 +15,25 @@ module LeapCli; module Commands c.flag 'print', :desc => 'What attributes to print (optional)' c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false c.action do |global_options,options,args| + # don't rely on default manager(), because we want to pass custom options to load() + manager = LeapCli::Config::Manager.new if global_options[:color] colors = ['cyan', 'white'] else colors = [nil, nil] end puts - if options['disabled'] - manager.load(:include_disabled => true) # reload, with disabled nodes - end + manager.load(:include_disabled => options['disabled'], :continue_on_error => true) if options['print'] print_node_properties(manager.filter(args), options['print']) else if args.any? NodeTable.new(manager.filter(args), colors).run else - TagTable.new('SERVICES', manager.services, colors).run - TagTable.new('TAGS', manager.tags, colors).run - NodeTable.new(manager.nodes, colors).run + environment = LeapCli.leapfile.environment || '_all_' + TagTable.new('SERVICES', manager.env(environment).services, colors).run + TagTable.new('TAGS', manager.env(environment).tags, colors).run + NodeTable.new(manager.filter(), colors).run end end end @@ -41,11 +42,9 @@ module LeapCli; module Commands private def self.print_node_properties(nodes, properties) - node_list = manager.nodes properties = properties.split(',') max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max} nodes.each_node do |node| - node.evaluate value = properties.collect{|prop| if node[prop].nil? "null" @@ -68,7 +67,7 @@ module LeapCli; module Commands @colors = colors end def run - tags = @tag_list.keys.sort + tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max table :border => false do row :color => @colors[0] do @@ -76,6 +75,7 @@ module LeapCli; module Commands column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2 end tags.each do |tag| + next if @tag_list[tag].node_list.empty? row :color => @colors[1] do column tag column @tag_list[tag].node_list.keys.sort.join(', ') diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb index 304d86b..f1e1cf8 100644 --- a/lib/leap_cli/commands/node.rb +++ b/lib/leap_cli/commands/node.rb @@ -1,6 +1,4 @@ -require 'net/ssh/known_hosts' -require 'tempfile' -require 'ipaddr' +autoload :IPAddr, 'ipaddr' module LeapCli; module Commands @@ -194,18 +192,40 @@ module LeapCli; module Commands end end + # + # get the public host key for a host. + # return SshKey object representation of the key. + # + # Only supports ecdsa or rsa host keys. ecdsa is preferred if both are available. + # def get_public_key_for_ip(address, port=22) assert_bin!('ssh-keyscan') - output = assert_run! "ssh-keyscan -p #{port} -t ecdsa #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" - line = output.split("\n").grep(/^[^#]/).first - if line =~ /No route to host/ - bail! :failed, 'ssh-keyscan: no route to %s' % address - elsif line =~ /no hostkey alg/ - bail! :failed, 'ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)' + output = assert_run! "ssh-keyscan -p #{port} #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?" + if output.empty? + bail! :failed, "ssh-keyscan returned empty output." + end + + # key arrays [ip, key_type, public_key] + rsa_key = nil + ecdsa_key = nil + + lines = output.split("\n").grep(/^[^#]/) + lines.each do |line| + if line =~ /No route to host/ + bail! :failed, 'ssh-keyscan: no route to %s' % address + elsif line =~ / ssh-rsa / + rsa_key = line.split(' ') + elsif line =~ / ecdsa-sha2-nistp256 / + ecdsa_key = line.split(' ') + end + end + + if rsa_key.nil? && ecdsa_key.nil? + bail! "ssh-keyscan got zero host keys back! Output was: #{output}" + else + key = ecdsa_key || rsa_key + return SshKey.load(key[2], key[1]) end - assert! line, "Got zero host keys back!" - ip, key_type, public_key = line.split(' ') - return SshKey.load(public_key, key_type) end def is_node_alive(node, options) diff --git a/lib/leap_cli/commands/pre.rb b/lib/leap_cli/commands/pre.rb index 4b62b5b..7a64c15 100644 --- a/lib/leap_cli/commands/pre.rb +++ b/lib/leap_cli/commands/pre.rb @@ -21,7 +21,7 @@ module LeapCli; module Commands switch :yes, :negatable => false desc 'Enable debugging library (leap_cli development only)' - switch :debug, :negatable => false + switch [:d, :debug], :negatable => false desc 'Disable colors in output' default_value true @@ -31,12 +31,7 @@ module LeapCli; module Commands # # set verbosity # - LeapCli.log_level = global[:verbose].to_i - if LeapCli.log_level > 1 - ENV['GLI_DEBUG'] = "true" - else - ENV['GLI_DEBUG'] = "false" - end + LeapCli.set_log_level(global[:verbose].to_i) # # load Leapfile @@ -53,13 +48,6 @@ module LeapCli; module Commands bail! { log :missing, "platform directory '#{Path.platform}'" } end - if LeapCli.leapfile.platform_branch && LeapCli::Util.is_git_directory?(Path.platform) - branch = LeapCli::Util.current_git_branch(Path.platform) - if branch != LeapCli.leapfile.platform_branch - bail! "Wrong branch for #{Path.platform}. Was '#{branch}', should be '#{LeapCli.leapfile.platform_branch}'. Edit Leapfile to disable this check." - end - end - # # set log file # @@ -68,18 +56,7 @@ module LeapCli; module Commands log_version LeapCli.log_in_color = global[:color] - # - # load all the nodes everything - # - manager - - # - # check requirements - # - REQUIREMENTS.each do |key| - assert_config! key - end - + true end private diff --git a/lib/leap_cli/commands/shell.rb b/lib/leap_cli/commands/shell.rb index 2ccb3de..2138e9d 100644 --- a/lib/leap_cli/commands/shell.rb +++ b/lib/leap_cli/commands/shell.rb @@ -3,8 +3,10 @@ module LeapCli; module Commands desc 'Log in to the specified node with an interactive shell.' arg_name 'NAME' #, :optional => false, :multiple => false command :ssh do |c| + c.flag 'ssh', :desc => "Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')" + c.flag 'port', :desc => 'Override ssh port for remote host' c.action do |global_options,options,args| - exec_ssh(:ssh, args) + exec_ssh(:ssh, options, args) end end @@ -12,7 +14,7 @@ module LeapCli; module Commands arg_name 'NAME' command :mosh do |c| c.action do |global_options,options,args| - exec_ssh(:mosh, args) + exec_ssh(:mosh, options, args) end end @@ -44,8 +46,9 @@ module LeapCli; module Commands private - def exec_ssh(cmd, args) + def exec_ssh(cmd, cli_options, args) node = get_node_from_args(args, :include_disabled => true) + port = node.ssh.port options = [ "-o 'HostName=#{node.ip_address}'", # "-o 'HostKeyAlias=#{node.name}'", << oddly incompatible with ports in known_hosts file, so we must not use this or non-standard ports break. @@ -65,7 +68,13 @@ module LeapCli; module Commands elsif LeapCli.log_level >= 2 options << "-v" end - ssh = "ssh -l #{username} -p #{node.ssh.port} #{options.join(' ')}" + if cli_options[:port] + port = cli_options[:port] + end + if cli_options[:ssh] + options << cli_options[:ssh] + end + ssh = "ssh -l #{username} -p #{port} #{options.join(' ')}" if cmd == :ssh command = "#{ssh} #{node.domain.full}" elsif cmd == :mosh diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb index 2584a69..2f146b7 100644 --- a/lib/leap_cli/commands/test.rb +++ b/lib/leap_cli/commands/test.rb @@ -33,9 +33,9 @@ module LeapCli; module Commands def test_cmd(options) if options[:continue] - "#{PUPPET_DESTINATION}/bin/run_tests --continue" + "#{Leap::Platform.leap_dir}/bin/run_tests --continue" else - "#{PUPPET_DESTINATION}/bin/run_tests" + "#{Leap::Platform.leap_dir}/bin/run_tests" end end diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb index d7c21db..6c33878 100644 --- a/lib/leap_cli/commands/user.rb +++ b/lib/leap_cli/commands/user.rb @@ -1,4 +1,3 @@ -require 'gpgme' # # perhaps we want to verify that the key files are actually the key files we expect. @@ -75,8 +74,10 @@ module LeapCli if `which ssh-add`.strip.any? `ssh-add -L 2> /dev/null`.split("\n").compact.each do |line| key = SshKey.load(line) - key.comment = 'ssh-agent' - ssh_keys << key unless ssh_keys.include?(key) + if key + key.comment = 'ssh-agent' + ssh_keys << key unless ssh_keys.include?(key) + end end end ssh_keys.compact! @@ -98,13 +99,20 @@ module LeapCli # let the the user choose among the gpg public keys that we encounter, or just pick the key if there is only one. # def pick_pgp_key + begin + return unless `which gpg`.strip.any? + require 'gpgme' + rescue LoadError + return + end + secret_keys = GPGME::Key.find(:secret) if secret_keys.empty? log "Skipping OpenPGP setup because I could not find any OpenPGP keys for you" return nil end - assert_bin! 'gpg' + secret_keys.select!{|key| !key.expired} if secret_keys.length > 1 key_index = numbered_choice_menu('Choose your OpenPGP public key', secret_keys) do |key, i| diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index 5219161..41fda03 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -1,4 +1,4 @@ -require 'ipaddr' +autoload :IPAddr, 'ipaddr' require 'fileutils' module LeapCli; module Commands diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb new file mode 100644 index 0000000..123533f --- /dev/null +++ b/lib/leap_cli/config/filter.rb @@ -0,0 +1,151 @@ +# +# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to +# be applied to. This class is a helper for manager to run these filters. +# +# Classes other than Manager should not use this class. +# + +module LeapCli + module Config + class Filter + + # + # filter -- array of strings, each one a filter + # options -- hash, possible keys include + # :nopin -- disregard environment pinning + # :local -- if false, disallow local nodes + # + def initialize(filters, options, manager) + @filters = filters.nil? ? [] : filters.dup + @environments = [] + @options = options + @manager = manager + + # split filters by pulling out items that happen + # to be environment names. + if LeapCli.leapfile.environment.nil? || @options[:nopin] + @environments = [] + else + @environments = [LeapCli.leapfile.environment] + end + @filters.select! do |filter| + filter_text = filter.sub(/^\+/,'') + if is_environment?(filter_text) + if filter_text == LeapCli.leapfile.environment + # silently ignore already pinned environments + elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty? + LeapCli::Util.bail! do + LeapCli::Util.log "Environments are exclusive: no node is in two environments." do + LeapCli::Util.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'" + end + end + else + @environments << filter_text + end + false + else + true + end + end + + # don't let the first filter have a + prefix + if @filters[0] =~ /^\+/ + @filters[0] = @filters[0][1..-1] + end + end + + # actually run the filter, returns a filtered list of nodes + def nodes() + if @filters.empty? + return nodes_for_empty_filter + else + return nodes_for_filter + end + end + + private + + def nodes_for_empty_filter + node_list = @manager.nodes + if @environments.any? + node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ] + end + if @options[:local] === false + node_list = node_list[:environment => '!local'] + end + node_list + end + + def nodes_for_filter + node_list = Config::ObjectList.new + @filters.each do |filter| + if filter =~ /^\+/ + keep_list = nodes_for_name(filter[1..-1]) + node_list.delete_if do |name, node| + if keep_list[name] + false + else + true + end + end + else + node_list.merge!(nodes_for_name(filter)) + end + end + node_list + end + + private + + # + # returns a set of nodes corresponding to a single name, + # where name could be a node name, service name, or tag name. + # + # For services and tags, we only include nodes for the + # environments that are active + # + def nodes_for_name(name) + if node = @manager.nodes[name] + return Config::ObjectList.new(node) + elsif @environments.empty? + if @manager.services[name] + return @manager.env('_all_').services[name].node_list + elsif @manager.tags[name] + return @manager.env('_all_').tags[name].node_list + else + LeapCli::Util.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + return Config::ObjectList.new + end + else + node_list = Config::ObjectList.new + if @manager.services[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).services[name].node_list) + end + elsif @manager.tags[name] + @environments.each do |env| + node_list.merge!(@manager.env(env).tags[name].node_list) + end + else + LeapCli::Util.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments." + end + return node_list + end + end + + # + # when pinning, we use the name 'default' to specify nodes + # without an environment set, but when filtering, we need to filter + # on :environment => nil. + # + def env_to_filter(environment) + environment == 'default' ? nil : environment + end + + def is_environment?(text) + text == 'default' || @manager.environment_names.include?(text) + end + + end + end +end diff --git a/lib/leap_cli/config/macros.rb b/lib/leap_cli/config/macros.rb deleted file mode 100644 index 59453b0..0000000 --- a/lib/leap_cli/config/macros.rb +++ /dev/null @@ -1,514 +0,0 @@ -# encoding: utf-8 -# -# MACROS -# these are methods available when eval'ing a value in the .json configuration -# -# This module is included in Config::Object -# - -require 'base32' - -module LeapCli; module Config - module Macros - ## - ## NODES - ## - - # - # the list of all the nodes - # - def nodes - global.nodes - end - - # - # grab an environment appropriate provider - # - def provider - global.env(@node.environment).provider - end - - # - # returns a list of nodes that match the same environment - # - # if @node.environment is not set, we return other nodes - # where environment is not set. - # - def nodes_like_me - nodes[:environment => @node.environment] - end - - # - # returns a list of nodes that match the location name - # and environment of @node. - # - def nodes_near_me - if @node['location'] && @node['location']['name'] - nodes_like_me['location.name' => @node.location.name] - else - nodes_like_me['location' => nil] - end - end - - # - # - # picks a node out from the node list in such a way that: - # - # (1) which nodes picked which nodes is saved in secrets.json - # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node - # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. - # - # if the node_list is empty, an exception is raised. - # if node_list size is 1, then that node is returned and nothing is - # memorized via the secrets.json file. - # - # `label` is needed to distinguish between pools of nodes for different purposes. - # - # TODO: more evenly balance after all the nodes have been picked. - # - def pick_node(label, node_list) - if node_list.any? - if node_list.size == 1 - return node_list.values.first - else - secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" - secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} - secrets_value[@node.name] ||= begin - node_to_pick = nil - node_list.each_node do |node| - next if secrets_value.values.include?(node.name) - node_to_pick = node.name - end - node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. - node_to_pick - end - picked_node_name = secrets_value[@node.name] - @manager.secrets.set(secrets_key, secrets_value, @node.environment) - return node_list[picked_node_name] - end - else - raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') - end - end - - ## - ## FILES - ## - - class FileMissing < Exception - attr_accessor :path, :options - def initialize(path, options={}) - @path = path - @options = options - end - def to_s - @path - end - end - - # - # inserts the contents of a file - # - def file(filename, options={}) - if filename.is_a? Symbol - filename = [filename, @node.name] - end - filepath = Path.find_file(filename) - if filepath - if filepath =~ /\.erb$/ - ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) - else - File.read(filepath, :encoding => 'UTF-8') - end - else - raise FileMissing.new(Path.named_path(filename), options) - "" - end - end - - # - # like #file, but allow missing files - # - def try_file(filename) - return file(filename) - rescue FileMissing - return nil - end - - # - # returns what the file path will be, once the file is rsynced to the server. - # an internal list of discovered file paths is saved, in order to rsync these files when needed. - # - # notes: - # - # * argument 'path' is relative to Path.provider/files or Path.provider_base/files - # * the path returned by this method is absolute - # * the path stored for use later by rsync is relative to Path.provider - # * if the path does not exist locally, but exists in provider_base, then the default file from - # provider_base is copied locally. this is required for rsync to work correctly. - # - def file_path(path) - if path.is_a? Symbol - path = [path, @node.name] - end - actual_path = Path.find_file(path) - if actual_path.nil? - Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." - nil - else - if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ - # if file is under Path.provider_base, we must copy the default file to - # to Path.provider in order for rsync to be able to sync the file. - local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) - FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 - FileUtils.install actual_path, local_provider_path, :mode => 0600 - Util.log :created, Path.relative_path(local_provider_path) - actual_path = local_provider_path - end - if File.directory?(actual_path) && actual_path !~ /\/$/ - actual_path += '/' # ensure directories end with /, important for building rsync command - end - relative_path = Path.relative_path(actual_path) - @node.file_paths << relative_path - @node.manager.provider.hiera_sync_destination + '/' + relative_path - end - end - - # - # inserts a named secret, generating it if needed. - # - # manager.export_secrets should be called later to capture any newly generated secrets. - # - # +length+ is the character length of the generated password. - # - def secret(name, length=32) - @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment]) - end - - # inserts a base32 encoded secret - def base32_secret(name, length=20) - @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment]) - end - - # Picks a random obfsproxy port from given range - def rand_range(name, range) - @manager.secrets.set(name, rand(range), @node[:environment]) - end - - # - # inserts an hexidecimal secret string, generating it if needed. - # - # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) - # - def hex_secret(name, bit_length=128) - @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment]) - end - - # - # return a fingerprint for a x509 certificate - # - def fingerprint(filename) - "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) - end - - ## - ## HOSTS - ## - - # - # records the list of hosts that are encountered for this node - # - def hostnames(nodes) - @referenced_nodes ||= ObjectList.new - nodes = listify(nodes) - nodes.each_node do |node| - @referenced_nodes[node.name] ||= node - end - return nodes.values.collect {|node| node.domain.name} - end - - # - # Generates entries needed for updating /etc/hosts on a node (as a hash). - # - # Argument `nodes` can be nil or a list of nodes. If nil, only include the - # IPs of the other nodes this @node as has encountered (plus all mx nodes). - # - # Also, for virtual machines, we use the local address if this @node is in - # the same location as the node in question. - # - # We include the ssh public key for each host, so that the hash can also - # be used to generate the /etc/ssh/known_hosts - # - def hosts_file(nodes=nil) - if nodes.nil? - if @referenced_nodes && @referenced_nodes.any? - nodes = @referenced_nodes - nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. - end - end - return {} unless nodes - hosts = {} - my_location = @node['location'] ? @node['location']['name'] : nil - nodes.each_node do |node| - hosts[node.name] = {'ip_address' => node.ip_address, 'domain_internal' => node.domain.internal, 'domain_full' => node.domain.full} - node_location = node['location'] ? node['location']['name'] : nil - if my_location == node_location - if facts = @node.manager.facts[node.name] - if facts['ec2_public_ipv4'] - hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] - end - end - end - host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) - if host_pub_key - hosts[node.name]['host_pub_key'] = host_pub_key - end - end - hosts - end - - ## - ## STUNNEL - ## - - # - # stunnel configuration for the client side. - # - # +node_list+ is a ObjectList of nodes running stunnel servers. - # - # +port+ is the real port of the ultimate service running on the servers - # that the client wants to connect to. - # - # About ths stunnel puppet names: - # - # * accept_port is the port on localhost to which local clients - # can connect. it is auto generated serially. - # * connect_port is the port on the stunnel server to connect to. - # it is auto generated from the +port+ argument. - # - # The network looks like this: - # - # |------ stunnel client ---------------| |--------- stunnel server -----------------------| - # consumer app -> localhost:accept_port -> server:connect_port -> server:port -> service app - # - # generates an entry appropriate to be passed directly to - # create_resources(stunnel::service, hiera('..'), defaults) - # - # local ports are automatically generated, starting at 4000 - # and incrementing in sorted order (by node name). - # - def stunnel_client(node_list, port, options={}) - @next_stunnel_port ||= 4000 - node_list = listify(node_list) - hostnames(node_list) # record the hosts - result = Config::ObjectList.new - node_list.each_node do |node| - if node.name != self.name || options[:include_self] - result["#{node.name}_#{port}"] = Config::Object[ - 'accept_port', @next_stunnel_port, - 'connect', node.domain.internal, - 'connect_port', stunnel_port(port) - ] - @next_stunnel_port += 1 - end - end - result - end - - # - # generates a stunnel server entry. - # - # +port+ is the real port targeted service. - # - def stunnel_server(port) - {"accept" => stunnel_port(port), "connect" => "127.0.0.1:#{port}"} - end - - # - # maps a real port to a stunnel port (used as the connect_port in the client config - # and the accept_port in the server config) - # - def stunnel_port(port) - port = port.to_i - if port < 50000 - return port + 10000 - else - return port - 10000 - end - end - - ## - ## HAPROXY - ## - - # - # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. - # - # * node_list - a hash of nodes for the haproxy servers - # * stunnel_client - contains the mappings to local ports for each server node. - # * non_stunnel_port - in case self is included in node_list, the port to connect to. - # - # 1000 weight is used for nodes in the same location. - # 100 otherwise. - # - def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) - default_weight = 10 - local_weight = 100 - - # record the hosts_file - hostnames(node_list) - - # create a simple map for node name -> local stunnel accept port - accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| - name = stunnel_entry.first.sub /_[0-9]+$/, '' - hsh[name] = stunnel_entry.last['accept_port'] - hsh - end - - # if one the nodes in the node list is ourself, then there will not be a stunnel to it, - # but we need to include it anyway in the haproxy config. - if node_list[self.name] && non_stunnel_port - accept_ports[self.name] = non_stunnel_port - end - - # create the first pass of the servers hash - servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| - weight = default_weight - if self['location'] && node['location'] - if self.location['name'] == node.location['name'] - weight = local_weight - end - end - hsh[node.name] = Config::Object[ - 'backup', false, - 'host', 'localhost', - 'port', accept_ports[node.name] || 0, - 'weight', weight - ] - hsh - end - - # if there are some local servers, make the others backup - if servers.detect{|k,v| v.weight == local_weight} - servers.each do |k,server| - server['backup'] = server['weight'] == default_weight - end - end - - return servers - end - - ## - ## SSH - ## - - # - # Creates a hash from the ssh key info in users directory, for use in - # updating authorized_keys file. Additionally, the 'monitor' public key is - # included, which is used by the monitor nodes to run particular commands - # remotely. - # - def authorized_keys - hash = {} - keys = Dir.glob(Path.named_path([:user_ssh, '*'])) - keys.sort.each do |keyfile| - ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") - name = File.basename(File.dirname(keyfile)) - hash[name] = { - "type" => ssh_type, - "key" => ssh_key - } - end - ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") - hash[Leap::Platform.monitor_username] = { - "type" => ssh_type, - "key" => ssh_key - } - hash - end - - # - # this is not currently used, because we put key information in the 'hosts' hash. - # see 'hosts_file()' - # - # def known_hosts_file(nodes=nil) - # if nodes.nil? - # if @referenced_nodes && @referenced_nodes.any? - # nodes = @referenced_nodes - # end - # end - # return nil unless nodes - # entries = [] - # nodes.each_node do |node| - # hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',') - # pub_key = Util::read_file([:node_ssh_pub_key,node.name]) - # if pub_key - # entries << [hostnames, pub_key].join(' ') - # end - # end - # entries.join("\n") - # end - - ## - ## UTILITY - ## - - class AssertionFailed < Exception - attr_accessor :assertion - def initialize(assertion) - @assertion = assertion - end - def to_s - @assertion - end - end - - def assert(assertion) - if instance_eval(assertion) - true - else - raise AssertionFailed.new(assertion) - end - end - - # - # applies a JSON partial to this node - # - def apply_partial(partial_path) - manager.partials(partial_path).each do |partial_data| - self.deep_merge!(partial_data) - end - end - - # - # If at first you don't succeed, then it is time to give up. - # - # try{} returns nil if anything in the block throws an exception. - # - # You can wrap something that might fail in `try`, like so. - # - # "= try{ nodes[:services => 'tor'].first.ip_address } " - # - def try(&block) - yield - rescue NoMethodError - nil - end - - private - - # - # returns a node list, if argument is not already one - # - def listify(node_list) - if node_list.is_a? Config::ObjectList - node_list - elsif node_list.is_a? Config::Object - Config::ObjectList.new(node_list) - else - raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller - end - end - - end -end; end diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb index 7b3fb27..be95831 100644 --- a/lib/leap_cli/config/manager.rb +++ b/lib/leap_cli/config/manager.rb @@ -20,6 +20,16 @@ module LeapCli def initialize @environments = {} # hash of `Environment` objects, keyed by name. + + # load macros and other custom ruby in provider base + platform_ruby_files = Dir[Path.provider_base + '/lib/*.rb'] + if platform_ruby_files.any? + $: << Path.provider_base + '/lib' + platform_ruby_files.each do |rb_file| + require rb_file + end + end + Config::Object.send(:include, LeapCli::Macro) end ## @@ -54,9 +64,24 @@ module LeapCli e end - def services; env('default').services; end - def tags; env('default').tags; end - def provider; env('default').provider; end + # + # The default accessors for services, tags, and provider. + # For these defaults, use 'default' environment, or whatever + # environment is pinned. + # + def services + env(default_environment).services + end + def tags + env(default_environment).tags + end + def provider + env(default_environment).provider + end + + def default_environment + LeapCli.leapfile.environment + end ## ## IMPORT EXPORT @@ -80,8 +105,8 @@ module LeapCli @secrets = load_json( Path.named_path(:secrets_config, @provider_dir), Config::Secrets) @common.inherit_from! @base_common - # load provider services, tags, and provider.json, DEFAULT environment - log 3, :loading, 'default environment.........' + # For the default environment, load provider services, tags, and provider.json + log 3, :loading, 'default environment...' env('default') do |e| e.services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag, :no_dots => true) e.tags = load_all_json(Path.named_path([:tag_config, '*'], @provider_dir), Config::Tag, :no_dots => true) @@ -92,17 +117,28 @@ module LeapCli validate_provider(e.provider) end - # load provider services, tags, and provider.json, OTHER environments + # create a special '_all_' environment, used for tracking the union + # of all the environments + env('_all_') do |e| + e.services = Config::ObjectList.new + e.tags = Config::ObjectList.new + e.provider = Config::Provider.new + e.services.inherit_from! env('default').services + e.tags.inherit_from! env('default').tags + e.provider.inherit_from! env('default').provider + end + + # For each defined environment, load provider services, tags, and provider.json. environment_names.each do |ename| next unless ename - log 3, :loading, '%s environment.........' % ename + log 3, :loading, '%s environment...' % ename env(ename) do |e| e.services = load_all_json(Path.named_path([:service_env_config, '*', ename], @provider_dir), Config::Tag) e.tags = load_all_json(Path.named_path([:tag_env_config, '*', ename], @provider_dir), Config::Tag) e.provider = load_json( Path.named_path([:provider_env_config, ename], @provider_dir), Config::Provider) - e.services.inherit_from! env.services - e.tags.inherit_from! env.tags - e.provider.inherit_from! env.provider + e.services.inherit_from! env('default').services + e.tags.inherit_from! env('default').tags + e.provider.inherit_from! env('default').provider validate_provider(e.provider) end end @@ -113,15 +149,21 @@ module LeapCli @nodes[name] = apply_inheritance(node) end - # remove disabled nodes - unless options[:include_disabled] - remove_disabled_nodes - end + # do some node-list post-processing + cleanup_node_lists(options) # apply control files @nodes.each do |name, node| control_files(node).each do |file| - node.instance_eval File.read(file), file, 1 + begin + node.eval_file file + rescue ConfigError => exc + if options[:continue_on_error] + exc.log + else + raise exc + end + end end end end @@ -187,42 +229,19 @@ module LeapCli # returns a node list consisting only of nodes that satisfy the filter criteria. # # filter: condition [condition] [condition] [+condition] - # condition: [node_name | service_name | tag_name] + # condition: [node_name | service_name | tag_name | environment_name] # # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR. # + # args: + # filter -- array of filter terms, one per item + # # options: # :local -- if :local is false and the filter is empty, then local nodes are excluded. + # :nopin -- if true, ignore environment pinning # - def filter(filters, options={}) - if filters.empty? - if options[:local] === false - return nodes[:environment => '!local'] - else - return nodes - end - end - if filters[0] =~ /^\+/ - # don't let the first filter have a + prefix - filters[0] = filters[0][1..-1] - end - - node_list = Config::ObjectList.new - filters.each do |filter| - if filter =~ /^\+/ - keep_list = nodes_for_name(filter[1..-1]) - node_list.delete_if do |name, node| - if keep_list[name] - false - else - true - end - end - else - node_list.merge!(nodes_for_name(filter)) - end - end - return node_list + def filter(filters=nil, options={}) + Filter.new(filters, options, self).nodes() end # @@ -396,7 +415,6 @@ module LeapCli raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions else new_node.deep_merge!(service) - self.services[node_service].node_list.add(name, new_node) end end end @@ -414,7 +432,6 @@ module LeapCli raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions else new_node.deep_merge!(tag) - self.tags[node_tag].node_list.add(name, new_node) end end end @@ -428,43 +445,35 @@ module LeapCli apply_inheritance(node, true) end - def remove_disabled_nodes + # + # does some final clean at the end of loading nodes. + # this includes removing disabled nodes, and populating + # the services[x].node_list and tags[x].node_list + # + def cleanup_node_lists(options) @disabled_nodes = Config::ObjectList.new @nodes.each do |name, node| - unless node.enabled - log 2, :skipping, "disabled node #{name}." - @nodes.delete(name) - @disabled_nodes[name] = node + if node.enabled || options[:include_disabled] if node['services'] node['services'].to_a.each do |node_service| - self.services[node_service].node_list.delete(node.name) + env(node.environment).services[node_service].node_list.add(node.name, node) + env('_all_').services[node_service].node_list.add(node.name, node) end end if node['tags'] node['tags'].to_a.each do |node_tag| - self.tags[node_tag].node_list.delete(node.name) + env(node.environment).tags[node_tag].node_list.add(node.name, node) + env('_all_').tags[node_tag].node_list.add(node.name, node) end end + elsif !options[:include_disabled] + log 2, :skipping, "disabled node #{name}." + @nodes.delete(name) + @disabled_nodes[name] = node end end end - - # - # returns a set of nodes corresponding to a single name, where name could be a node name, service name, or tag name. - # - def nodes_for_name(name) - if node = self.nodes[name] - Config::ObjectList.new(node) - elsif service = self.services[name] - service.node_list - elsif tag = self.tags[name] - tag.node_list - else - {} - end - end - def validate_provider(provider) # nothing yet. end diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb index 2392d1c..a0d402b 100644 --- a/lib/leap_cli/config/object.rb +++ b/lib/leap_cli/config/object.rb @@ -8,8 +8,6 @@ if $ruby_version < [1,9] end require 'ya2yaml' # pure ruby yaml -require 'leap_cli/config/macros' - module LeapCli module Config # @@ -20,8 +18,6 @@ module LeapCli # class Object < Hash - include Config::Macros - attr_reader :node attr_reader :manager alias :global :manager @@ -44,7 +40,7 @@ module LeapCli # def dump_yaml evaluate(@node) - ya2yaml(:syck_compatible => true) + sorted_ya2yaml(:syck_compatible => true) end # @@ -68,6 +64,11 @@ module LeapCli get(key) end + # Overrride some default methods in Hash that are likely to + # be used as attributes. + alias_method :hkey, :key + def key; get('key'); end + # # make hash addressable like an object (e.g. obj['name'] available as obj.name) # @@ -134,7 +135,18 @@ module LeapCli # def deep_merge!(object, prefer_self=false) object.each do |key,new_value| - old_value = self.fetch key, nil + if self.has_key?('+'+key) + mode = :add + old_value = self.fetch '+'+key, nil + self.delete('+'+key) + elsif self.has_key?('-'+key) + mode = :subtract + old_value = self.fetch '-'+key, nil + self.delete('-'+key) + else + mode = :normal + old_value = self.fetch key, nil + end # clean up boolean new_value = true if new_value == "true" @@ -160,6 +172,18 @@ module LeapCli elsif new_value.is_a?(Array) && !old_value.is_a?(Array) (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED') + # merge two arrays + elsif old_value.is_a?(Array) && new_value.is_a?(Array) + if mode == :add + value = (old_value + new_value).sort.uniq + elsif mode == :subtract + value = new_value - old_value + elsif prefer_self + value = old_value + else + value = new_value + end + # catch errors elsif type_mismatch?(old_value, new_value) raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [ @@ -168,7 +192,7 @@ module LeapCli key, self.class ] - # merge strings, numbers, and sometimes arrays + # merge simple strings & numbers else if prefer_self value = old_value @@ -206,6 +230,10 @@ module LeapCli end end + def eval_file(filename) + evaluate_ruby(filename, File.read(filename)) + end + protected # @@ -246,45 +274,42 @@ module LeapCli # (`key` is just passed for debugging purposes) # def evaluate_ruby(key, value) - result = nil - if LeapCli.log_level >= 2 - result = self.instance_eval(value) - else - begin - result = self.instance_eval(value) - rescue SystemStackError => exc - Util::log 0, :error, "while evaluating node '#{self.name}'" - Util::log 0, "offending key: #{key}", :indent => 1 - Util::log 0, "offending string: #{value}", :indent => 1 - Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1 - raise SystemExit.new(1) - rescue FileMissing => exc - Util::bail! do - if exc.options[:missing] - Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path) - else - Util::log :error, "while evaluating node '#{self.name}'" - Util::log "offending key: #{key}", :indent => 1 - Util::log "offending string: #{value}", :indent => 1 - Util::log "error message: no file '#{exc}'", :indent => 1 - end - end - rescue AssertionFailed => exc - Util.bail! do - Util::log :failed, "assertion while evaluating node '#{self.name}'" - Util::log 'assertion: %s' % exc.assertion, :indent => 1 - Util::log "offending key: #{key}", :indent => 1 - end - rescue SyntaxError, StandardError => exc - Util::bail! do - Util::log :error, "while evaluating node '#{self.name}'" - Util::log "offending key: #{key}", :indent => 1 - Util::log "offending string: #{value}", :indent => 1 - Util::log "error message: #{exc.inspect}", :indent => 1 - end + self.instance_eval(value, key, 1) + rescue ConfigError => exc + raise exc # pass through + rescue SystemStackError => exc + Util::log 0, :error, "while evaluating node '#{self.name}'" + Util::log 0, "offending key: #{key}", :indent => 1 + Util::log 0, "offending string: #{value}", :indent => 1 + Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1 + raise SystemExit.new(1) + rescue FileMissing => exc + Util::bail! do + if exc.options[:missing] + Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path) + else + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: no file '#{exc}'", :indent => 1 end + raise exc if DEBUG + end + rescue AssertionFailed => exc + Util.bail! do + Util::log :failed, "assertion while evaluating node '#{self.name}'" + Util::log 'assertion: %s' % exc.assertion, :indent => 1 + Util::log "offending key: #{key}", :indent => 1 + raise exc if DEBUG + end + rescue SyntaxError, StandardError => exc + Util::bail! do + Util::log :error, "while evaluating node '#{self.name}'" + Util::log "offending key: #{key}", :indent => 1 + Util::log "offending string: #{value}", :indent => 1 + Util::log "error message: #{exc.inspect}", :indent => 1 + raise exc if DEBUG end - return result end private diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb index cd69d9b..33ca4dd 100644 --- a/lib/leap_cli/config/object_list.rb +++ b/lib/leap_cli/config/object_list.rb @@ -20,8 +20,6 @@ module LeapCli # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition. # A new ObjectList is returned. # - # If the key is an array, it is treated as an array of node names - # # Examples: # # nodes['vpn1'] @@ -30,47 +28,22 @@ module LeapCli # nodes[:public_dns => true] # all nodes with public dns # - # nodes[:services => 'openvpn', :services => 'tor'] + # nodes[:services => 'openvpn', 'location.country_code' => 'US'] + # all nodes with services containing 'openvpn' OR country code of US + # + # Sometimes, you want to do an OR condition with multiple conditions + # for the same field. Since hash keys must be unique, you can use + # an array representation instead: + # + # nodes[[:services, 'openvpn'], [:services, 'tor']] # nodes with openvpn OR tor service # # nodes[:services => 'openvpn'][:tags => 'production'] # nodes with openvpn AND are production # def [](key) - if key.is_a? Hash - results = Config::ObjectList.new - key.each do |field, match_value| - field = field.is_a?(Symbol) ? field.to_s : field - match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value - if match_value.is_a?(String) && match_value =~ /^!/ - operator = :not_equal - match_value = match_value.sub(/^!/, '') - else - operator = :equal - end - each do |name, config| - value = config[field] - if value.is_a? Array - if operator == :equal && value.include?(match_value) - results[name] = config - elsif operator == :not_equal && !value.include?(match_value) - results[name] = config - end - else - if operator == :equal && value == match_value - results[name] = config - elsif operator == :not_equal && value != match_value - results[name] = config - end - end - end - end - results - elsif key.is_a? Array - key.inject(Config::ObjectList.new) do |list, node_name| - list[node_name] = super(node_name.to_s) - list - end + if key.is_a?(Hash) || key.is_a?(Array) + filter(key) else super key.to_s end @@ -88,15 +61,40 @@ module LeapCli end end - # def <<(object) - # if object.is_a? Config::ObjectList - # self.merge!(object) - # elsif object['name'] - # self[object['name']] = object - # else - # raise ArgumentError.new('argument must be a Config::Object or a Config::ObjectList') - # end - # end + # + # filters this object list, producing a new list. + # filter is an array or a hash. see [] + # + def filter(filter) + results = Config::ObjectList.new + filter.each do |field, match_value| + field = field.is_a?(Symbol) ? field.to_s : field + match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value + if match_value.is_a?(String) && match_value =~ /^!/ + operator = :not_equal + match_value = match_value.sub(/^!/, '') + else + operator = :equal + end + each do |name, config| + value = config[field] + if value.is_a? Array + if operator == :equal && value.include?(match_value) + results[name] = config + elsif operator == :not_equal && !value.include?(match_value) + results[name] = config + end + else + if operator == :equal && value == match_value + results[name] = config + elsif operator == :not_equal && value != match_value + results[name] = config + end + end + end + end + results + end def add(name, object) self[name] = object diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb index e5e719d..31f4f76 100644 --- a/lib/leap_cli/config/tag.rb +++ b/lib/leap_cli/config/tag.rb @@ -13,6 +13,13 @@ module LeapCli; module Config super(manager) @node_list = Config::ObjectList.new end + + # don't copy the node list pointer when this object is dup'ed. + def initialize_copy(orig) + super + @node_list = Config::ObjectList.new + end + end end; end diff --git a/lib/leap_cli/constants.rb b/lib/leap_cli/constants.rb deleted file mode 100644 index bf30df1..0000000 --- a/lib/leap_cli/constants.rb +++ /dev/null @@ -1,7 +0,0 @@ -module LeapCli - - PUPPET_DESTINATION = '/srv/leap' - INITIALIZED_FILE = "#{PUPPET_DESTINATION}/initialized" - DEFAULT_TAGS = ['leap_base','leap_service'] - -end
\ No newline at end of file diff --git a/lib/leap_cli/exceptions.rb b/lib/leap_cli/exceptions.rb index cd27f14..24a0fa7 100644 --- a/lib/leap_cli/exceptions.rb +++ b/lib/leap_cli/exceptions.rb @@ -6,6 +6,30 @@ module LeapCli @node = node super(msg) end + def log + Util.log(0, :error, "in node `#{@node.name}`: " + self.message) + end + end + + class FileMissing < StandardError + attr_accessor :path, :options + def initialize(path, options={}) + @path = path + @options = options + end + def to_s + @path + end + end + + class AssertionFailed < StandardError + attr_accessor :assertion + def initialize(assertion) + @assertion = assertion + end + def to_s + @assertion + end end end
\ No newline at end of file diff --git a/lib/leap_cli/leapfile.rb b/lib/leap_cli/leapfile.rb index bdf2c37..8895f4d 100644 --- a/lib/leap_cli/leapfile.rb +++ b/lib/leap_cli/leapfile.rb @@ -16,13 +16,28 @@ module LeapCli attr_accessor :leap_version attr_accessor :log attr_accessor :vagrant_network - attr_accessor :platform_branch - attr_accessor :allow_production_deploy + attr_accessor :environment def initialize @vagrant_network = '10.5.5.0/24' end + # + # The way the Leapfile handles pinning of environment (self.environment) is a little tricky. + # If self.environment is nil, then there is no pin. If self.environment is 'default', then + # there is a pin to the default environment. The problem is that an environment of nil + # is used to indicate the default environment in node properties. + # + # This method returns the environment tag as needed when filtering nodes. + # + def environment_filter + if self.environment == 'default' + nil + else + self.environment + end + end + def load(search_directory=nil) directory = File.expand_path(find_in_directory_tree('Leapfile', search_directory)) if directory == '/' @@ -33,7 +48,7 @@ module LeapCli # @provider_directory_path = directory read_settings(directory + '/Leapfile') - read_settings(ENV['HOME'] + '/.leaprc') + read_settings(leaprc_path) @platform_directory_path = File.expand_path(@platform_directory_path || '../leap_platform', @provider_directory_path) # @@ -51,20 +66,55 @@ module LeapCli "You need platform version #{LeapCli::COMPATIBLE_PLATFORM_VERSION.first} to #{LeapCli::COMPATIBLE_PLATFORM_VERSION.last}." end - # - # set defaults - # - if @allow_production_deploy.nil? - # by default, only allow production deploys from 'master' or if not a git repo - @allow_production_deploy = !LeapCli::Util.is_git_directory?(@provider_directory_path) || - LeapCli::Util.current_git_branch(@provider_directory_path) == 'master' + unless @allow_production_deploy.nil? + Util::log 0, :warning, "in Leapfile: @allow_production_deploy is no longer supported." + end + unless @platform_branch.nil? + Util::log 0, :warning, "in Leapfile: @platform_branch is no longer supported." end return true end end + def set(property, value) + edit_leaprc(property, value) + end + + def unset(property) + edit_leaprc(property) + end + private + # + # adds or removes a line to .leaprc for this particular provider directory. + # if value is nil, the line is removed. if not nil, it is added or replaced. + # + def edit_leaprc(property, value=nil) + file_path = leaprc_path + lines = [] + if File.exists?(file_path) + regexp = /self\.#{Regexp.escape(property)} = .*? if @provider_directory_path == '#{Regexp.escape(@provider_directory_path)}'/ + File.readlines(file_path).each do |line| + unless line =~ regexp + lines << line + end + end + end + unless value.nil? + lines << "self.#{property} = #{value.inspect} if @provider_directory_path == '#{@provider_directory_path}'\n" + end + File.open(file_path, 'w') do |f| + f.write(lines.join) + end + rescue Errno::EACCES, IOError => exc + Util::bail! :error, "trying to save ~/.leaprc (#{exc})." + end + + def leaprc_path + File.join(ENV['HOME'], '.leaprc') + end + def read_settings(file) if File.exists? file Util::log 2, :read, file diff --git a/lib/leap_cli/log.rb b/lib/leap_cli/log.rb index f496b9a..c345107 100644 --- a/lib/leap_cli/log.rb +++ b/lib/leap_cli/log.rb @@ -15,7 +15,7 @@ module LeapCli def log_level @log_level ||= 1 end - def log_level=(value) + def set_log_level(value) @log_level = value end diff --git a/lib/leap_cli/logger.rb b/lib/leap_cli/logger.rb index 954ffe2..cc23aa8 100644 --- a/lib/leap_cli/logger.rb +++ b/lib/leap_cli/logger.rb @@ -138,7 +138,7 @@ module LeapCli # TESTS { :match => /^PASS: /, :color => :green, :priority => -20}, { :match => /^(FAIL|ERROR): /, :color => :red, :priority => -20}, - { :match => /^SKIP: /, :color => :yellow, :priority => -20} + { :match => /^(SKIP|WARN): /, :color => :yellow, :priority => -20} ] diff --git a/lib/leap_cli/markdown_document_listener.rb b/lib/leap_cli/markdown_document_listener.rb index 60b012e..c25a243 100644 --- a/lib/leap_cli/markdown_document_listener.rb +++ b/lib/leap_cli/markdown_document_listener.rb @@ -11,7 +11,7 @@ require 'gli/commands/help_modules/arg_name_formatter' module LeapCli class MarkdownDocumentListener - def initialize(global_options,options,arguments) + def initialize(global_options,options,arguments,app) @io = File.new(File.basename($0) + ".md",'w') @nest = '' @commands = [File.basename($0)] diff --git a/lib/leap_cli/path.rb b/lib/leap_cli/path.rb index cd0e169..1f6726a 100644 --- a/lib/leap_cli/path.rb +++ b/lib/leap_cli/path.rb @@ -26,15 +26,29 @@ module LeapCli; module Path end # - # tries to find a file somewhere + # Tries to find a file somewhere. + # Path can be a named path or a relative path. + # + # relative paths are checked against + # provider/<path> + # provider/files/<path> + # provider_base/<path> + # provider_base/files/<path> + # # def self.find_file(arg) [Path.provider, Path.provider_base].each do |base| - file_path = named_path(arg, base) - return file_path if File.exists?(file_path) - if arg.is_a? String - file_path = base + '/files/' + arg - return file_path if File.exists?(file_path) + if arg.is_a?(Symbol) || arg.is_a?(Array) + named_path(arg, base).tap {|path| + return path if File.exists?(path) + } + else + File.join(base, arg).tap {|path| + return path if File.exists?(path) + } + File.join(base, 'files', arg).tap {|path| + return path if File.exists?(path) + } end end return nil diff --git a/lib/leap_cli/remote/leap_plugin.rb b/lib/leap_cli/remote/leap_plugin.rb index a284712..af88c2a 100644 --- a/lib/leap_cli/remote/leap_plugin.rb +++ b/lib/leap_cli/remote/leap_plugin.rb @@ -26,7 +26,7 @@ module LeapCli; module Remote; module LeapPlugin # def assert_initialized begin - test_initialized_file = "test -f #{INITIALIZED_FILE}" + test_initialized_file = "test -f #{Leap::Platform.init_path}" check_required_packages = "! dpkg-query -W --showformat='${Status}\n' #{required_packages} 2>&1 | grep -q -E '(deinstall|no packages)'" run "#{test_initialized_file} && #{check_required_packages} && echo ok" rescue Capistrano::CommandError => exc @@ -57,7 +57,7 @@ module LeapCli; module Remote; module LeapPlugin end def mark_initialized - run "touch #{INITIALIZED_FILE}" + run "touch #{Leap::Platform.init_path}" end # diff --git a/lib/leap_cli/remote/puppet_plugin.rb b/lib/leap_cli/remote/puppet_plugin.rb index 9c41380..e3f6be2 100644 --- a/lib/leap_cli/remote/puppet_plugin.rb +++ b/lib/leap_cli/remote/puppet_plugin.rb @@ -6,7 +6,7 @@ module LeapCli; module Remote; module PuppetPlugin def apply(options) - run "#{PUPPET_DESTINATION}/bin/puppet_command set_hostname apply #{flagize(options)}" + run "#{Leap::Platform.leap_dir}/bin/puppet_command set_hostname apply #{flagize(options)}" end private diff --git a/lib/leap_cli/remote/rsync_plugin.rb b/lib/leap_cli/remote/rsync_plugin.rb index 48f82d3..a6708f4 100644 --- a/lib/leap_cli/remote/rsync_plugin.rb +++ b/lib/leap_cli/remote/rsync_plugin.rb @@ -3,7 +3,7 @@ # (see RemoteCommand::new_capistrano) # -require 'rsync_command' +autoload :RsyncCommand, 'rsync_command' module LeapCli; module Remote; module RsyncPlugin diff --git a/lib/leap_cli/remote/tasks.rb b/lib/leap_cli/remote/tasks.rb index e66b0a8..7fd8d64 100644 --- a/lib/leap_cli/remote/tasks.rb +++ b/lib/leap_cli/remote/tasks.rb @@ -34,7 +34,7 @@ BAD_APT_GET_UPDATE = /(BADSIG|NO_PUBKEY|KEYEXPIRED|REVKEYSIG|NODATA)/ task :install_prerequisites, :max_hosts => MAX_HOSTS do apt_get = "DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Options::=--force-confold" - leap.mkdirs LeapCli::PUPPET_DESTINATION + leap.mkdirs Leap::Platform.leap_dir run "echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen" leap.log :updating, "package list" do run "apt-get update" do |channel, stream, data| diff --git a/lib/leap_cli/requirements.rb b/lib/leap_cli/requirements.rb deleted file mode 100644 index f1f0952..0000000 --- a/lib/leap_cli/requirements.rb +++ /dev/null @@ -1,19 +0,0 @@ -# run 'rake update-requirements' to generate this file. -module LeapCli - REQUIREMENTS = [ - "provider.ca.name", - "provider.ca.server_certificates.bit_size", - "provider.ca.server_certificates.digest", - "provider.ca.server_certificates.life_span", - "common.x509.use", - "provider.domain", - "provider.name", - "provider.ca.server_certificates.bit_size", - "provider.ca.server_certificates.digest", - "provider.ca.name", - "provider.ca.bit_size", - "provider.ca.life_span", - "provider.ca.client_certificates.unlimited_prefix", - "provider.ca.client_certificates.limited_prefix" - ] -end diff --git a/lib/leap_cli/ssh_key.rb b/lib/leap_cli/ssh_key.rb index a525128..bd5bf43 100644 --- a/lib/leap_cli/ssh_key.rb +++ b/lib/leap_cli/ssh_key.rb @@ -33,6 +33,7 @@ module LeapCli end end return key + rescue StandardError => exc end def self.load_from_file(filename) diff --git a/lib/leap_cli/util.rb b/lib/leap_cli/util.rb index 0174158..07ffcec 100644 --- a/lib/leap_cli/util.rb +++ b/lib/leap_cli/util.rb @@ -30,7 +30,7 @@ module LeapCli # def bail!(*message) if block_given? - LeapCli.log_level = 3 + LeapCli.set_log_level(3) yield elsif message log 0, *message diff --git a/lib/leap_cli/util/remote_command.rb b/lib/leap_cli/util/remote_command.rb index 6b4d75f..6353e36 100644 --- a/lib/leap_cli/util/remote_command.rb +++ b/lib/leap_cli/util/remote_command.rb @@ -78,10 +78,20 @@ module LeapCli; module Util; module RemoteCommand :keys_only => false, # Don't you dare change this. :global_known_hosts_file => path(:known_hosts), :user_known_hosts_file => '/dev/null', - :paranoid => true + :paranoid => true, + :verbose => net_ssh_log_level } end + def net_ssh_log_level + case LeapCli.log_level + when 1 then 3 + when 2 then 2 + when 3 then 1 + else 0 + end + end + # # For notes on advanced ways to set server-specific options, see # http://railsware.com/blog/2011/11/02/advanced-server-definitions-in-capistrano/ diff --git a/lib/leap_cli/util/secret.rb b/lib/leap_cli/util/secret.rb index a6bd7a3..837a0af 100644 --- a/lib/leap_cli/util/secret.rb +++ b/lib/leap_cli/util/secret.rb @@ -4,7 +4,7 @@ # # Uses OpenSSL random number generator instead of Ruby's rand function # -require 'openssl' +autoload :OpenSSL, 'openssl' module LeapCli; module Util class Secret diff --git a/lib/leap_cli/util/x509.rb b/lib/leap_cli/util/x509.rb index 9ecd92d..787fdfa 100644 --- a/lib/leap_cli/util/x509.rb +++ b/lib/leap_cli/util/x509.rb @@ -1,5 +1,6 @@ -require 'openssl' -require 'certificate_authority' +autoload :OpenSSL, 'openssl' +autoload :CertificateAuthority, 'certificate_authority' + require 'digest' require 'digest/md5' require 'digest/sha1' diff --git a/lib/leap_cli/version.rb b/lib/leap_cli/version.rb index df0f87a..0248036 100644 --- a/lib/leap_cli/version.rb +++ b/lib/leap_cli/version.rb @@ -1,7 +1,7 @@ module LeapCli unless defined?(LeapCli::VERSION) - VERSION = '1.5.7' - COMPATIBLE_PLATFORM_VERSION = '0.5.2'..'1.99' + VERSION = '1.6.1' + COMPATIBLE_PLATFORM_VERSION = '0.6.0'..'1.99' SUMMARY = 'Command line interface to the LEAP platform' DESCRIPTION = 'The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home.' LOAD_PATHS = ['lib', 'vendor/certificate_authority/lib', 'vendor/rsync_command/lib'] diff --git a/lib/override/json.rb b/lib/override/json.rb new file mode 100644 index 0000000..a7ae328 --- /dev/null +++ b/lib/override/json.rb @@ -0,0 +1,11 @@ +# +# This exists solely to prevent other gems we depend on from +# importing json/ext (e.g. require 'json'). +# +# If json/ext is imported, json/pure cannot work, and we heavily +# rely on the specific behavior of json/pure. +# +# This trick only works if this directory is early in the +# include path. +# +require 'json/pure'
\ No newline at end of file diff --git a/test/leap_platform/platform.rb b/test/leap_platform/platform.rb index 52bb8df..4d4f22c 100644 --- a/test/leap_platform/platform.rb +++ b/test/leap_platform/platform.rb @@ -4,8 +4,8 @@ # Leap::Platform.define do - self.version = "0.5.2" - self.compatible_cli = "1.5.4".."1.99" + self.version = "0.5.3" + self.compatible_cli = "1.5.8".."1.99" # # the facter facts that should be gathered diff --git a/test/leap_platform/provider_base/lib/macros.rb b/test/leap_platform/provider_base/lib/macros.rb new file mode 100644 index 0000000..854b92b --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros.rb @@ -0,0 +1,14 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' diff --git a/test/leap_platform/provider_base/lib/macros/core.rb b/test/leap_platform/provider_base/lib/macros/core.rb new file mode 100644 index 0000000..d4d9171 --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/core.rb @@ -0,0 +1,86 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # return a fingerprint for a x509 certificate + # + def fingerprint(filename) + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + end + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion) + end + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + manager.partials(partial_path).each do |partial_data| + self.deep_merge!(partial_data) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/test/leap_platform/provider_base/lib/macros/files.rb b/test/leap_platform/provider_base/lib/macros/files.rb new file mode 100644 index 0000000..0a49132 --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/files.rb @@ -0,0 +1,79 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + "" + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns what the file path will be, once the file is rsynced to the server. + # an internal list of discovered file paths is saved, in order to rsync these files when needed. + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, then the default file from + # provider_base is copied locally. this is required for rsync to work correctly. + # + def file_path(path) + if path.is_a? Symbol + path = [path, @node.name] + end + actual_path = Path.find_file(path) + if actual_path.nil? + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + nil + else + if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install actual_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + actual_path = local_provider_path + end + if File.directory?(actual_path) && actual_path !~ /\/$/ + actual_path += '/' # ensure directories end with /, important for building rsync command + end + relative_path = Path.relative_path(actual_path) + @node.file_paths << relative_path + @node.manager.provider.hiera_sync_destination + '/' + relative_path + end + end + + end +end
\ No newline at end of file diff --git a/test/leap_platform/provider_base/lib/macros/haproxy.rb b/test/leap_platform/provider_base/lib/macros/haproxy.rb new file mode 100644 index 0000000..c0f9ede --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/haproxy.rb @@ -0,0 +1,69 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + weight = default_weight + try { + weight = local_weight if self.location.name == node.location.name + } + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name] || 0, + 'weight', weight + ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end
\ No newline at end of file diff --git a/test/leap_platform/provider_base/lib/macros/hosts.rb b/test/leap_platform/provider_base/lib/macros/hosts.rb new file mode 100644 index 0000000..8a4058a --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/hosts.rb @@ -0,0 +1,63 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = {'ip_address' => node.ip_address, 'domain_internal' => node.domain.internal, 'domain_full' => node.domain.full} + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end
\ No newline at end of file diff --git a/test/leap_platform/provider_base/lib/macros/nodes.rb b/test/leap_platform/provider_base/lib/macros/nodes.rb new file mode 100644 index 0000000..0c6668a --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + global.nodes + end + + # + # grab an environment appropriate provider + # + def provider + global.env(@node.environment).provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end
\ No newline at end of file diff --git a/test/leap_platform/provider_base/lib/macros/secrets.rb b/test/leap_platform/provider_base/lib/macros/secrets.rb new file mode 100644 index 0000000..51bf397 --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment]) + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment]) + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + @manager.secrets.set(name, rand(range), @node[:environment]) + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment]) + end + + end +end
\ No newline at end of file diff --git a/test/leap_platform/provider_base/lib/macros/stunnel.rb b/test/leap_platform/provider_base/lib/macros/stunnel.rb new file mode 100644 index 0000000..f16308c --- /dev/null +++ b/test/leap_platform/provider_base/lib/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', stunnel_port(port), + 'original_port', port + ] + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end
\ No newline at end of file diff --git a/test/test_helper.rb b/test/test_helper.rb index b631c23..ee687a9 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,7 +4,7 @@ require 'bundler/setup' require 'minitest/autorun' require 'leap_cli' -class MiniTest::Unit::TestCase +class Minitest::Test attr_accessor :ruby_path # Add global extensions to the test case class here diff --git a/test/unit/command_line_test.rb b/test/unit/command_line_test.rb index 4f8333a..2aaf1c1 100644 --- a/test/unit/command_line_test.rb +++ b/test/unit/command_line_test.rb @@ -1,6 +1,6 @@ require File.expand_path('../test_helper', __FILE__) -class CommandLineTest < MiniTest::Unit::TestCase +class CommandLineTest < Minitest::Test def test_help with_multiple_rubies do diff --git a/test/unit/config_object_list_test.rb b/test/unit/config_object_list_test.rb index d38f441..a0ee3fc 100644 --- a/test/unit/config_object_list_test.rb +++ b/test/unit/config_object_list_test.rb @@ -1,6 +1,6 @@ require File.expand_path('../test_helper', __FILE__) -class ConfigObjectListTest < MiniTest::Unit::TestCase +class ConfigObjectListTest < Minitest::Test def test_node_search nodes = manager.nodes['name' => 'vpn1'] diff --git a/test/unit/config_object_test.rb b/test/unit/config_object_test.rb index b50318a..54b45d1 100644 --- a/test/unit/config_object_test.rb +++ b/test/unit/config_object_test.rb @@ -1,6 +1,6 @@ require File.expand_path('../test_helper', __FILE__) -class ConfigObjectTest < MiniTest::Unit::TestCase +class ConfigObjectTest < Minitest::Test def test_bracket_lookup domain = provider.domain diff --git a/vendor/certificate_authority/certificate_authority.gemspec b/vendor/certificate_authority/certificate_authority.gemspec index be8cd91..b7e8676 100644 --- a/vendor/certificate_authority/certificate_authority.gemspec +++ b/vendor/certificate_authority/certificate_authority.gemspec @@ -61,7 +61,7 @@ Gem::Specification.new do |s| "spec/units/units_helper.rb", "spec/units/working_with_openssl_spec.rb" ] - s.homepage = "http://github.com/cchandler/certificate_authority" + s.homepage = "https://github.com/cchandler/certificate_authority" s.licenses = ["MIT"] s.require_paths = ["lib"] s.rubygems_version = "1.8.15" @@ -72,15 +72,18 @@ Gem::Specification.new do |s| if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_runtime_dependency(%q<activemodel>, [">= 3.0.6"]) + s.add_runtime_dependency(%q<activesupport>, [">= 3.0.6"]) s.add_development_dependency(%q<rspec>, [">= 0"]) s.add_development_dependency(%q<jeweler>, [">= 1.5.2"]) else s.add_dependency(%q<activemodel>, [">= 3.0.6"]) + s.add_dependency(%q<activesupport>, [">= 3.0.6"]) s.add_dependency(%q<rspec>, [">= 0"]) s.add_dependency(%q<jeweler>, [">= 1.5.2"]) end else s.add_dependency(%q<activemodel>, [">= 3.0.6"]) + s.add_dependency(%q<activesupport>, [">= 3.0.6"]) s.add_dependency(%q<rspec>, [">= 0"]) s.add_dependency(%q<jeweler>, [">= 1.5.2"]) end diff --git a/vendor/certificate_authority/lib/certificate_authority/certificate.rb b/vendor/certificate_authority/lib/certificate_authority/certificate.rb index ca8bc7c..3fcae90 100644 --- a/vendor/certificate_authority/lib/certificate_authority/certificate.rb +++ b/vendor/certificate_authority/lib/certificate_authority/certificate.rb @@ -33,7 +33,7 @@ module CertificateAuthority self.serial_number = SerialNumber.new self.key_material = MemoryKeyMaterial.new self.not_before = Time.now - self.not_after = Time.now + 60 * 60 * 24 * 365 #One year + self.not_after = Time.now + 60 * 60 * 24 * 365 # One year self.parent = self self.extensions = load_extensions() @@ -41,12 +41,31 @@ module CertificateAuthority end +=begin + def self.from_openssl openssl_cert + unless openssl_cert.is_a? OpenSSL::X509::Certificate + raise "Can only construct from an OpenSSL::X509::Certificate" + end + + certificate = Certificate.new + # Only subject, key_material, and body are used for signing + certificate.distinguished_name = DistinguishedName.from_openssl openssl_cert.subject + certificate.key_material.public_key = openssl_cert.public_key + certificate.openssl_body = openssl_cert + certificate.serial_number.number = openssl_cert.serial.to_i + certificate.not_before = openssl_cert.not_before + certificate.not_after = openssl_cert.not_after + # TODO extensions + certificate + end +=end + def sign!(signing_profile={}) raise "Invalid certificate #{self.errors.full_messages}" unless valid? merge_profile_with_extensions(signing_profile) openssl_cert = OpenSSL::X509::Certificate.new - openssl_cert.version = 2 + openssl_cert.version = 2 openssl_cert.not_before = self.not_before openssl_cert.not_after = self.not_after openssl_cert.public_key = self.key_material.public_key @@ -58,7 +77,6 @@ module CertificateAuthority require 'tempfile' t = Tempfile.new("bullshit_conf") - # t = File.new("/tmp/openssl.cnf") ## The config requires a file even though we won't use it openssl_config = OpenSSL::Config.new(t.path) @@ -85,7 +103,7 @@ module CertificateAuthority self.extensions.keys.sort{|a,b| b<=>a}.each do |k| e = extensions[k] next if e.to_s.nil? or e.to_s == "" ## If the extension returns an empty string we won't include it - ext = factory.create_ext(e.openssl_identifier, e.to_s) + ext = factory.create_ext(e.openssl_identifier, e.to_s, e.critical) openssl_cert.add_extension(ext) end @@ -94,9 +112,10 @@ module CertificateAuthority else digest = OpenSSL::Digest::Digest.new(signing_profile["digest"]) end - self.openssl_body = openssl_cert.sign(parent.key_material.private_key,digest) - t.close! if t.is_a?(Tempfile)# We can get rid of the ridiculous temp file - self.openssl_body + + self.openssl_body = openssl_cert.sign(parent.key_material.private_key, digest) + ensure + t.close! if t # We can get rid of the ridiculous temp file end def is_signing_entity? @@ -116,6 +135,34 @@ module CertificateAuthority self.openssl_body.to_pem end + def to_csr + csr = SigningRequest.new + csr.distinguished_name = self.distinguished_name + csr.key_material = self.key_material + factory = OpenSSL::X509::ExtensionFactory.new + exts = [] + self.extensions.keys.each do |k| + ## Don't copy over key identifiers for CSRs + next if k == "subjectKeyIdentifier" || k == "authorityKeyIdentifier" + e = extensions[k] + ## If the extension returns an empty string we won't include it + next if e.to_s.nil? or e.to_s == "" + exts << factory.create_ext(e.openssl_identifier, e.to_s, e.critical) + end + attrval = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence(exts)]) + attrs = [ + OpenSSL::X509::Attribute.new("extReq", attrval), + OpenSSL::X509::Attribute.new("msExtReq", attrval) + ] + csr.attributes = attrs + csr + end + + def self.from_x509_cert(raw_cert) + openssl_cert = OpenSSL::X509::Certificate.new(raw_cert) + Certificate.from_openssl(openssl_cert) + end + def is_root_entity? self.parent == self && is_signing_entity? end @@ -134,6 +181,16 @@ module CertificateAuthority items = signing_config[k] items.keys.each do |profile_item_key| if extension.respond_to?("#{profile_item_key}=".to_sym) + if k == 'subjectAltName' && profile_item_key == 'emails' + items[profile_item_key].map do |email| + if email == 'email:copy' + fail "no email address provided for subject: #{subject.to_x509_name}" unless subject.email_address + "email:#{subject.email_address}" + else + email + end + end + end extension.send("#{profile_item_key}=".to_sym, items[profile_item_key] ) else p "Tried applying '#{profile_item_key}' to #{extension.class} but it doesn't respond!" @@ -142,30 +199,25 @@ module CertificateAuthority end end + # Enumeration of the extensions. Not the worst option since + # the likelihood of these needing to be updated is low at best. + EXTENSIONS = [ + CertificateAuthority::Extensions::BasicConstraints, + CertificateAuthority::Extensions::CrlDistributionPoints, + CertificateAuthority::Extensions::SubjectKeyIdentifier, + CertificateAuthority::Extensions::AuthorityKeyIdentifier, + CertificateAuthority::Extensions::AuthorityInfoAccess, + CertificateAuthority::Extensions::KeyUsage, + CertificateAuthority::Extensions::ExtendedKeyUsage, + CertificateAuthority::Extensions::SubjectAlternativeName, + CertificateAuthority::Extensions::CertificatePolicies + ] + def load_extensions extension_hash = {} - temp_extensions = [] - basic_constraints = CertificateAuthority::Extensions::BasicConstraints.new - temp_extensions << basic_constraints - crl_distribution_points = CertificateAuthority::Extensions::CrlDistributionPoints.new - temp_extensions << crl_distribution_points - subject_key_identifier = CertificateAuthority::Extensions::SubjectKeyIdentifier.new - temp_extensions << subject_key_identifier - authority_key_identifier = CertificateAuthority::Extensions::AuthorityKeyIdentifier.new - temp_extensions << authority_key_identifier - authority_info_access = CertificateAuthority::Extensions::AuthorityInfoAccess.new - temp_extensions << authority_info_access - key_usage = CertificateAuthority::Extensions::KeyUsage.new - temp_extensions << key_usage - extended_key_usage = CertificateAuthority::Extensions::ExtendedKeyUsage.new - temp_extensions << extended_key_usage - subject_alternative_name = CertificateAuthority::Extensions::SubjectAlternativeName.new - temp_extensions << subject_alternative_name - certificate_policies = CertificateAuthority::Extensions::CertificatePolicies.new - temp_extensions << certificate_policies - - temp_extensions.each do |extension| + EXTENSIONS.each do |klass| + extension = klass.new extension_hash[extension.openssl_identifier] = extension end @@ -192,7 +244,11 @@ module CertificateAuthority certificate.serial_number.number = openssl_cert.serial.to_i certificate.not_before = openssl_cert.not_before certificate.not_after = openssl_cert.not_after - # TODO extensions + EXTENSIONS.each do |klass| + _,v,c = (openssl_cert.extensions.detect { |e| e.to_a.first == klass::OPENSSL_IDENTIFIER } || []).to_a + certificate.extensions[klass::OPENSSL_IDENTIFIER] = klass.parse(v, c) if v + end + certificate end diff --git a/vendor/certificate_authority/lib/certificate_authority/distinguished_name.rb b/vendor/certificate_authority/lib/certificate_authority/distinguished_name.rb index 165fe29..32d9c1e 100644 --- a/vendor/certificate_authority/lib/certificate_authority/distinguished_name.rb +++ b/vendor/certificate_authority/lib/certificate_authority/distinguished_name.rb @@ -32,11 +32,16 @@ module CertificateAuthority alias :emailAddress :email_address alias :emailAddress= :email_address= + attr_accessor :serial_number + alias :serialNumber :serial_number + alias :serialNumber= :serial_number= + def to_x509_name raise "Invalid Distinguished Name" unless valid? # NB: the capitalization in the strings counts name = OpenSSL::X509::Name.new + name.add_entry("serialNumber", serial_number) unless serial_number.blank? name.add_entry("C", country) unless country.blank? name.add_entry("ST", state) unless state.blank? name.add_entry("L", locality) unless locality.blank? diff --git a/vendor/certificate_authority/lib/certificate_authority/extensions.rb b/vendor/certificate_authority/lib/certificate_authority/extensions.rb index e5a8e85..7bc4fab 100644 --- a/vendor/certificate_authority/lib/certificate_authority/extensions.rb +++ b/vendor/certificate_authority/lib/certificate_authority/extensions.rb @@ -5,6 +5,10 @@ module CertificateAuthority raise "Implementation required" end + def self.parse(value, critical) + raise "Implementation required" + end + def config_extensions {} end @@ -12,21 +16,40 @@ module CertificateAuthority def openssl_identifier raise "Implementation required" end + + def ==(value) + raise "Implementation required" + end end + # Specifies whether an X.509v3 certificate can act as a CA, signing other + # certificates to be verified. If set, a path length constraint can also be + # specified. + # Reference: Section 4.2.1.10 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.10 class BasicConstraints + OPENSSL_IDENTIFIER = "basicConstraints" + include ExtensionAPI include ActiveModel::Validations + + attr_accessor :critical attr_accessor :ca attr_accessor :path_len + validates :critical, :inclusion => [true,false] validates :ca, :inclusion => [true,false] def initialize - self.ca = false + @critical = false + @ca = false + end + + def openssl_identifier + OPENSSL_IDENTIFIER end def is_ca? - self.ca + @ca end def path_len=(value) @@ -34,29 +57,54 @@ module CertificateAuthority @path_len = value end - def openssl_identifier - "basicConstraints" + def to_s + res = [] + res << "CA:#{@ca}" + res << "pathlen:#{@path_len}" unless @path_len.nil? + res.join(',') end - def to_s - result = "" - result += "CA:#{self.ca}" - result += ",pathlen:#{self.path_len}" unless self.path_len.nil? - result + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + value.split(/,\s*/).each do |v| + c = v.split(':', 2) + obj.ca = (c.last.upcase == "TRUE") if c.first == "CA" + obj.path_len = c.last.to_i if c.first == "pathlen" + end + obj + end + + protected + def state + [@critical,@ca,@path_len] end end + # Specifies where CRL information be be retrieved. This extension isn't + # critical, but is recommended for proper CAs. + # Reference: Section 4.2.1.14 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.14 class CrlDistributionPoints + OPENSSL_IDENTIFIER = "crlDistributionPoints" + include ExtensionAPI - attr_accessor :uri + attr_accessor :critical + attr_accessor :uris def initialize - # self.uri = "http://moo.crlendPoint.example.com/something.crl" + @critical = false + @uris = [] end def openssl_identifier - "crlDistributionPoints" + OPENSSL_IDENTIFIER end ## NB: At this time it seems OpenSSL's extension handlers don't support @@ -69,99 +117,302 @@ module CertificateAuthority } end + # This is for legacy support. Technically it can (and probably should) + # be an array. But if someone is calling the old accessor we shouldn't + # necessarily break it. + def uri=(value) + @uris << value + end + def to_s - return "" if self.uri.nil? - "URI:#{self.uri}" + res = [] + @uris.each do |uri| + res << "URI:#{uri}" + end + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + value.split(/,\s*/).each do |v| + c = v.split(':', 2) + obj.uris << c.last if c.first == "URI" + end + obj + end + + protected + def state + [@critical,@uri] end end + # Identifies the public key associated with a given certificate. + # Should be required for "CA" certificates. + # Reference: Section 4.2.1.2 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.2 class SubjectKeyIdentifier + OPENSSL_IDENTIFIER = "subjectKeyIdentifier" + include ExtensionAPI + + attr_accessor :critical + attr_accessor :identifier + + def initialize + @critical = false + @identifier = "hash" + end + def openssl_identifier - "subjectKeyIdentifier" + OPENSSL_IDENTIFIER end def to_s - "hash" + res = [] + res << @identifier + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.identifier = value + obj + end + + protected + def state + [@critical,@identifier] end end + # Identifies the public key associated with a given private key. + # Reference: Section 4.2.1.1 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.1 class AuthorityKeyIdentifier + OPENSSL_IDENTIFIER = "authorityKeyIdentifier" + include ExtensionAPI + attr_accessor :critical + attr_accessor :identifier + + def initialize + @critical = false + @identifier = ["keyid", "issuer"] + end + def openssl_identifier - "authorityKeyIdentifier" + OPENSSL_IDENTIFIER end def to_s - "keyid,issuer" + res = [] + res += @identifier + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.identifier = value.split(/,\s*/).last.chomp + obj + end + + protected + def state + [@critical,@identifier] end end + # Specifies how to access CA information and services for the CA that + # issued this certificate. + # Generally used to specify OCSP servers. + # Reference: Section 4.2.2.1 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.2.1 class AuthorityInfoAccess + OPENSSL_IDENTIFIER = "authorityInfoAccess" + include ExtensionAPI + attr_accessor :critical attr_accessor :ocsp + attr_accessor :ca_issuers def initialize - self.ocsp = [] + @critical = false + @ocsp = [] + @ca_issuers = [] end def openssl_identifier - "authorityInfoAccess" + OPENSSL_IDENTIFIER end def to_s - return "" if self.ocsp.empty? - "OCSP;URI:#{self.ocsp}" + res = [] + res += @ocsp.map {|o| "OCSP;URI:#{o}" } + res += @ca_issuers.map {|c| "caIssuers;URI:#{c}" } + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + value.split("\n").each do |v| + if v =~ /^OCSP/ + obj.ocsp << v.split.last + end + + if v =~ /^CA Issuers/ + obj.ca_issuers << v.split.last + end + end + obj + end + + protected + def state + [@critical,@ocsp,@ca_issuers] end end + # Specifies the allowed usage purposes of the keypair specified in this certificate. + # Reference: Section 4.2.1.3 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.3 + # + # Note: OpenSSL when parsing an extension will return results in the form + # 'Digital Signature', but on signing you have to set it to 'digitalSignature'. + # So copying an extension from an imported cert isn't going to work yet. class KeyUsage + OPENSSL_IDENTIFIER = "keyUsage" + include ExtensionAPI + attr_accessor :critical attr_accessor :usage def initialize - self.usage = ["digitalSignature", "nonRepudiation"] + @critical = false + @usage = ["digitalSignature", "nonRepudiation"] end def openssl_identifier - "keyUsage" + OPENSSL_IDENTIFIER end def to_s - "#{self.usage.join(',')}" + res = [] + res += @usage + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.usage = value.split(/,\s*/) + obj + end + + protected + def state + [@critical,@usage] end end + # Specifies even more allowed usages in addition to what is specified in + # the Key Usage extension. + # Reference: Section 4.2.1.13 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.13 class ExtendedKeyUsage + OPENSSL_IDENTIFIER = "extendedKeyUsage" + include ExtensionAPI + attr_accessor :critical attr_accessor :usage def initialize - self.usage = ["serverAuth","clientAuth"] + @critical = false + @usage = ["serverAuth"] end def openssl_identifier - "extendedKeyUsage" + OPENSSL_IDENTIFIER end def to_s - "#{self.usage.join(',')}" + res = [] + res += @usage + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.usage = value.split(/,\s*/) + obj + end + + protected + def state + [@critical,@usage] end end + # Specifies additional "names" for which this certificate is valid. + # Reference: Section 4.2.1.7 of RFC3280 + # http://tools.ietf.org/html/rfc3280#section-4.2.1.7 class SubjectAlternativeName + OPENSSL_IDENTIFIER = "subjectAltName" + include ExtensionAPI - attr_accessor :uris, :dns_names, :ips + attr_accessor :critical + attr_accessor :uris, :dns_names, :ips, :emails def initialize - self.uris = [] - self.dns_names = [] - self.ips = [] + @critical = false + @uris = [] + @dns_names = [] + @ips = [] + @emails = [] + end + + def openssl_identifier + OPENSSL_IDENTIFIER end def uris=(value) @@ -179,22 +430,50 @@ module CertificateAuthority @ips = value end - def openssl_identifier - "subjectAltName" + def emails=(value) + raise "Emails must be an array" unless value.is_a?(Array) + @emails = value end def to_s - res = self.uris.map {|u| "URI:#{u}" } - res += self.dns_names.map {|d| "DNS:#{d}" } - res += self.ips.map {|i| "IP:#{i}" } + res = [] + res += @uris.map {|u| "URI:#{u}" } + res += @dns_names.map {|d| "DNS:#{d}" } + res += @ips.map {|i| "IP:#{i}" } + res += @emails.map {|i| "email:#{i}" } + res.join(',') + end + + def ==(o) + o.class == self.class && o.state == state + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + value.split(/,\s*/).each do |v| + c = v.split(':', 2) + obj.uris << c.last if c.first == "URI" + obj.dns_names << c.last if c.first == "DNS" + obj.ips << c.last if c.first == "IP" + obj.emails << c.last if c.first == "EMAIL" + end + obj + end - return res.join(',') + protected + def state + [@critical,@uris,@dns_names,@ips,@emails] end end class CertificatePolicies + OPENSSL_IDENTIFIER = "certificatePolicies" + include ExtensionAPI + attr_accessor :critical attr_accessor :policy_identifier attr_accessor :cps_uris ##User notice @@ -203,12 +482,12 @@ module CertificateAuthority attr_accessor :notice_numbers def initialize + self.critical = false @contains_data = false end - def openssl_identifier - "certificatePolicies" + OPENSSL_IDENTIFIER end def user_notice=(value={}) @@ -258,7 +537,93 @@ module CertificateAuthority def to_s return "" unless @contains_data - "ia5org,@custom_policies" + res = [] + res << "ia5org" + res += @config_extensions["custom_policies"] unless @config_extensions.nil? + res.join(',') + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + value.split(/,\s*/).each do |v| + c = v.split(':', 2) + obj.policy_identifier = c.last if c.first == "policyIdentifier" + obj.cps_uris << c.last if c.first =~ %r{CPS.\d+} + # TODO: explicit_text, organization, notice_numbers + end + obj + end + end + + # DEPRECATED + # Specifics the purposes for which a certificate can be used. + # The basicConstraints, keyUsage, and extendedKeyUsage extensions are now used instead. + # https://www.openssl.org/docs/apps/x509v3_config.html#Netscape_Certificate_Type + class NetscapeCertificateType + OPENSSL_IDENTIFIER = "nsCertType" + + include ExtensionAPI + + attr_accessor :critical + attr_accessor :flags + + def initialize + self.critical = false + self.flags = [] + end + + def openssl_identifier + OPENSSL_IDENTIFIER + end + + def to_s + res = [] + res += self.flags + res.join(',') + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.flags = value.split(/,\s*/) + obj + end + end + + # DEPRECATED + # Contains a comment which will be displayed when the certificate is viewed in some browsers. + # https://www.openssl.org/docs/apps/x509v3_config.html#Netscape_String_extensions_ + class NetscapeComment + OPENSSL_IDENTIFIER = "nsComment" + + include ExtensionAPI + + attr_accessor :critical + attr_accessor :comment + + def initialize + self.critical = false + end + + def openssl_identifier + OPENSSL_IDENTIFIER + end + + def to_s + res = [] + res << self.comment if self.comment + res.join(',') + end + + def self.parse(value, critical) + obj = self.new + return obj if value.nil? + obj.critical = critical + obj.comment = value + obj end end diff --git a/vendor/certificate_authority/lib/certificate_authority/key_material.rb b/vendor/certificate_authority/lib/certificate_authority/key_material.rb index 75ec62e..1fd4dd9 100644 --- a/vendor/certificate_authority/lib/certificate_authority/key_material.rb +++ b/vendor/certificate_authority/lib/certificate_authority/key_material.rb @@ -111,38 +111,4 @@ module CertificateAuthority @public_key end end - - class SigningRequestKeyMaterial - include KeyMaterial - include ActiveModel::Validations - - validates_each :public_key do |record, attr, value| - record.errors.add :public_key, "cannot be blank" if record.public_key.nil? - end - - attr_accessor :public_key - - def initialize(request=nil) - if request.is_a? OpenSSL::X509::Request - raise "Invalid certificate signing request" unless request.verify request.public_key - self.public_key = request.public_key - end - end - - def is_in_hardware? - false - end - - def is_in_memory? - true - end - - def private_key - nil - end - - def public_key - @public_key - end - end end diff --git a/vendor/certificate_authority/lib/certificate_authority/serial_number.rb b/vendor/certificate_authority/lib/certificate_authority/serial_number.rb index ec0b836..b9a43cc 100644 --- a/vendor/certificate_authority/lib/certificate_authority/serial_number.rb +++ b/vendor/certificate_authority/lib/certificate_authority/serial_number.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module CertificateAuthority class SerialNumber include ActiveModel::Validations @@ -6,5 +8,9 @@ module CertificateAuthority attr_accessor :number validates :number, :presence => true, :numericality => {:greater_than => 0} + + def initialize + self.number = SecureRandom.random_number(2**128-1) + end end end diff --git a/vendor/certificate_authority/lib/certificate_authority/signing_request.rb b/vendor/certificate_authority/lib/certificate_authority/signing_request.rb index 590d5be..72d9e2b 100644 --- a/vendor/certificate_authority/lib/certificate_authority/signing_request.rb +++ b/vendor/certificate_authority/lib/certificate_authority/signing_request.rb @@ -5,6 +5,29 @@ module CertificateAuthority attr_accessor :raw_body attr_accessor :openssl_csr attr_accessor :digest + attr_accessor :attributes + + def initialize() + @attributes = [] + end + + # Fake attribute for convenience because adding + # alternative names on a CSR is remarkably non-trivial. + def subject_alternative_names=(alt_names) + raise "alt_names must be an Array" unless alt_names.is_a?(Array) + + factory = OpenSSL::X509::ExtensionFactory.new + name_list = alt_names.map{|m| "DNS:#{m}"}.join(",") + ext = factory.create_ext("subjectAltName",name_list,false) + ext_set = OpenSSL::ASN1::Set([OpenSSL::ASN1::Sequence([ext])]) + attr = OpenSSL::X509::Attribute.new("extReq", ext_set) + @attributes << attr + end + + def read_attributes_by_oid(*oids) + attributes.detect { |a| oids.include?(a.oid) } + end + protected :read_attributes_by_oid def to_cert cert = Certificate.new @@ -12,6 +35,15 @@ module CertificateAuthority cert.distinguished_name = @distinguished_name end cert.key_material = @key_material + if attribute = read_attributes_by_oid('extReq', 'msExtReq') + set = OpenSSL::ASN1.decode(attribute.value) + seq = set.value.first + seq.value.collect { |asn1ext| OpenSSL::X509::Extension.new(asn1ext).to_a }.each do |o, v, c| + Certificate::EXTENSIONS.each do |klass| + cert.extensions[klass::OPENSSL_IDENTIFIER] = klass.parse(v, c) if v && klass::OPENSSL_IDENTIFIER == o + end + end + end cert end @@ -24,10 +56,12 @@ module CertificateAuthority raise "Invalid DN in request" unless @distinguished_name.valid? raise "CSR must have key material" if @key_material.nil? raise "CSR must include a public key on key material" if @key_material.public_key.nil? + raise "Need a private key on key material for CSR generation" if @key_material.private_key.nil? opensslcsr = OpenSSL::X509::Request.new opensslcsr.subject = @distinguished_name.to_x509_name opensslcsr.public_key = @key_material.public_key + opensslcsr.attributes = @attributes unless @attributes.nil? opensslcsr.sign @key_material.private_key, OpenSSL::Digest::Digest.new(@digest || "SHA512") opensslcsr end @@ -38,6 +72,7 @@ module CertificateAuthority csr.distinguished_name = DistinguishedName.from_openssl openssl_csr.subject csr.raw_body = raw_csr csr.openssl_csr = openssl_csr + csr.attributes = openssl_csr.attributes key_material = SigningRequestKeyMaterial.new key_material.public_key = openssl_csr.public_key csr.key_material = key_material @@ -53,4 +88,4 @@ module CertificateAuthority csr end end -end
\ No newline at end of file +end |