code cleanup. better support for nested configs and templates.

6 files changed, 285 insertions, 250 deletions

diff --git a/lib/leap_cli/config/base.rb b/lib/leap_cli/config/base.rb
deleted file mode 100644
index c7f4bc9..0000000
--- a/lib/leap_cli/config/base.rb
+++ /dev/null
@@ -1,149 +0,0 @@
-module LeapCli
-  module Config
-    #
-    # This class represents the configuration for a single node, service, or tag.
-    #
-    class Base < Hash
-
-      def initialize(manager=nil, node=nil)
-        @manager = manager
-        @node = node || self
-      end
-
-      ##
-      ## FETCHING VALUES
-      ##
-
-      #
-      # lazily eval dynamic values when we encounter them.
-      #
-      def [](key)
-        value = fetch(key, nil)
-        if value.is_a? Array
-          value
-        elsif value.nil?
-          nil
-        else
-          if value =~ /^= (.*)$/
-            begin
-              value = eval($1)
-              self[key] = value
-            rescue Exception => exc
-              puts "Eval error in '#{name}'"
-              puts "   string: #{$1}"
-              puts "   error: #{exc}"
-            end
-          end
-          value
-        end
-      end
-
-      def name
-        @node['name']
-      end
-
-      #
-      # make hash addressable like an object (e.g. obj['name'] available as obj.name)
-      #
-      def method_missing(method, *args, &block)
-        method = method.to_s
-        if self.has_key?(method)
-          self[method]
-        elsif @node != self
-          @node.send(method) # send call up the tree...
-        else
-          raise NoMethodError.new(method)
-        end
-      end
-
-      #
-      # a deep (recursive) merge with another hash or node.
-      #
-      def deep_merge!(hsh)
-        hsh.each do |key,new_value|
-          old_value = self[key]
-          if old_value.is_a?(Hash) || new_value.is_a?(Hash)
-            # merge hashes
-            value = Base.new(@manager, @node)
-            old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if old_value.any?)
-            new_value.is_a?(Hash) ? value.deep_merge!(new_value) : (value[key] = new_value if new_value.any?)
-          elsif old_value.is_a?(Array) || new_value.is_a?(Array)
-            # merge arrays
-            value = []
-            old_value.is_a?(Array) ? value += old_value : value << old_value
-            new_value.is_a?(Array) ? value += new_value : value << new_value
-            value.compact!
-          elsif new_value.nil?
-            value = old_value
-          elsif old_value.nil?
-            value = new_value
-          elsif old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
-            value = new_value
-          elsif old_value.class != new_value.class
-            raise 'Type mismatch. Cannot merge %s with %s. Key value is %s, name is %s.' % [old_value.class, new_value.class, key, name]
-          else
-            value = new_value
-          end
-          self[key] = value
-        end
-        self
-      end
-
-      #def deep_merge!(new_node)
-      #  new_node.each do |key, value|
-      #    if value.is_a? self.class
-      #      value = Base.new(@manager, @node).deep_merge!(value)
-      #    self[key] = new_node[key]
-      #  end
-      #  self
-      #end
-
-      #
-      # like a normal deep_merge, but replace any hash it encounters with a Config::Base
-      #
-      #def deep_merge(other_hash)
-      #  p [self['name'], other_hash['name']]
-      #  self.merge(other_hash) do |key, oldval, newval|
-      #    oldval = oldval.to_hash if oldval.respond_to?(:to_hash)
-      #    newval = newval.to_hash if newval.respond_to?(:to_hash)
-      #    p key
-      #    p oldval.class
-      #    p newval.class
-      #    if oldval.class.to_s == 'Hash' && newval.class.to_s == 'Hash'
-      #      oldval.deep_merge(newval)
-      #    elsif newval.class.to_s == 'Hash'
-      #      p key
-      #      Base.new(@manager, node).replace(newval)
-      #    else
-      #      newval
-      #    end
-      #  end
-      #end
-      #
-      #def deep_merge!(other_hash)
-      #  replace(deep_merge(other_hash))
-      #end
-
-      private
-
-      ##
-      ## MACROS
-      ## these are methods used when eval'ing a value in the .json configuration
-      ##
-
-      #
-      # inserts the contents of a file
-      #
-      def file(filename)
-        filepath = Path.find_file(name, filename)
-        if filepath
-          File.read(filepath)
-        else
-          log0('no such file, "%s"' % filename)
-          ""
-        end
-      end
-
-    end # class
-  end # module
-end # module
\ No newline at end of file
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
index 6a7c1e9..55575cf 100644
--- a/lib/leap_cli/config/manager.rb
+++ b/lib/leap_cli/config/manager.rb
@@ -1,11 +1,14 @@
-require 'oj'
+require 'json/pure'
 require 'yaml'
 
 module LeapCli
   module Config
+    #
+    # A class to manage all the objects in all the configuration files.
+    #
     class Manager
 
-      attr_reader :services, :tags, :nodes
+      attr_reader :services, :tags, :nodes, :provider
 
       ##
       ## IMPORT EXPORT
@@ -15,10 +18,11 @@ module LeapCli
       # load .json configuration files
       #
       def load(dir)
-        @services = load_all_json("#{dir}/services/*.json", :tag)
-        @tags     = load_all_json("#{dir}/tags/*.json", :tag)
-        @common   = load_all_json("#{dir}/common.json", :tag)['common']
-        @nodes    = load_all_json("#{dir}/nodes/*.json", :node)
+        @services = load_all_json("#{dir}/services/*.json")
+        @tags     = load_all_json("#{dir}/tags/*.json")
+        @common   = load_all_json("#{dir}/common.json")['common']
+        @provider = load_all_json("#{dir}/provider.json")['provider']
+        @nodes    = load_all_json("#{dir}/nodes/*.json")
         @nodes.each do |name, node|
           @nodes[name] = apply_inheritance(node)
         end
@@ -61,7 +65,7 @@ module LeapCli
           filters[0] = filters[0][1..-1]
         end
 
-        node_list = Config::List.new
+        node_list = Config::ObjectList.new
         filters.each do |filter|
           if filter =~ /^\+/
             keep_list = nodes_for_name(filter[1..-1])
@@ -73,38 +77,21 @@ module LeapCli
               end
             end
           else
-            node_list << nodes_for_name(filter)
+            node_list.merge!(nodes_for_name(filter))
           end
         end
         return node_list
       end
 
-      ##
-      ## CLASS METHODS
-      ##
-
-      #def self.manager
-      #  @manager ||= begin
-      #    manager = ConfigManager.new
-      #    manager.load(Path.provider)
-      #    manager
-      #  end
-      #end
-
-      #def self.filter(filters); manager.filter(filters); end
-      #def self.nodes; manager.nodes; end
-      #def self.services; manager.services; end
-      #def self.tags; manager.tags; end
-
       private
 
       def load_all_json(pattern, config_type = :class)
-        results = Config::List.new
+        results = Config::ObjectList.new
         Dir.glob(pattern).each do |filename|
           obj = load_json(filename, config_type)
           if obj
             name = File.basename(filename).sub(/\.json$/,'')
-            obj['name'] = name
+            obj['name'] ||= name
             results[name] = obj
           end
         end
@@ -128,15 +115,16 @@ module LeapCli
 
         # parse json, and flatten hash
         begin
-          hash = Oj.load(buffer.string) || {}
+          #hash = Oj.load(buffer.string) || {}
+          hash = JSON.parse(buffer.string, :object_class => Hash, :array_class => Array) || {}
         rescue SyntaxError => exc
           log0 'Error in file "%s":' % filename
           log0 exc.to_s
           return nil
         end
-        config = config_type == :node ? Node.new(self) : Tag.new(self)
-        config.deep_merge!(hash)
-        return config
+        object = Config::Object.new(self)
+        object.deep_merge!(hash)
+        return object
       end
 
       #
@@ -156,17 +144,14 @@ module LeapCli
       # end
 
       #
-      # makes this node inherit options from the common, service, and tag json files.
-      #
-      # - takes a hash
-      # - returns a Node object.
+      # makes a node inherit options from appropriate the common, service, and tag json files.
       #
       def apply_inheritance(node)
-        new_hash = Node.new(self)
-        #new_node = Node.new(self)
+        new_node = Config::Object.new(self)
+        name = node.name
 
         # inherit from common
-        new_hash.deep_merge!(@common)
+        new_node.deep_merge!(@common)
 
         # inherit from services
         if node['services']
@@ -175,8 +160,8 @@ module LeapCli
             if service.nil?
               log0('Error in node "%s": the service "%s" does not exist.' % [node['name'], node_service])
             else
-              new_hash.deep_merge!(service)
-              service.nodes << new_hash
+              new_node.deep_merge!(service)
+              service.node_list.add(name, new_node)
             end
           end
         end
@@ -188,19 +173,15 @@ module LeapCli
             if tag.nil?
               log0('Error in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag])
             else
-              new_hash.deep_merge!(tag)
-              tag.nodes << new_hash
+              new_node.deep_merge!(tag)
+              tag.node_list.add(name, new_node)
             end
           end
         end
 
         # inherit from node
-        new_hash.deep_merge!(node)
-
-        # typecast full hash tree to type Node
-        #new_node.clone_from_plain_hash!(new_hash)
-
-        return new_hash
+        new_node.deep_merge!(node)
+        return new_node
       end
 
       #
@@ -208,11 +189,11 @@ module LeapCli
       #
       def nodes_for_name(name)
         if node = self.nodes[name]
-          Config::List.new(node)
+          Config::ObjectList.new(node)
         elsif service = self.services[name]
-          service.nodes
+          service.node_list
         elsif tag = self.tags[name]
-          tag.nodes
+          tag.node_list
         end
       end
 
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
deleted file mode 100644
index 5389b44..0000000
--- a/lib/leap_cli/config/node.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-module LeapCli
-  module Config
-    class Node < Base
-
-      def nodes
-        @manager.nodes
-      end
-
-      def services
-        self['services'] || []
-      end
-
-      def tags
-        self['tags'] || []
-      end
-
-    end
-  end
-end
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
new file mode 100644
index 0000000..2ef7fe8
--- /dev/null
+++ b/lib/leap_cli/config/object.rb
@@ -0,0 +1,233 @@
+require 'erb'
+require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
+
+module LeapCli
+  module Config
+    #
+    # This class represents the configuration for a single node, service, or tag.
+    # Also, all the nested hashes are also of this type.
+    #
+    # It is called 'object' because it corresponds to an Object in JSON.
+    #
+    class Object < Hash
+
+      attr_reader :node
+      attr_reader :manager
+      attr_reader :node_list
+      alias :global :manager
+
+      def initialize(manager=nil, node=nil)
+        # keep a global pointer around to the config manager. used a lot in the eval strings and templates
+        # (which are evaluated in the context of Config::Object)
+        @manager = manager
+
+        # an object that is a node as @node equal to self, otherwise all the child objects point back to the top level node.
+        @node = node || self
+
+        # this is only used by Config::Objects that correspond to services or tags.
+        @node_list = Config::ObjectList.new
+      end
+
+      ##
+      ## FETCHING VALUES
+      ##
+
+      #
+      # like a normal hash [], except:
+      # * lazily eval dynamic values when we encounter them.
+      # * support for nested hashes (e.g. ['a.b'] is the same as ['a']['b'])
+      #
+      def [](key)
+        get(key)
+      end
+
+      #
+      # make hash addressable like an object (e.g. obj['name'] available as obj.name)
+      #
+      def method_missing(method, *args, &block)
+        get!(method)
+      end
+
+      def get(key)
+        begin
+          get!(key)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def get!(key)
+        key = key.to_s
+        if key =~ /\./
+          keys = key.split('.')
+          value = get!(keys.first)
+          if value.is_a? Config::Object
+            value.get!(keys[1..-1])
+          else
+            value
+          end
+        elsif self.has_key?(key)
+          evaluate_value(key)
+        elsif @node != self
+          @node.get!(key)
+        else
+          raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
+        end
+      end
+
+      ##
+      ## COPYING
+      ##
+
+      #
+      # Make a copy of ourselves, except only including the specified keys.
+      #
+      # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+      #
+      def pick(*keys)
+        keys.map(&:to_s).inject(Config::Object.new(@manager,@node)) do |hsh, key|
+          value = self.get(key)
+          if value
+            hsh[key.gsub('.','_')] = value
+          end
+          hsh
+        end
+      end
+
+      #
+      # a deep (recursive) merge with another Config::Object.
+      #
+      def deep_merge!(object)
+        object.each do |key,new_value|
+          old_value = self[key]
+          if old_value.is_a?(Hash) || new_value.is_a?(Hash)
+            # merge hashes
+            value = Config::Object.new(@manager, @node)
+            old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if old_value.any?)
+            new_value.is_a?(Hash) ? value.deep_merge!(new_value) : (value[key] = new_value if new_value.any?)
+          elsif old_value.is_a?(Array) || new_value.is_a?(Array)
+            # merge arrays
+            value = []
+            old_value.is_a?(Array) ? value += old_value : value << old_value
+            new_value.is_a?(Array) ? value += new_value : value << new_value
+            value.compact!
+          elsif new_value.nil?
+            value = old_value
+          elsif old_value.nil?
+            value = new_value
+          elsif old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
+            value = new_value
+          elsif old_value.class != new_value.class
+            raise 'Type mismatch. Cannot merge %s with %s. Key value is %s, name is %s.' % [old_value.class, new_value.class, key, name]
+          else
+            value = new_value
+          end
+          self[key] = value
+        end
+        self
+      end
+
+      private
+
+      #
+      # fetches the value for the key, evaluating the value as ruby if it begins with '='
+      #
+      def evaluate_value(key)
+        value = fetch(key, nil)
+        if value.is_a? Array
+          value
+        elsif value.nil?
+          nil
+        else
+          if value =~ /^= (.*)$/
+            begin
+              value = eval($1, self.send(:binding))
+              self[key] = value
+            rescue SystemStackError => exc
+              puts "STACK OVERFLOW, BAILING OUT"
+              puts "There must be an eval loop of death (variables with circular dependencies). This is the offending string:"
+              puts
+              puts "    #{$1}"
+              puts
+              raise SystemExit.new()
+            rescue StandardError => exc
+              puts "Eval error in '#{@node.name}'"
+              puts "   string: #{$1}"
+              puts "   error: #{exc.name}"
+            end
+          end
+          value
+        end
+      end
+
+      ##
+      ## MACROS
+      ## these are methods used when eval'ing a value in the .json configuration
+      ##
+
+      #
+      # the list of all the nodes
+      #
+      def nodes
+        global.nodes
+      end
+
+      #
+      # inserts the contents of a file
+      #
+      def file(filename)
+        filepath = Path.find_file(@node.name, filename)
+        if filepath
+          if filepath =~ /\.erb$/
+            ERB.new(File.read(filepath), nil, '%<>').result(binding)
+          else
+            File.read(filepath)
+          end
+        else
+          log0('no such file, "%s"' % filename)
+          ""
+        end
+      end
+
+      #
+      # Output json from ruby objects in such a manner that all the hashes and arrays are output in alphanumeric sorted order.
+      # This is required so that our generated configs don't throw puppet or git for a tizzy fit.
+      #
+      # Beware: some hacky stuff ahead.
+      #
+      # This relies on the pure ruby implementation of JSON.generate (i.e. require 'json/pure')
+      # see https://github.com/flori/json/blob/master/lib/json/pure/generator.rb
+      #
+      # The Oj way that we are not using: Oj.dump(obj, :mode => :compat, :indent => 2)
+      #
+      def generate_json(obj)
+
+        # modify hash and array
+        Hash.class_eval do
+          alias_method :each_without_sort, :each
+          def each(&block)
+            keys.sort {|a,b| a.to_s <=> b.to_s }.each do |key|
+              yield key, self[key]
+            end
+          end
+        end
+        Array.class_eval do
+          alias_method :each_without_sort, :each
+          def each(&block)
+            sort {|a,b| a.to_s <=> b.to_s }.each_without_sort &block
+          end
+        end
+
+        # generate json
+        return_value = JSON.pretty_generate(obj)
+
+        # restore hash and array
+        Hash.class_eval  {alias_method :each, :each_without_sort}
+        Array.class_eval {alias_method :each, :each_without_sort}
+
+        return return_value
+      end
+
+    end # class
+  end # module
+end # module
\ No newline at end of file
diff --git a/lib/leap_cli/config/list.rb b/lib/leap_cli/config/object_list.rb
index 28ef499..bcc000d 100644
--- a/lib/leap_cli/config/list.rb
+++ b/lib/leap_cli/config/object_list.rb
@@ -1,10 +1,13 @@
 module LeapCli
   module Config
-    class List < Hash
+    #
+    # A list of Config::Object instances (internally stored as a hash)
+    #
+    class ObjectList < Hash
 
       def initialize(config=nil)
         if config
-          self << config
+          self.add(config['name'], config)
         end
       end
 
@@ -19,7 +22,7 @@ module LeapCli
       #
       def [](key)
         if key.is_a? Hash
-          results = List.new
+          results = Config::ObjectList.new
           field, match_value = key.to_a.first
           field = field.is_a?(Symbol) ? field.to_s : field
           match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
@@ -43,14 +46,19 @@ module LeapCli
         end
       end
 
-      def <<(config)
-        if config.is_a? Config::List
-          self.deep_merge!(config)
-        elsif config['name']
-          self[config['name']] = config
-        else
-          raise ArgumentError.new('argument must be a Config::Base or a Config::List')
-        end
+
+      # def <<(object)
+      #   if object.is_a? Config::ObjectList
+      #     self.merge!(object)
+      #   elsif object['name']
+      #     self[object['name']] = object
+      #   else
+      #     raise ArgumentError.new('argument must be a Config::Object or a Config::ObjectList')
+      #   end
+      # end
+
+      def add(name, object)
+        self[name] = object
       end
 
       #
@@ -71,7 +79,7 @@ module LeapCli
         field = field.to_s
         result = []
         keys.sort.each do |name|
-          result << self[name][field]
+          result << self[name].get(field)
         end
         result
       end
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
deleted file mode 100644
index 25c7246..0000000
--- a/lib/leap_cli/config/tag.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-module LeapCli
-  module Config
-    class Tag < Base
-
-      def nodes
-        @nodes ||= Config::List.new
-      end
-
-      def services
-        @manager.services
-      end
-
-      def tags
-        @manager.tags
-      end
-
-    end
-  end
-end