diff --git a/lib/leap/platform.rb b/lib/leap/platform.rb
new file mode 100644
index 0000000000000000000000000000000000000000..9e6cadd534cebf6cc8a31b9984b8a7884580a47b
--- /dev/null
+++ b/lib/leap/platform.rb
@@ -0,0 +1,99 @@
+module Leap
+
+ class Platform
+ class << self
+ #
+ # configuration
+ #
+
+ attr_reader :version
+ attr_reader :compatible_cli
+ attr_accessor :facts
+ attr_accessor :paths
+ attr_accessor :node_files
+ attr_accessor :monitor_username
+ attr_accessor :reserved_usernames
+
+ attr_accessor :hiera_dir
+ attr_accessor :hiera_path
+ attr_accessor :files_dir
+ attr_accessor :leap_dir
+ attr_accessor :init_path
+
+ attr_accessor :default_puppet_tags
+
+ def define(&block)
+ # some defaults:
+ @reserved_usernames = []
+ @hiera_dir = '/etc/leap'
+ @hiera_path = '/etc/leap/hiera.yaml'
+ @leap_dir = '/srv/leap'
+ @files_dir = '/srv/leap/files'
+ @init_path = '/srv/leap/initialized'
+ @default_puppet_tags = []
+
+ self.instance_eval(&block)
+
+ @version ||= Gem::Version.new("0.0")
+ end
+
+ def validate!(cli_version, compatible_platforms, leapfile)
+ if !compatible_with_cli?(cli_version) || !version_in_range?(compatible_platforms)
+ raise StandardError, "This leap command (v#{cli_version}) " +
+ "is not compatible with the platform #{leapfile.platform_directory_path} (v#{version}).\n " +
+ "You need either leap command #{compatible_cli.first} to #{compatible_cli.last} or " +
+ "platform version #{compatible_platforms.first} to #{compatible_platforms.last}"
+ end
+ end
+
+ def version=(version)
+ @version = Gem::Version.new(version)
+ end
+
+ def compatible_cli=(range)
+ @compatible_cli = range
+ @minimum_cli_version = Gem::Version.new(range.first)
+ @maximum_cli_version = Gem::Version.new(range.last)
+ end
+
+ #
+ # return true if the cli_version is compatible with this platform.
+ #
+ def compatible_with_cli?(cli_version)
+ cli_version = Gem::Version.new(cli_version)
+ cli_version >= @minimum_cli_version && cli_version <= @maximum_cli_version
+ end
+
+ #
+ # return true if the platform version is within the specified range.
+ #
+ def version_in_range?(range)
+ if range.is_a? String
+ range = range.split('..')
+ end
+ minimum_platform_version = Gem::Version.new(range.first)
+ maximum_platform_version = Gem::Version.new(range.last)
+ @version >= minimum_platform_version && @version <= maximum_platform_version
+ end
+
+ def major_version
+ if @version.segments.first == 0
+ @version.segments[0..1].join('.')
+ else
+ @version.segments.first
+ end
+ end
+
+ def method_missing(method, *args)
+ puts
+ puts "WARNING:"
+ puts " leap_cli is out of date and does not understand `#{method}`."
+ puts " called from: #{caller.first}"
+ puts " please upgrade to a newer leap_cli"
+ end
+
+ end
+
+ end
+
+end
\ No newline at end of file
diff --git a/lib/leap_cli/commands/deploy.rb b/lib/leap_cli/commands/deploy.rb
index 165ce588e56e3e3dca643e80aaec0c250c26242f..d26b9905da6606706d9a22bbe7d5448d1b72e4b4 100644
--- a/lib/leap_cli/commands/deploy.rb
+++ b/lib/leap_cli/commands/deploy.rb
@@ -89,6 +89,10 @@ module LeapCli
end
end
+ if nodes.empty?
+ return
+ end
+
log :synching, "configuration files" do
sync_hiera_config(nodes, options)
sync_support_files(nodes, options)
diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb
new file mode 100644
index 0000000000000000000000000000000000000000..398fd023002e35aad35c371ef0d6412dda3ed69c
--- /dev/null
+++ b/lib/leap_cli/config/environment.rb
@@ -0,0 +1,180 @@
+#
+# All configurations files can be isolated into separate environments.
+#
+# Each config json in each environment inherits from the default environment,
+# which in term inherits from the "_base_" environment:
+#
+# _base_ -- base provider in leap_platform
+# '- default -- environment in provider dir when no env is set
+# '- production -- example environment
+#
+
+module LeapCli; module Config
+
+ class Environment
+ # the String name of the environment
+ attr_accessor :name
+
+ # the shared Manager object
+ attr_accessor :manager
+
+ # hashes of {name => Config::Object}
+ attr_accessor :services, :tags, :partials
+
+ # a Config::Provider
+ attr_accessor :provider
+
+ # a Config::Object
+ attr_accessor :common
+
+ # shared, non-inheritable
+ def nodes; @@nodes; end
+ def secrets; @@secrets; end
+
+ def initialize(manager, name, search_dir, parent, options={})
+ @@nodes ||= nil
+ @@secrets ||= nil
+
+ @manager = manager
+ @name = name
+
+ load_provider_files(search_dir, options)
+
+ if parent
+ @services.inherit_from! parent.services, self
+ @tags.inherit_from! parent.tags , self
+ @partials.inherit_from! parent.partials, self
+ @common.inherit_from! parent.common
+ @provider.inherit_from! parent.provider
+ end
+
+ if @provider
+ @provider.set_env(name)
+ @provider.validate!
+ end
+ end
+
+ def load_provider_files(search_dir, options)
+ #
+ # load empty environment if search_dir doesn't exist
+ #
+ if search_dir.nil? || !Dir.exist?(search_dir)
+ @services = Config::ObjectList.new
+ @tags = Config::ObjectList.new
+ @partials = Config::ObjectList.new
+ @provider = Config::Provider.new
+ @common = Config::Object.new
+ return
+ end
+
+ #
+ # inheritable
+ #
+ if options[:scope]
+ scope = options[:scope]
+ @services = load_all_json(Path.named_path([:service_env_config, '*', scope], search_dir), Config::Tag, options)
+ @tags = load_all_json(Path.named_path([:tag_env_config, '*', scope], search_dir), Config::Tag, options)
+ @partials = load_all_json(Path.named_path([:service_env_config, '_*', scope], search_dir), Config::Tag, options)
+ @provider = load_json( Path.named_path([:provider_env_config, scope], search_dir), Config::Provider, options)
+ @common = load_json( Path.named_path([:common_env_config, scope], search_dir), Config::Object, options)
+ else
+ @services = load_all_json(Path.named_path([:service_config, '*'], search_dir), Config::Tag, options)
+ @tags = load_all_json(Path.named_path([:tag_config, '*'], search_dir), Config::Tag, options)
+ @partials = load_all_json(Path.named_path([:service_config, '_*'], search_dir), Config::Tag, options)
+ @provider = load_json( Path.named_path(:provider_config, search_dir), Config::Provider, options)
+ @common = load_json( Path.named_path(:common_config, search_dir), Config::Object, options)
+ end
+
+ # remove 'name' from partials, since partials get merged with nodes
+ @partials.values.each {|partial| partial.delete('name'); }
+
+ #
+ # shared: currently non-inheritable
+ # load the first ones we find, and only those.
+ #
+ if @@nodes.nil? || @@nodes.empty?
+ @@nodes = load_all_json(Path.named_path([:node_config, '*'], search_dir), Config::Node, options)
+ end
+ if @@secrets.nil? || @@secrets.empty?
+ @@secrets = load_json(Path.named_path(:secrets_config, search_dir), Config::Secrets, options)
+ end
+ end
+
+ #
+ # Loads a json template file as a Hash (used only when creating a new node .json
+ # file for the first time).
+ #
+ def template(template)
+ path = Path.named_path([:template_config, template], Path.provider_base)
+ if File.exist?(path)
+ return load_json(path, Config::Object)
+ else
+ return nil
+ end
+ end
+
+ private
+
+ def load_all_json(pattern, object_class, options={})
+ results = Config::ObjectList.new
+ Dir.glob(pattern).each do |filename|
+ next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/
+ obj = load_json(filename, object_class)
+ if obj
+ name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1')
+ obj['name'] ||= name
+ if options[:env]
+ obj.environment = options[:env]
+ end
+ results[name] = obj
+ end
+ end
+ results
+ end
+
+ def load_json(filename, object_class, options={})
+ if !File.exist?(filename)
+ return object_class.new(self)
+ end
+
+ Util::log :loading, filename, 3
+
+ #
+ # Read a JSON file, strip out comments.
+ #
+ # UTF8 is the default encoding for JSON, but others are allowed:
+ # https://www.ietf.org/rfc/rfc4627.txt
+ #
+ buffer = StringIO.new
+ File.open(filename, "rb", :encoding => 'UTF-8') do |f|
+ while (line = f.gets)
+ next if line =~ /^\s*\/\//
+ buffer << line
+ end
+ end
+
+ #
+ # force UTF-8
+ #
+ if $ruby_version >= [1,9]
+ string = buffer.string.force_encoding('utf-8')
+ else
+ string = Iconv.conv("UTF-8//IGNORE", "UTF-8", buffer.string)
+ end
+
+ # parse json
+ begin
+ hash = JSON.parse(string, :object_class => Hash, :array_class => Array) || {}
+ rescue SyntaxError, JSON::ParserError => exc
+ Util::log 0, :error, 'in file "%s":' % filename
+ Util::log 0, exc.to_s, :indent => 1
+ return nil
+ end
+ object = object_class.new(self)
+ object.deep_merge!(hash)
+ return object
+ end
+
+ end # end Environment
+
+end; end
\ No newline at end of file
diff --git a/lib/leap_cli/config/filter.rb b/lib/leap_cli/config/filter.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2750257782b8c26053da106b21a8089137213aed
--- /dev/null
+++ b/lib/leap_cli/config/filter.rb
@@ -0,0 +1,178 @@
+#
+# Many leap_cli commands accept a list of filters to select a subset of nodes for the command to
+# be applied to. This class is a helper for manager to run these filters.
+#
+# Classes other than Manager should not use this class.
+#
+# Filter rules:
+#
+# * A filter consists of a list of tokens
+# * A token may be a service name, tag name, environment name, or node name.
+# * Each token may be optionally prefixed with a plus sign.
+# * Multiple tokens with a plus are treated as an OR condition,
+# but treated as an AND condition with the plus sign.
+#
+# For example
+#
+# * openvpn +development => all nodes with service 'openvpn' AND environment 'development'
+# * openvpn seattle => all nodes with service 'openvpn' OR tag 'seattle'.
+#
+# There can only be one environment specified. Typically, there are also tags
+# for each environment name. These name are treated as environments, not tags.
+#
+module LeapCli
+ module Config
+ class Filter
+
+ #
+ # filter -- array of strings, each one a filter
+ # options -- hash, possible keys include
+ # :nopin -- disregard environment pinning
+ # :local -- if false, disallow local nodes
+ #
+ # A nil value in the filters array indicates
+ # the default environment. This is in order to support
+ # calls like `manager.filter(environments)`
+ #
+ def initialize(filters, options, manager)
+ @filters = filters.nil? ? [] : filters.dup
+ @environments = []
+ @options = options
+ @manager = manager
+
+ # split filters by pulling out items that happen
+ # to be environment names.
+ if LeapCli.leapfile.environment.nil? || @options[:nopin]
+ @environments = []
+ else
+ @environments = [LeapCli.leapfile.environment]
+ end
+ @filters.select! do |filter|
+ if filter.nil?
+ @environments << nil unless @environments.include?(nil)
+ false
+ else
+ filter_text = filter.sub(/^\+/,'')
+ if is_environment?(filter_text)
+ if filter_text == LeapCli.leapfile.environment
+ # silently ignore already pinned environments
+ elsif (filter =~ /^\+/ || @filters.first == filter) && !@environments.empty?
+ LeapCli::Util.bail! do
+ LeapCli.log "Environments are exclusive: no node is in two environments." do
+ LeapCli.log "Tried to filter on '#{@environments.join('\' AND \'')}' AND '#{filter_text}'"
+ end
+ end
+ else
+ @environments << filter_text
+ end
+ false
+ else
+ true
+ end
+ end
+ end
+
+ # don't let the first filter have a + prefix
+ if @filters[0] =~ /^\+/
+ @filters[0] = @filters[0][1..-1]
+ end
+ end
+
+ # actually run the filter, returns a filtered list of nodes
+ def nodes()
+ if @filters.empty?
+ return nodes_for_empty_filter
+ else
+ return nodes_for_filter
+ end
+ end
+
+ private
+
+ def nodes_for_empty_filter
+ node_list = @manager.nodes
+ if @environments.any?
+ node_list = node_list[ @environments.collect{|e|[:environment, env_to_filter(e)]} ]
+ end
+ if @options[:local] === false
+ node_list = node_list[:environment => '!local']
+ end
+ if @options[:disabled] === false
+ node_list = node_list[:environment => '!disabled']
+ end
+ node_list
+ end
+
+ def nodes_for_filter
+ node_list = Config::ObjectList.new
+ @filters.each do |filter|
+ if filter =~ /^\+/
+ keep_list = nodes_for_name(filter[1..-1])
+ node_list.delete_if do |name, node|
+ if keep_list[name]
+ false
+ else
+ true
+ end
+ end
+ else
+ node_list.merge!(nodes_for_name(filter))
+ end
+ end
+ node_list
+ end
+
+ private
+
+ #
+ # returns a set of nodes corresponding to a single name,
+ # where name could be a node name, service name, or tag name.
+ #
+ # For services and tags, we only include nodes for the
+ # environments that are active
+ #
+ def nodes_for_name(name)
+ if node = @manager.nodes[name]
+ return Config::ObjectList.new(node)
+ elsif @environments.empty?
+ if @manager.services[name]
+ return @manager.env('_all_').services[name].node_list
+ elsif @manager.tags[name]
+ return @manager.env('_all_').tags[name].node_list
+ else
+ LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+ return Config::ObjectList.new
+ end
+ else
+ node_list = Config::ObjectList.new
+ if @manager.services[name]
+ @environments.each do |env|
+ node_list.merge!(@manager.env(env).services[name].node_list)
+ end
+ elsif @manager.tags[name]
+ @environments.each do |env|
+ node_list.merge!(@manager.env(env).tags[name].node_list)
+ end
+ else
+ LeapCli.log :warning, "filter '#{name}' does not match any node names, tags, services, or environments."
+ end
+ return node_list
+ end
+ end
+
+ #
+ # when pinning, we use the name 'default' to specify nodes
+ # without an environment set, but when filtering, we need to filter
+ # on :environment => nil.
+ #
+ def env_to_filter(environment)
+ environment == 'default' ? nil : environment
+ end
+
+ def is_environment?(text)
+ text == 'default' || @manager.environment_names.include?(text)
+ end
+
+ end
+ end
+end
diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb
new file mode 100644
index 0000000000000000000000000000000000000000..aea1d322325aed83bec76537d86cf70f2a02266c
--- /dev/null
+++ b/lib/leap_cli/config/manager.rb
@@ -0,0 +1,422 @@
+# encoding: utf-8
+
+require 'json/pure'
+
+if $ruby_version < [1,9]
+ require 'iconv'
+end
+
+module LeapCli
+ module Config
+
+ #
+ # A class to manage all the objects in all the configuration files.
+ #
+ class Manager
+
+ def initialize
+ @environments = {} # hash of `Environment` objects, keyed by name.
+ Config::Object.send(:include, LeapCli::Macro)
+ end
+
+ ##
+ ## ATTRIBUTES
+ ##
+
+ #
+ # returns the Hash of the contents of facts.json
+ #
+ def facts
+ @facts ||= begin
+ content = Util.read_file(:facts)
+ if !content || content.empty?
+ content = "{}"
+ end
+ JSON.parse(content)
+ rescue SyntaxError, JSON::ParserError => exc
+ Util::bail! "Could not parse facts.json -- #{exc}"
+ end
+ end
+
+ #
+ # returns an Array of all the environments defined for this provider.
+ # the returned array includes nil (for the default environment)
+ #
+ def environment_names
+ @environment_names ||= begin
+ [nil] + (env.tags.field('environment') + env.nodes.field('environment')).compact.uniq
+ end
+ end
+
+ #
+ # Returns the appropriate environment variable
+ #
+ def env(env=nil)
+ @environments[env || 'default']
+ end
+
+ #
+ # The default accessors
+ #
+ # For these defaults, use 'default' environment, or whatever
+ # environment is pinned.
+ #
+ # I think it might be an error that these are ever used
+ # and I would like to get rid of them.
+ #
+ def services; env(default_environment).services; end
+ def tags; env(default_environment).tags; end
+ def partials; env(default_environment).partials; end
+ def provider; env(default_environment).provider; end
+ def common; env(default_environment).common; end
+ def secrets; env(default_environment).secrets; end
+ def nodes; env(default_environment).nodes; end
+ def template(*args)
+ self.env.template(*args)
+ end
+
+ def default_environment
+ LeapCli.leapfile.environment
+ end
+
+ ##
+ ## IMPORT EXPORT
+ ##
+
+ def add_environment(args)
+ if args[:inherit]
+ parent = @environments[args.delete(:inherit)]
+ else
+ parent = nil
+ end
+ @environments[args[:name]] = Environment.new(
+ self,
+ args.delete(:name),
+ args.delete(:dir),
+ parent,
+ args
+ )
+ end
+
+ #
+ # load .json configuration files
+ #
+ def load(options = {})
+ @provider_dir = Path.provider
+
+ # load base
+ add_environment(name: '_base_', dir: Path.provider_base)
+
+ # load provider
+ Util::assert_files_exist!(Path.named_path(:provider_config, @provider_dir))
+ add_environment(name: 'default', dir: @provider_dir,
+ inherit: '_base_', no_dots: true)
+
+ # create a special '_all_' environment, used for tracking
+ # the union of all the environments
+ add_environment(name: '_all_', inherit: 'default')
+
+ # load environments
+ environment_names.each do |ename|
+ if ename
+ LeapCli.log 3, :loading, '%s environment...' % ename
+ add_environment(name: ename, dir: @provider_dir,
+ inherit: 'default', scope: ename)
+ end
+ end
+
+ # apply inheritance
+ env.nodes.each do |name, node|
+ Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'"
+ env.nodes[name] = apply_inheritance(node)
+ end
+
+ # do some node-list post-processing
+ cleanup_node_lists(options)
+
+ # apply control files
+ env.nodes.each do |name, node|
+ control_files(node).each do |file|
+ begin
+ node.eval_file file
+ rescue ConfigError => exc
+ if options[:continue_on_error]
+ exc.log
+ else
+ raise exc
+ end
+ end
+ end
+ end
+ end
+
+ #
+ # save compiled hiera .yaml files
+ #
+ # if a node_list is specified, only update those .yaml files.
+ # otherwise, update all files, destroying files that are no longer used.
+ #
+ def export_nodes(node_list=nil)
+ updated_hiera = []
+ updated_files = []
+ existing_hiera = nil
+ existing_files = nil
+
+ unless node_list
+ node_list = env.nodes
+ existing_hiera = Dir.glob(Path.named_path([:hiera, '*'], @provider_dir))
+ existing_files = Dir.glob(Path.named_path([:node_files_dir, '*'], @provider_dir))
+ end
+
+ node_list.each_node do |node|
+ filepath = Path.named_path([:node_files_dir, node.name], @provider_dir)
+ hierapath = Path.named_path([:hiera, node.name], @provider_dir)
+ Util::write_file!(hierapath, node.dump_yaml)
+ updated_files << filepath
+ updated_hiera << hierapath
+ end
+
+ if @disabled_nodes
+ # make disabled nodes appear as if they are still active
+ @disabled_nodes.each_node do |node|
+ updated_files << Path.named_path([:node_files_dir, node.name], @provider_dir)
+ updated_hiera << Path.named_path([:hiera, node.name], @provider_dir)
+ end
+ end
+
+ # remove files that are no longer needed
+ if existing_hiera
+ (existing_hiera - updated_hiera).each do |filepath|
+ Util::remove_file!(filepath)
+ end
+ end
+ if existing_files
+ (existing_files - updated_files).each do |filepath|
+ Util::remove_directory!(filepath)
+ end
+ end
+ end
+
+ def export_secrets(clean_unused_secrets = false)
+ if env.secrets.any?
+ Util.write_file!([:secrets_config, @provider_dir], env.secrets.dump_json(clean_unused_secrets) + "\n")
+ end
+ end
+
+ ##
+ ## FILTERING
+ ##
+
+ #
+ # returns a node list consisting only of nodes that satisfy the filter criteria.
+ #
+ # filter: condition [condition] [condition] [+condition]
+ # condition: [node_name | service_name | tag_name | environment_name]
+ #
+ # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR.
+ #
+ # args:
+ # filter -- array of filter terms, one per item
+ #
+ # options:
+ # :local -- if :local is false and the filter is empty, then local nodes are excluded.
+ # :nopin -- if true, ignore environment pinning
+ #
+ def filter(filters=nil, options={})
+ Filter.new(filters, options, self).nodes()
+ end
+
+ #
+ # same as filter(), but exits if there is no matching nodes
+ #
+ def filter!(filters, options={})
+ node_list = filter(filters, options)
+ Util::assert! node_list.any?, "Could not match any nodes from '#{filters.join ' '}'"
+ return node_list
+ end
+
+ #
+ # returns a single Config::Object that corresponds to a Node.
+ #
+ def node(name)
+ if name =~ /\./
+ # probably got a fqdn, since periods are not allowed in node names.
+ # so, take the part before the first period as the node name
+ name = name.split('.').first
+ end
+ env.nodes[name]
+ end
+
+ #
+ # returns a single node that is disabled
+ #
+ def disabled_node(name)
+ @disabled_nodes[name]
+ end
+
+ #
+ # yields each node, in sorted order
+ #
+ def each_node(&block)
+ env.nodes.each_node(&block)
+ end
+
+ def reload_node!(node)
+ env.nodes[node.name] = apply_inheritance!(node)
+ end
+
+ ##
+ ## CONNECTIONS
+ ##
+
+ class ConnectionList < Array
+ def add(data={})
+ self << {
+ "from" => data[:from],
+ "to" => data[:to],
+ "port" => data[:port]
+ }
+ end
+ end
+
+ def connections
+ @connections ||= ConnectionList.new
+ end
+
+ ##
+ ## PRIVATE
+ ##
+
+ private
+
+ #
+ # makes a node inherit options from appropriate the common, service, and tag json files.
+ #
+ def apply_inheritance(node, throw_exceptions=false)
+ new_node = Config::Node.new(nil)
+ node_env = guess_node_env(node)
+ new_node.set_environment(node_env, new_node)
+
+ # inherit from common
+ new_node.deep_merge!(node_env.common)
+
+ # inherit from services
+ if node['services']
+ node['services'].to_a.each do |node_service|
+ service = node_env.services[node_service]
+ if service.nil?
+ msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service]
+ LeapCli.log 0, :error, msg
+ raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+ else
+ new_node.deep_merge!(service)
+ end
+ end
+ end
+
+ # inherit from tags
+ if node.vagrant?
+ node['tags'] = (node['tags'] || []).to_a + ['local']
+ end
+ if node['tags']
+ node['tags'].to_a.each do |node_tag|
+ tag = node_env.tags[node_tag]
+ if tag.nil?
+ msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag]
+ log 0, :error, msg
+ raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
+ else
+ new_node.deep_merge!(tag)
+ end
+ end
+ end
+
+ # inherit from node
+ new_node.deep_merge!(node)
+ return new_node
+ end
+
+ def apply_inheritance!(node)
+ apply_inheritance(node, true)
+ end
+
+ #
+ # Guess the environment of the node from the tag names.
+ #
+ # Technically, this is wrong: a tag that sets the environment might not be
+ # named the same as the environment. This code assumes that it is.
+ #
+ # Unfortunately, it is a chicken and egg problem. We need to know the nodes
+ # likely environment in order to apply the inheritance that will actually
+ # determine the node's properties.
+ #
+ def guess_node_env(node)
+ if node.vagrant?
+ return self.env("local")
+ else
+ environment = self.env(default_environment)
+ if node['tags']
+ node['tags'].to_a.each do |tag|
+ if self.environment_names.include?(tag)
+ environment = self.env(tag)
+ end
+ end
+ end
+ return environment
+ end
+ end
+
+ #
+ # does some final clean at the end of loading nodes.
+ # this includes removing disabled nodes, and populating
+ # the services[x].node_list and tags[x].node_list
+ #
+ def cleanup_node_lists(options)
+ @disabled_nodes = Config::ObjectList.new
+ env.nodes.each do |name, node|
+ if node.enabled || options[:include_disabled]
+ if node['services']
+ node['services'].to_a.each do |node_service|
+ env(node.environment).services[node_service].node_list.add(node.name, node)
+ env('_all_').services[node_service].node_list.add(node.name, node)
+ end
+ end
+ if node['tags']
+ node['tags'].to_a.each do |node_tag|
+ env(node.environment).tags[node_tag].node_list.add(node.name, node)
+ env('_all_').tags[node_tag].node_list.add(node.name, node)
+ end
+ end
+ elsif !options[:include_disabled]
+ LeapCli.log 2, :skipping, "disabled node #{name}."
+ env.nodes.delete(name)
+ @disabled_nodes[name] = node
+ end
+ end
+ end
+
+ #
+ # returns a list of 'control' files for this node.
+ # a control file is like a service or a tag JSON file, but it contains
+ # raw ruby code that gets evaluated in the context of the node.
+ # Yes, this entirely breaks our functional programming model
+ # for JSON generation.
+ #
+ def control_files(node)
+ files = []
+ [Path.provider_base, @provider_dir].each do |provider_dir|
+ [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym|
+ node[attribute].each do |attr_value|
+ path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'')
+ if File.exist?(path)
+ files << path
+ end
+ end
+ end
+ end
+ return files
+ end
+
+ end
+ end
+end
diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f8ec052761ec6796a82971333feabda438eab665
--- /dev/null
+++ b/lib/leap_cli/config/node.rb
@@ -0,0 +1,78 @@
+#
+# Configuration for a 'node' (a server in the provider's infrastructure)
+#
+
+require 'ipaddr'
+
+module LeapCli; module Config
+
+ class Node < Object
+ attr_accessor :file_paths
+
+ def initialize(environment=nil)
+ super(environment)
+ @node = self
+ @file_paths = []
+ end
+
+ #
+ # returns true if this node has an ip address in the range of the vagrant network
+ #
+ def vagrant?
+ begin
+ vagrant_range = IPAddr.new LeapCli.leapfile.vagrant_network
+ rescue ArgumentError => exc
+ Util::bail! { Util::log :invalid, "ip address '#{@node.ip_address}' vagrant.network" }
+ end
+
+ begin
+ ip_address = IPAddr.new @node.get('ip_address')
+ rescue ArgumentError => exc
+ Util::log :warning, "invalid ip address '#{@node.get('ip_address')}' for node '#{@node.name}'"
+ end
+ return vagrant_range.include?(ip_address)
+ end
+
+ #
+ # Return a hash table representation of ourselves, with the key equal to the @node.name,
+ # and the value equal to the fields specified in *keys.
+ #
+ # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+ #
+ # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields,
+ # but works on a single node.
+ #
+ # Example:
+ #
+ # node.pick('domain.internal') =>
+ #
+ # {
+ # 'node1': {
+ # 'domain_internal': 'node1.example.i'
+ # }
+ # }
+ #
+ def pick_fields(*keys)
+ {@node.name => self.pick(*keys)}
+ end
+
+ #
+ # can be overridden by the platform.
+ # returns a list of node names that should be tested before this node
+ #
+ def test_dependencies
+ []
+ end
+
+ # returns a string list of supported ssh host key algorithms for this node.
+ # or an empty string if it could not be determined
+ def supported_ssh_host_key_algorithms
+ require 'leap_cli/ssh'
+ @host_key_algo ||= LeapCli::SSH::Key.supported_host_key_algorithms(
+ Util.read_file([:node_ssh_pub_key, @node.name])
+ )
+ end
+
+ end
+
+end; end
diff --git a/lib/leap_cli/config/object.rb b/lib/leap_cli/config/object.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b117c2f08598c6141de27fd80bf3b080f07dd0a5
--- /dev/null
+++ b/lib/leap_cli/config/object.rb
@@ -0,0 +1,428 @@
+# encoding: utf-8
+
+require 'erb'
+require 'json/pure' # pure ruby implementation is required for our sorted trick to work.
+
+if $ruby_version < [1,9]
+ $KCODE = 'UTF8'
+end
+require 'ya2yaml' # pure ruby yaml
+
+module LeapCli
+ module Config
+
+ #
+ # This class represents the configuration for a single node, service, or tag.
+ # Also, all the nested hashes are also of this type.
+ #
+ # It is called 'object' because it corresponds to an Object in JSON.
+ #
+ class Object < Hash
+
+ attr_reader :env
+ attr_reader :node
+
+ def initialize(environment=nil, node=nil)
+ raise ArgumentError unless environment.nil? || environment.is_a?(Config::Environment)
+ @env = environment
+ # an object that is a node as @node equal to self, otherwise all the
+ # child objects point back to the top level node.
+ @node = node || self
+ end
+
+ def manager
+ @env.manager
+ end
+
+ #
+ # TODO: deprecate node.global()
+ #
+ def global
+ @env
+ end
+
+ def environment=(e)
+ self.store('environment', e)
+ end
+
+ def environment
+ self['environment']
+ end
+
+ def duplicate(env)
+ new_object = self.deep_dup
+ new_object.set_environment(env, new_object)
+ end
+
+ #
+ # export YAML
+ #
+ # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it
+ # allows us greater compatibility regardless of installed ruby version and
+ # greater control over how the yaml is exported (sorted keys, in particular).
+ #
+ def dump_yaml
+ evaluate(@node)
+ sorted_ya2yaml(:syck_compatible => true)
+ end
+
+ #
+ # export JSON
+ #
+ def dump_json(options={})
+ evaluate(@node)
+ if options[:format] == :compact
+ return self.to_json
+ else
+ excluded = {}
+ if options[:exclude]
+ options[:exclude].each do |key|
+ excluded[key] = self[key]
+ self.delete(key)
+ end
+ end
+ json_str = JSON.sorted_generate(self)
+ if excluded.any?
+ self.merge!(excluded)
+ end
+ return json_str
+ end
+ end
+
+ def evaluate(context=@node)
+ evaluate_everything(context)
+ late_evaluate_everything(context)
+ end
+
+ ##
+ ## FETCHING VALUES
+ ##
+
+ def [](key)
+ get(key)
+ end
+
+ # Overrride some default methods in Hash that are likely to
+ # be used as attributes.
+ alias_method :hkey, :key
+ def key; get('key'); end
+
+ #
+ # make hash addressable like an object (e.g. obj['name'] available as obj.name)
+ #
+ def method_missing(method, *args, &block)
+ get!(method)
+ end
+
+ def get(key)
+ begin
+ get!(key)
+ rescue NoMethodError
+ nil
+ end
+ end
+
+ # override behavior of #default() from Hash
+ def default
+ get!('default')
+ end
+
+ #
+ # Like a normal Hash#[], except:
+ #
+ # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ")
+ #
+ # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b'])
+ # the dot path is always absolute, starting at the top-most object.
+ #
+ def get!(key)
+ key = key.to_s
+ if self.has_key?(key)
+ fetch_value(key)
+ elsif key =~ /\./
+ # for keys with with '.' in them, we start from the root object (@node).
+ keys = key.split('.')
+ value = self.get!(keys.first)
+ if value.is_a? Config::Object
+ value.get!(keys[1..-1].join('.'))
+ else
+ value
+ end
+ else
+ raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
+ end
+ end
+
+ ##
+ ## COPYING
+ ##
+
+ #
+ # A deep (recursive) merge with another Config::Object.
+ #
+ # If prefer_self is set to true, the value from self will be picked when there is a conflict
+ # that cannot be merged.
+ #
+ # Merging rules:
+ #
+ # - If a value is a hash, we recursively merge it.
+ # - If the value is simple, like a string, the new one overwrites the value.
+ # - If the value is an array:
+ # - If both old and new values are arrays, the new one replaces the old.
+ # - If one of the values is simple but the other is an array, the simple is added to the array.
+ #
+ def deep_merge!(object, prefer_self=false)
+ object.each do |key,new_value|
+ if self.has_key?('+'+key)
+ mode = :add
+ old_value = self.fetch '+'+key, nil
+ self.delete('+'+key)
+ elsif self.has_key?('-'+key)
+ mode = :subtract
+ old_value = self.fetch '-'+key, nil
+ self.delete('-'+key)
+ elsif self.has_key?('!'+key)
+ mode = :replace
+ old_value = self.fetch '!'+key, nil
+ self.delete('!'+key)
+ else
+ mode = :normal
+ old_value = self.fetch key, nil
+ end
+
+ # clean up boolean
+ new_value = true if new_value == "true"
+ new_value = false if new_value == "false"
+ old_value = true if old_value == "true"
+ old_value = false if old_value == "false"
+
+ # force replace?
+ if mode == :replace && prefer_self
+ value = old_value
+
+ # merge hashes
+ elsif old_value.is_a?(Hash) || new_value.is_a?(Hash)
+ value = Config::Object.new(@env, @node)
+ old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?)
+ new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?)
+
+ # merge nil
+ elsif new_value.nil?
+ value = old_value
+ elsif old_value.nil?
+ value = new_value
+
+ # merge arrays when one value is not an array
+ elsif old_value.is_a?(Array) && !new_value.is_a?(Array)
+ (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED')
+ elsif new_value.is_a?(Array) && !old_value.is_a?(Array)
+ (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED')
+
+ # merge two arrays
+ elsif old_value.is_a?(Array) && new_value.is_a?(Array)
+ if mode == :add
+ value = (old_value + new_value).sort.uniq
+ elsif mode == :subtract
+ value = new_value - old_value
+ elsif prefer_self
+ value = old_value
+ else
+ value = new_value
+ end
+
+ # catch errors
+ elsif type_mismatch?(old_value, new_value)
+ raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [
+ old_value.inspect, old_value.class,
+ new_value.inspect, new_value.class,
+ key, self.class
+ ]
+
+ # merge simple strings & numbers
+ else
+ if prefer_self
+ value = old_value
+ else
+ value = new_value
+ end
+ end
+
+ # save value
+ self[key] = value
+ end
+ self
+ end
+
+ def set_environment(env, node)
+ @env = env
+ @node = node
+ self.each do |key, value|
+ if value.is_a?(Config::Object)
+ value.set_environment(env, node)
+ end
+ end
+ end
+
+ #
+ # like a reverse deep merge
+ # (self takes precedence)
+ #
+ def inherit_from!(object)
+ self.deep_merge!(object, true)
+ end
+
+ #
+ # Make a copy of ourselves, except only including the specified keys.
+ #
+ # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+ #
+ def pick(*keys)
+ keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+ value = self.get(key)
+ if !value.nil?
+ hsh[key.gsub('.','_')] = value
+ end
+ hsh
+ end
+ end
+
+ def eval_file(filename)
+ evaluate_ruby(filename, File.read(filename))
+ end
+
+ protected
+
+ #
+ # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ')
+ #
+ def evaluate_everything(context)
+ keys.each do |key|
+ obj = fetch_value(key, context)
+ if is_required_value_not_set?(obj)
+ Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+ elsif obj.is_a? Config::Object
+ obj.evaluate_everything(context)
+ end
+ end
+ end
+
+ #
+ # some keys need to be evaluated 'late', after all the other keys have been evaluated.
+ #
+ def late_evaluate_everything(context)
+ if @late_eval_list
+ @late_eval_list.each do |key, value|
+ self[key] = context.evaluate_ruby(key, value)
+ if is_required_value_not_set?(self[key])
+ Util::log 0, :warning, "required property \"#{key}\" is not set in node \"#{node.name}\"."
+ end
+ end
+ end
+ values.each do |obj|
+ if obj.is_a? Config::Object
+ obj.late_evaluate_everything(context)
+ end
+ end
+ end
+
+ #
+ # evaluates the string `value` as ruby in the context of self.
+ # (`key` is just passed for debugging purposes)
+ #
+ def evaluate_ruby(key, value)
+ self.instance_eval(value, key, 1)
+ rescue ConfigError => exc
+ raise exc # pass through
+ rescue SystemStackError => exc
+ Util::log 0, :error, "while evaluating node '#{self.name}'"
+ Util::log 0, "offending key: #{key}", :indent => 1
+ Util::log 0, "offending string: #{value}", :indent => 1
+ Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
+ raise SystemExit.new(1)
+ rescue FileMissing => exc
+ Util::bail! do
+ if exc.options[:missing]
+ Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path)
+ else
+ Util::log :error, "while evaluating node '#{self.name}'"
+ Util::log "offending key: #{key}", :indent => 1
+ Util::log "offending string: #{value}", :indent => 1
+ Util::log "error message: no file '#{exc}'", :indent => 1
+ end
+ raise exc if DEBUG
+ end
+ rescue AssertionFailed => exc
+ Util.bail! do
+ Util::log :failed, "assertion while evaluating node '#{self.name}'"
+ Util::log 'assertion: %s' % exc.assertion, :indent => 1
+ Util::log "offending key: #{key}", :indent => 1
+ raise exc if DEBUG
+ end
+ rescue SyntaxError, StandardError => exc
+ Util::bail! do
+ Util::log :error, "while evaluating node '#{self.name}'"
+ Util::log "offending key: #{key}", :indent => 1
+ Util::log "offending string: #{value}", :indent => 1
+ Util::log "error message: #{exc.inspect}", :indent => 1
+ raise exc if DEBUG
+ end
+ end
+
+ private
+
+ #
+ # fetches the value for the key, evaluating the value as ruby if it begins with '='
+ #
+ def fetch_value(key, context=@node)
+ value = fetch(key, nil)
+ if value.is_a?(String) && value =~ /^=/
+ if value =~ /^=> (.*)$/
+ value = evaluate_later(key, $1)
+ elsif value =~ /^= (.*)$/
+ value = context.evaluate_ruby(key, $1)
+ end
+ self[key] = value
+ end
+ return value
+ end
+
+ def evaluate_later(key, value)
+ @late_eval_list ||= []
+ @late_eval_list << [key, value]
+ '<evaluate later>'
+ end
+
+ #
+ # when merging, we raise an error if this method returns true for the two values.
+ #
+ def type_mismatch?(old_value, new_value)
+ if old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
+ # note: FalseClass and TrueClass are different classes
+ # so we can't do old_value.class == new_value.class
+ return false
+ elsif old_value.is_a?(String) && old_value =~ /^=/
+ # pass through macros, since we don't know what the type will eventually be.
+ return false
+ elsif new_value.is_a?(String) && new_value =~ /^=/
+ return false
+ elsif old_value.class == new_value.class
+ return false
+ else
+ return true
+ end
+ end
+
+ #
+ # returns true if the value has not been changed and the default is "REQUIRED"
+ #
+ def is_required_value_not_set?(value)
+ if value.is_a? Array
+ value == ["REQUIRED"]
+ else
+ value == "REQUIRED"
+ end
+ end
+
+ end # class
+ end # module
+end # module
\ No newline at end of file
diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f9299a611de0b9511757d1940da0c126e72295e3
--- /dev/null
+++ b/lib/leap_cli/config/object_list.rb
@@ -0,0 +1,209 @@
+require 'tsort'
+
+module LeapCli
+ module Config
+ #
+ # A list of Config::Object instances (internally stored as a hash)
+ #
+ class ObjectList < Hash
+ include TSort
+
+ def initialize(config=nil)
+ if config
+ self.add(config['name'], config)
+ end
+ end
+
+ #
+ # If the key is a string, the Config::Object it references is returned.
+ #
+ # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition.
+ # A new ObjectList is returned.
+ #
+ # Examples:
+ #
+ # nodes['vpn1']
+ # node named 'vpn1'
+ #
+ # nodes[:public_dns => true]
+ # all nodes with public dns
+ #
+ # nodes[:services => 'openvpn', 'location.country_code' => 'US']
+ # all nodes with services containing 'openvpn' OR country code of US
+ #
+ # Sometimes, you want to do an OR condition with multiple conditions
+ # for the same field. Since hash keys must be unique, you can use
+ # an array representation instead:
+ #
+ # nodes[[:services, 'openvpn'], [:services, 'tor']]
+ # nodes with openvpn OR tor service
+ #
+ # nodes[:services => 'openvpn'][:tags => 'production']
+ # nodes with openvpn AND are production
+ #
+ def [](key)
+ if key.is_a?(Hash) || key.is_a?(Array)
+ filter(key)
+ else
+ super key.to_s
+ end
+ end
+
+ def exclude(node)
+ list = self.dup
+ list.delete(node.name)
+ return list
+ end
+
+ def each_node(&block)
+ self.keys.sort.each do |node_name|
+ yield self[node_name]
+ end
+ end
+
+ #
+ # filters this object list, producing a new list.
+ # filter is an array or a hash. see []
+ #
+ def filter(filter)
+ results = Config::ObjectList.new
+ filter.each do |field, match_value|
+ field = field.is_a?(Symbol) ? field.to_s : field
+ match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
+ if match_value.is_a?(String) && match_value =~ /^!/
+ operator = :not_equal
+ match_value = match_value.sub(/^!/, '')
+ else
+ operator = :equal
+ end
+ each do |name, config|
+ value = config[field]
+ if value.is_a? Array
+ if operator == :equal && value.include?(match_value)
+ results[name] = config
+ elsif operator == :not_equal && !value.include?(match_value)
+ results[name] = config
+ end
+ else
+ if operator == :equal && value == match_value
+ results[name] = config
+ elsif operator == :not_equal && value != match_value
+ results[name] = config
+ end
+ end
+ end
+ end
+ results
+ end
+
+ def add(name, object)
+ self[name] = object
+ end
+
+ #
+ # converts the hash of configs into an array of hashes, with ONLY the specified fields
+ #
+ def fields(*fields)
+ result = []
+ keys.sort.each do |name|
+ result << self[name].pick(*fields)
+ end
+ result
+ end
+
+ #
+ # like fields(), but returns an array of values instead of an array of hashes.
+ #
+ def field(field)
+ field = field.to_s
+ result = []
+ keys.sort.each do |name|
+ result << self[name].get(field)
+ end
+ result
+ end
+
+ #
+ # pick_fields(field1, field2, ...)
+ #
+ # generates a Hash from the object list, but with only the fields that are picked.
+ #
+ # If there are more than one field, then the result is a Hash of Hashes.
+ # If there is just one field, it is a simple map to the value.
+ #
+ # For example:
+ #
+ # "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')"
+ #
+ # generates this:
+ #
+ # neighbors:
+ # couch1:
+ # domain_full: couch1.bitmask.net
+ # ip_address: "10.5.5.44"
+ # couch2:
+ # domain_full: couch2.bitmask.net
+ # ip_address: "10.5.5.52"
+ #
+ # But this:
+ #
+ # "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')"
+ #
+ # will generate this:
+ #
+ # neighbors:
+ # couch1: couch1.bitmask.net
+ # couch2: couch2.bitmask.net
+ #
+ def pick_fields(*fields)
+ self.values.inject({}) do |hsh, node|
+ value = self[node.name].pick(*fields)
+ if fields.size == 1
+ value = value.values.first
+ end
+ hsh[node.name] = value
+ hsh
+ end
+ end
+
+ #
+ # Applies inherit_from! to all objects.
+ #
+ # 'env' specifies what environment should be for
+ # each object in the list.
+ #
+ def inherit_from!(object_list, env)
+ object_list.each do |name, object|
+ if self[name]
+ self[name].inherit_from!(object)
+ else
+ self[name] = object.duplicate(env)
+ end
+ end
+ end
+
+ #
+ # topographical sort based on test dependency
+ #
+ def tsort_each_node(&block)
+ self.each_key(&block)
+ end
+
+ def tsort_each_child(node_name, &block)
+ if self[node_name]
+ self[node_name].test_dependencies.each do |test_me_first|
+ if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies.
+ # not just the ones that pass the node filter.
+ yield(test_me_first)
+ end
+ end
+ end
+ end
+
+ def names_in_test_dependency_order
+ self.tsort
+ end
+
+ end
+ end
+end
diff --git a/lib/leap_cli/config/provider.rb b/lib/leap_cli/config/provider.rb
new file mode 100644
index 0000000000000000000000000000000000000000..0d8bc1f3f59dfce473eba4af8d46fadba71518e1
--- /dev/null
+++ b/lib/leap_cli/config/provider.rb
@@ -0,0 +1,22 @@
+#
+# Configuration class for provider.json
+#
+
+module LeapCli; module Config
+ class Provider < Object
+ attr_reader :environment
+ def set_env(e)
+ if e == 'default'
+ @environment = nil
+ else
+ @environment = e
+ end
+ end
+ def provider
+ self
+ end
+ def validate!
+ # nothing here yet :(
+ end
+ end
+end; end
diff --git a/lib/leap_cli/config/secrets.rb b/lib/leap_cli/config/secrets.rb
new file mode 100644
index 0000000000000000000000000000000000000000..ca851c743718fe03c29da130663de29931ae5ae6
--- /dev/null
+++ b/lib/leap_cli/config/secrets.rb
@@ -0,0 +1,87 @@
+# encoding: utf-8
+#
+# A class for the secrets.json file
+#
+
+module LeapCli; module Config
+
+ class Secrets < Object
+ attr_reader :node_list
+
+ def initialize(manager=nil)
+ super(manager)
+ @discovered_keys = {}
+ end
+
+ # we can't use fetch() or get(), since those already have special meanings
+ def retrieve(key, environment)
+ environment ||= 'default'
+ self.fetch(environment, {})[key.to_s]
+ end
+
+ def set(*args, &block)
+ if block_given?
+ set_with_block(*args, &block)
+ else
+ set_without_block(*args)
+ end
+ end
+
+ # searches over all keys matching the regexp, checking to see if the value
+ # has been already used by any of them.
+ def taken?(regexp, value, environment)
+ self.keys.grep(regexp).each do |key|
+ return true if self.retrieve(key, environment) == value
+ end
+ return false
+ end
+
+ def set_without_block(key, value, environment)
+ set_with_block(key, environment) {value}
+ end
+
+ def set_with_block(key, environment, &block)
+ environment ||= 'default'
+ key = key.to_s
+ @discovered_keys[environment] ||= {}
+ @discovered_keys[environment][key] = true
+ self[environment] ||= {}
+ self[environment][key] ||= yield
+ end
+
+ #
+ # if clean is true, then only secrets that have been discovered
+ # during this run will be exported.
+ #
+ # if environment is also pinned, then we will clean those secrets
+ # just for that environment.
+ #
+ # the clean argument should only be used when all nodes have
+ # been processed, otherwise secrets that are actually in use will
+ # get mistakenly removed.
+ #
+ def dump_json(clean=false)
+ pinned_env = LeapCli.leapfile.environment
+ if clean
+ self.each_key do |environment|
+ if pinned_env.nil? || pinned_env == environment
+ env = self[environment]
+ if env.nil?
+ raise StandardError.new("secrets.json file seems corrupted. No such environment '#{environment}'")
+ end
+ env.each_key do |key|
+ unless @discovered_keys[environment] && @discovered_keys[environment][key]
+ self[environment].delete(key)
+ end
+ end
+ if self[environment].empty?
+ self.delete(environment)
+ end
+ end
+ end
+ end
+ super()
+ end
+ end
+
+end; end
diff --git a/lib/leap_cli/config/sources.rb b/lib/leap_cli/config/sources.rb
new file mode 100644
index 0000000000000000000000000000000000000000..aee860de8c035883f749bf138865ac5e175ef653
--- /dev/null
+++ b/lib/leap_cli/config/sources.rb
@@ -0,0 +1,11 @@
+# encoding: utf-8
+#
+# A class for the sources.json file
+#
+
+module LeapCli
+ module Config
+ class Sources < Object
+ end
+ end
+end
diff --git a/lib/leap_cli/config/tag.rb b/lib/leap_cli/config/tag.rb
new file mode 100644
index 0000000000000000000000000000000000000000..6bd8d1e91366a03284d4f0817ecc8953961dfd3f
--- /dev/null
+++ b/lib/leap_cli/config/tag.rb
@@ -0,0 +1,25 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+ class Tag < Object
+ attr_reader :node_list
+
+ def initialize(environment=nil)
+ super(environment)
+ @node_list = Config::ObjectList.new
+ end
+
+ # don't copy the node list pointer when this object is dup'ed.
+ def initialize_copy(orig)
+ super
+ @node_list = Config::ObjectList.new
+ end
+
+ end
+
+end; end
diff --git a/lib/leap_cli/leapfile_extensions.rb b/lib/leap_cli/leapfile_extensions.rb
new file mode 100644
index 0000000000000000000000000000000000000000..cba321f498bd340ee00beb292b43ecd61843ac73
--- /dev/null
+++ b/lib/leap_cli/leapfile_extensions.rb
@@ -0,0 +1,24 @@
+module LeapCli
+ class Leapfile
+ attr_reader :custom_vagrant_vm_line
+ attr_reader :leap_version
+ attr_reader :log
+ attr_reader :vagrant_basebox
+
+ def vagrant_network
+ @vagrant_network ||= '10.5.5.0/24'
+ end
+
+ private
+
+ PRIVATE_IP_RANGES = /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/
+
+ def validate
+ Util::assert! vagrant_network =~ PRIVATE_IP_RANGES do
+ Util::log 0, :error, "in #{file}: vagrant_network is not a local private network"
+ end
+ return true
+ end
+
+ end
+end
diff --git a/lib/leap_cli/load_libraries.rb b/lib/leap_cli/load_libraries.rb
new file mode 100644
index 0000000000000000000000000000000000000000..19f4edb53ae3bb5ec7cf613d69a72a75ca865932
--- /dev/null
+++ b/lib/leap_cli/load_libraries.rb
@@ -0,0 +1,20 @@
+#
+# load the commonly needed leap_cli libraries that live in the platform.
+#
+# loaded by leap_cli's bootstrap.rb
+#
+
+require 'leap_cli/log_filter'
+
+require 'leap_cli/config/object'
+require 'leap_cli/config/node'
+require 'leap_cli/config/tag'
+require 'leap_cli/config/provider'
+require 'leap_cli/config/secrets'
+require 'leap_cli/config/object_list'
+require 'leap_cli/config/filter'
+require 'leap_cli/config/environment'
+require 'leap_cli/config/manager'
+
+require 'leap_cli/util/secret'
+require 'leap_cli/util/x509'
diff --git a/lib/leap_cli/log_filter.rb b/lib/leap_cli/log_filter.rb
new file mode 100644
index 0000000000000000000000000000000000000000..0d745cc2b2845b079d75f98c41cf147491cdf36a
--- /dev/null
+++ b/lib/leap_cli/log_filter.rb
@@ -0,0 +1,171 @@
+#
+# A module to hide, modify, and colorize log entries.
+#
+
+module LeapCli
+ module LogFilter
+ #
+ # options for formatters:
+ #
+ # :match => regexp for matching a log line
+ # :color => what color the line should be
+ # :style => what style the line should be
+ # :priority => what order the formatters are applied in. higher numbers first.
+ # :match_level => only apply filter at the specified log level
+ # :level => make this line visible at this log level or higher
+ # :replace => replace the matched text
+ # :prepend => insert text at start of message
+ # :append => append text to end of message
+ # :exit => force the exit code to be this (does not interrupt program, just
+ # ensures a specific exit code when the program eventually exits)
+ #
+ FORMATTERS = [
+ # TRACE
+ { :match => /command finished/, :color => :white, :style => :dim, :match_level => 3, :priority => -10 },
+ { :match => /executing locally/, :color => :yellow, :match_level => 3, :priority => -20 },
+
+ # DEBUG
+ #{ :match => /executing .*/, :color => :green, :match_level => 2, :priority => -10, :timestamp => true },
+ #{ :match => /.*/, :color => :yellow, :match_level => 2, :priority => -30 },
+ { :match => /^transaction:/, :level => 3 },
+
+ # INFO
+ { :match => /.*out\] (fatal:|ERROR:).*/, :color => :red, :match_level => 1, :priority => -10 },
+ { :match => /Permission denied/, :color => :red, :match_level => 1, :priority => -20 },
+ { :match => /sh: .+: command not found/, :color => :magenta, :match_level => 1, :priority => -30 },
+
+ # IMPORTANT
+ { :match => /^(E|e)rr ::/, :color => :red, :match_level => 0, :priority => -10, :exit => 1},
+ { :match => /^ERROR:/, :color => :red, :priority => -10, :exit => 1},
+ #{ :match => /.*/, :color => :blue, :match_level => 0, :priority => -20 },
+
+ # CLEANUP
+ #{ :match => /\s+$/, :replace => '', :priority => 0},
+
+ # DEBIAN PACKAGES
+ { :match => /^(Hit|Ign) /, :color => :green, :priority => -20},
+ { :match => /^Err /, :color => :red, :priority => -20},
+ { :match => /^W(ARNING)?: /, :color => :yellow, :priority => -20},
+ { :match => /^E: /, :color => :red, :priority => -20},
+ { :match => /already the newest version/, :color => :green, :priority => -20},
+ { :match => /WARNING: The following packages cannot be authenticated!/, :color => :red, :level => 0, :priority => -10},
+
+ # PUPPET
+ { :match => /^(W|w)arning: Not collecting exported resources without storeconfigs/, :level => 2, :color => :yellow, :priority => -10},
+ { :match => /^(W|w)arning: Found multiple default providers for vcsrepo:/, :level => 2, :color => :yellow, :priority => -10},
+ { :match => /^(W|w)arning: .*is deprecated.*$/, :level => 2, :color => :yellow, :priority => -10},
+ { :match => /^(W|w)arning: Scope.*$/, :level => 2, :color => :yellow, :priority => -10},
+ #{ :match => /^(N|n)otice:/, :level => 1, :color => :cyan, :priority => -20},
+ #{ :match => /^(N|n)otice:.*executed successfully$/, :level => 2, :color => :cyan, :priority => -15},
+ { :match => /^(W|w)arning:/, :level => 0, :color => :yellow, :priority => -20},
+ { :match => /^Duplicate declaration:/, :level => 0, :color => :red, :priority => -20},
+ #{ :match => /Finished catalog run/, :level => 0, :color => :green, :priority => -10},
+ { :match => /^APPLY COMPLETE \(changes made\)/, :level => 0, :color => :green, :style => :bold, :priority => -10},
+ { :match => /^APPLY COMPLETE \(no changes\)/, :level => 0, :color => :green, :style => :bold, :priority => -10},
+
+ # PUPPET FATAL ERRORS
+ { :match => /^(E|e)rr(or|):/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Wrapped exception:/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Failed to parse template/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Execution of.*returned/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Parameter matches failed:/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Syntax error/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Cannot reassign variable/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^Could not find template/, :level => 0, :color => :red, :priority => -1, :exit => 1},
+ { :match => /^APPLY COMPLETE.*fail/, :level => 0, :color => :red, :style => :bold, :priority => -1, :exit => 1},
+
+ # TESTS
+ { :match => /^PASS: /, :color => :green, :priority => -20},
+ { :match => /^(FAIL|ERROR): /, :color => :red, :priority => -20},
+ { :match => /^(SKIP|WARN): /, :color => :yellow, :priority => -20},
+ { :match => /\d+ tests: \d+ passes, \d+ skips, 0 warnings, 0 failures, 0 errors/,
+ :color => :green, :style => :bold, :priority => -20 },
+ { :match => /\d+ tests: \d+ passes, \d+ skips, [1-9][0-9]* warnings, 0 failures, 0 errors/,
+ :color => :yellow, :style => :bold, :priority => -20 },
+ { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, \d+ failures, [1-9][0-9]* errors/,
+ :color => :red, :style => :bold, :priority => -20 },
+ { :match => /\d+ tests: \d+ passes, \d+ skips, \d+ warnings, [1-9][0-9]* failures, \d+ errors/,
+ :color => :red, :style => :bold, :priority => -20 },
+
+ # LOG SUPPRESSION
+ { :match => /^(W|w)arning: You cannot collect without storeconfigs being set/, :level => 2, :priority => 10},
+ { :match => /^(W|w)arning: You cannot collect exported resources without storeconfigs being set/, :level => 2, :priority => 10}
+ ]
+
+ SORTED_FORMATTERS = FORMATTERS.sort_by { |i| -(i[:priority] || i[:prio] || 0) }
+
+ #
+ # same as normal formatters, but only applies to the title, not the message.
+ #
+ TITLE_FORMATTERS = [
+ # red
+ { :match => /error/, :color => :red, :style => :bold },
+ { :match => /fatal_error/, :replace => 'fatal error:', :color => :red, :style => :bold },
+ { :match => /removed/, :color => :red, :style => :bold },
+ { :match => /failed/, :replace => 'FAILED', :color => :red, :style => :bold },
+ { :match => /bail/, :replace => 'bailing out', :color => :red, :style => :bold },
+ { :match => /invalid/, :color => :red, :style => :bold },
+
+ # yellow
+ { :match => /warning/, :replace => 'warning:', :color => :yellow, :style => :bold },
+ { :match => /missing/, :color => :yellow, :style => :bold },
+ { :match => /skipping/, :color => :yellow, :style => :bold },
+
+ # green
+ { :match => /created/, :color => :green, :style => :bold },
+ { :match => /completed/, :color => :green, :style => :bold },
+ { :match => /ran/, :color => :green, :style => :bold },
+
+ # cyan
+ { :match => /note/, :replace => 'NOTE:', :color => :cyan, :style => :bold },
+
+ # magenta
+ { :match => /nochange/, :replace => 'no change', :color => :magenta },
+ { :match => /loading/, :color => :magenta },
+ ]
+
+ def self.apply_message_filters(message)
+ return self.apply_filters(SORTED_FORMATTERS, message)
+ end
+
+ def self.apply_title_filters(title)
+ return self.apply_filters(TITLE_FORMATTERS, title)
+ end
+
+ private
+
+ def self.apply_filters(formatters, message)
+ level = LeapCli.logger.log_level
+ result = {}
+ formatters.each do |formatter|
+ if (formatter[:match_level] == level || formatter[:match_level].nil?)
+ if message =~ formatter[:match]
+ # puts "applying formatter #{formatter.inspect}"
+ result[:level] = formatter[:level] if formatter[:level]
+ result[:color] = formatter[:color] if formatter[:color]
+ result[:style] = formatter[:style] || formatter[:attribute] # (support original cap colors)
+
+ message.gsub!(formatter[:match], formatter[:replace]) if formatter[:replace]
+ message.replace(formatter[:prepend] + message) unless formatter[:prepend].nil?
+ message.replace(message + formatter[:append]) unless formatter[:append].nil?
+ message.replace(Time.now.strftime('%Y-%m-%d %T') + ' ' + message) if formatter[:timestamp]
+
+ if formatter[:exit]
+ LeapCli::Util.exit_status(formatter[:exit])
+ end
+
+ # stop formatting, unless formatter was just for string replacement
+ break unless formatter[:replace]
+ end
+ end
+ end
+
+ if result[:color] == :hide
+ return [nil, {}]
+ else
+ return [message, result]
+ end
+ end
+
+ end
+end
diff --git a/lib/leap_cli/macros.rb b/lib/leap_cli/macros.rb
deleted file mode 100644
index fdb9a94e23615bcdd02cde9b905fe599da1e9038..0000000000000000000000000000000000000000
--- a/lib/leap_cli/macros.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# MACROS
-#
-# The methods in these files are available in the context of a .json configuration file.
-# (The module LeapCli::Macro is included in Config::Object)
-#
-
-require_relative 'macros/core'
-require_relative 'macros/files'
-require_relative 'macros/haproxy'
-require_relative 'macros/hosts'
-require_relative 'macros/keys'
-require_relative 'macros/nodes'
-require_relative 'macros/secrets'
-require_relative 'macros/stunnel'
-require_relative 'macros/provider'
diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb
index f42379cc96cad32d78ea4383fb9638da608b91a7..67c6ec9a0dca589395a35aa2c19d048b95e6d16e 100644
--- a/lib/leap_cli/ssh/backend.rb
+++ b/lib/leap_cli/ssh/backend.rb
@@ -94,7 +94,7 @@ module LeapCli
# some prewritten servers-side scripts
def scripts
- @scripts ||= LeapCli::SSH::Scripts.new(self, @host)
+ @scripts ||= LeapCli::SSH::Scripts.new(self, @host.hostname)
end
private
@@ -139,6 +139,9 @@ module LeapCli
@logger.log(:failed, args.join(' '), host: @host.hostname) do
@logger.log("Connection timed out")
end
+ if @options[:raise_error]
+ raise LeapCli::SSH::TimeoutError, exc.to_s
+ end
else
raise
end
diff --git a/lib/leap_cli/ssh/remote_command.rb b/lib/leap_cli/ssh/remote_command.rb
index fe9a344afc18edc511624c8c4fb64586ea75b443..3ba86740d88ab203c4360aca128a0fbb261b4166 100644
--- a/lib/leap_cli/ssh/remote_command.rb
+++ b/lib/leap_cli/ssh/remote_command.rb
@@ -21,6 +21,9 @@ module LeapCli
class ExecuteError < StandardError
end
+ class TimeoutError < ExecuteError
+ end
+
# override default runner mode
class CustomCoordinator < SSHKit::Coordinator
private
diff --git a/lib/leap_cli/ssh/scripts.rb b/lib/leap_cli/ssh/scripts.rb
index 3d8b6570f495543ff8a9a6997305d200c3869b0b..feefdd46c3c3eaa13e272beca50cef6bc91d17af 100644
--- a/lib/leap_cli/ssh/scripts.rb
+++ b/lib/leap_cli/ssh/scripts.rb
@@ -15,7 +15,7 @@ module LeapCli
REQUIRED_PACKAGES = "puppet rsync lsb-release locales"
attr_reader :ssh, :host
- def initialize(backend, host)
+ def initialize(backend, hostname)
@ssh = backend
@host = host
end
@@ -48,6 +48,8 @@ module LeapCli
def check_for_no_deploy
begin
ssh.stream "test ! -f /etc/leap/no-deploy", :raise_error => true, :log_output => false
+ rescue SSH::TimeoutError
+ raise
rescue SSH::ExecuteError
ssh.log :warning, "can't continue because file /etc/leap/no-deploy exists", :host => host
raise # will skip further action on this node
@@ -59,7 +61,7 @@ module LeapCli
#
def debug
output = ssh.capture "#{Leap::Platform.leap_dir}/bin/debug.sh"
- ssh.log(output, :wrap => true, :host => host.hostname, :color => :cyan)
+ ssh.log(output, :wrap => true, :host => host, :color => :cyan)
end
#
@@ -69,7 +71,7 @@ module LeapCli
cmd = "(test -s /var/log/leap/deploy-summary.log && tail -n #{lines} /var/log/leap/deploy-summary.log) || (test -s /var/log/leap/deploy-summary.log.1 && tail -n #{lines} /var/log/leap/deploy-summary.log.1) || (echo 'no history')"
history = ssh.capture(cmd, :log_output => false)
if history
- ssh.log host.hostname, :color => :cyan, :style => :bold do
+ ssh.log host, :color => :cyan, :style => :bold do
ssh.log history, :wrap => true
end
end
diff --git a/lib/leap_cli/util/secret.rb b/lib/leap_cli/util/secret.rb
new file mode 100644
index 0000000000000000000000000000000000000000..749b95957c111cb22102e59ea393af68964d4a62
--- /dev/null
+++ b/lib/leap_cli/util/secret.rb
@@ -0,0 +1,55 @@
+# encoding: utf-8
+#
+# A simple secret generator
+#
+# Uses OpenSSL random number generator instead of Ruby's rand function
+#
+autoload :OpenSSL, 'openssl'
+
+module LeapCli; module Util
+ class Secret
+ CHARS = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a) - "i1loO06G".split(//u)
+ HEX = (0..9).to_a + ('a'..'f').to_a
+
+ #
+ # generate a secret with with no ambiguous characters.
+ #
+ # +length+ is in chars
+ #
+ # Only alphanumerics are allowed, in order to make these passwords work
+ # for REST url calls and to allow you to easily copy and paste them.
+ #
+ def self.generate(length = 16)
+ seed
+ OpenSSL::Random.random_bytes(length).bytes.to_a.collect { |byte|
+ CHARS[ byte % CHARS.length ]
+ }.join
+ end
+
+ #
+ # generates a hex secret, instead of an alphanumeric on.
+ #
+ # length is in bits
+ #
+ def self.generate_hex(length = 128)
+ seed
+ OpenSSL::Random.random_bytes(length/4).bytes.to_a.collect { |byte|
+ HEX[ byte % HEX.length ]
+ }.join
+ end
+
+ private
+
+ def self.seed
+ @pid ||= 0
+ pid = $$
+ if @pid != pid
+ now = Time.now
+ ary = [now.to_i, now.nsec, @pid, pid]
+ OpenSSL::Random.seed(ary.to_s)
+ @pid = pid
+ end
+ end
+
+ end
+end; end
diff --git a/lib/leap_cli/util/x509.rb b/lib/leap_cli/util/x509.rb
new file mode 100644
index 0000000000000000000000000000000000000000..787fdfac7c410af904cc45964989a1e17bacbe80
--- /dev/null
+++ b/lib/leap_cli/util/x509.rb
@@ -0,0 +1,33 @@
+autoload :OpenSSL, 'openssl'
+autoload :CertificateAuthority, 'certificate_authority'
+
+require 'digest'
+require 'digest/md5'
+require 'digest/sha1'
+
+module LeapCli; module X509
+ extend self
+
+ #
+ # returns a fingerprint of a x509 certificate
+ #
+ def fingerprint(digest, cert_file)
+ if cert_file.is_a? String
+ cert = OpenSSL::X509::Certificate.new(Util.read_file!(cert_file))
+ elsif cert_file.is_a? OpenSSL::X509::Certificate
+ cert = cert_file
+ elsif cert_file.is_a? CertificateAuthority::Certificate
+ cert = cert_file.openssl_body
+ end
+ digester = case digest
+ when "MD5" then Digest::MD5.new
+ when "SHA1" then Digest::SHA1.new
+ when "SHA256" then Digest::SHA256.new
+ when "SHA384" then Digest::SHA384.new
+ when "SHA512" then Digest::SHA512.new
+ end
+ digester.hexdigest(cert.to_der)
+ end
+
+
+end; end