Merge pull request #194 from cloudamatic/development

tfw you're addicted to microservices

Dockerfile

@@ -8,7 +8,7 @@ RUN df -h
 
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client
 
 RUN apt-get upgrade -y
 

bin/mu-adopt

@@ -45,6 +45,7 @@ $opt = Optimist::options do
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
   opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
+  opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string
 end
 
 ok = true
@@ -60,6 +61,16 @@ if $opt[:diff]
   $opt[:savedeploys] = false
 end
 
+pattern = nil
+if $opt[:pattern]
+  begin
+    pattern = Regexp.new($opt[:pattern], true)
+  rescue RegexpError => e
+    MU.log "Invalid --pattern option: #{e.message}", MU::ERR
+    exit 1
+  end
+end
+
 types = []
 $opt[:types].each { |t|
   t_name = t.gsub(/-/, "_")
@@ -98,7 +109,7 @@ if !ok
   exit 1
 end
 
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern)
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }

bin/mu-load-config.rb

@@ -134,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     }
   end
 
-  global_cfg = { "config_files" => [] }
+  global_cfg = { "config_files" => [], "overridden_keys" => [] }
   if File.exist?(cfgPath)
     global_cfg = YAML.load(File.read(cfgPath))
     global_cfg["config_files"] = [cfgPath]
@@ -147,6 +147,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     if localfile
       global_cfg.merge!(localfile)
       global_cfg["config_files"] << "#{home}/.mu.yaml"
+      global_cfg["overridden_keys"] = localfile.keys
     end
   end
   if !global_cfg.has_key?("installdir")

bin/mu-run-tests

@@ -34,6 +34,7 @@ Usage:
 #{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
   EOS
   opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
+  opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
   opt :verbose, "Show more information while running", :require => false, :default => false
 end
@@ -121,8 +122,19 @@ def execCommand(cmd, results_stash)
   }
 
   ok = true
-  output = %x{#{cmd} 2>&1}
-  ok = false if $?.exitstatus != 0
+  retries = 0
+  begin
+    output = %x{#{cmd} 2>&1}
+    if $?.exitstatus != 0
+      ok = false
+      retries += 1
+      if $opts[:verbose] and !$opts[:full] and retries <= $opts[:max_retries]
+        puts "#{cmd} RETRY #{retries.to_s}".light_red
+      end
+    else
+      ok = true
+    end
+  end while !ok and !$opts[:full] and retries <= $opts[:max_retries]
 
   results_stash["output"] += output
 

cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.2.0'
-  s.date        = '2020-06-16'
+  s.version     = '3.3.0'
+  s.date        = '2020-09-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -57,7 +57,7 @@ EOF
   s.add_runtime_dependency 'rack', "~> 2.0"
   s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
-  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'rubyzip', "~> 2.3"
   s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
   s.add_runtime_dependency 'solve', '~> 4.0'