From ddd71216501e0c110ecbc7954783cd04316b4706 Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Mon, 12 Nov 2018 15:28:05 +1300 Subject: [PATCH 1/7] generate_types support for MCO r10k plugin --- files/agent/r10k.ddl | 32 ++++++++++++++++++++++++++++++++ files/application/r10k.rb | 17 +++++++++++------ templates/agent/r10k.rb.erb | 11 +++++++++-- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/files/agent/r10k.ddl b/files/agent/r10k.ddl index 0f154adb..0cc93189 100644 --- a/files/agent/r10k.ddl +++ b/files/agent/r10k.ddl @@ -126,4 +126,36 @@ action 'deploy_module', :description => "Deploy a specific module" do display :always end + +action 'generate_types', :description => "Generate metadata files" do + input :environment, + :prompt => "Specific environment", + :description => "Generate metadata files for a particular environment", + :type => :string, + :validation => '.', + :optional => true, + :maxlength => 256 + + input :user, + :prompt => "User", + :description => "User to run as", + :type => :string, + :validation => '\w+', + :optional => true, + :maxlength => 32 + + output :environment, + :description => "Generate metadata files for a particular environment", + :display_as => "Specific environment" + + output :output, + :description => "Output from r10k", + :display_as => "Output" + + output :error, + :description => "Error from r10k", + :display_as => "Errors" + + display :always +end # vim: set syntax=ruby: diff --git a/files/application/r10k.rb b/files/application/r10k.rb index aebf6992..e1ee224c 100644 --- a/files/application/r10k.rb +++ b/files/application/r10k.rb @@ -5,30 +5,35 @@ def post_option_parser(configuration) case configuration[:command] when 'push','pull','status' configuration[:path] = ARGV.shift || docs - when 'deploy' + when 'deploy', 'generate_types' configuration[:environment] = ARGV.shift || docs when 'deploy_module' configuration[:module_name] = ARGV.shift || docs end - else + else docs end end def docs - puts "Usage: #{$0} push | pull | status | deploy | deploy_module" + puts "Usage: #{$0} push | pull | status | deploy | deploy_module | generate_types" end def main mc = rpcclient("r10k", :chomp => true) - options = {:path => configuration[:path]} if ['push','pull','status'].include? configuration[:command] - options = {:environment => configuration[:environment]} if ['deploy'].include? configuration[:command] + options = {:path => configuration[:path]} if ['push','pull','status'] + .include? configuration[:command] + options = {:environment => configuration[:environment]} if [ + 'deploy', 'generate_types' + ].include? configuration[:command] options = {:module_name => configuration[:module_name]} if ['deploy_module'].include? configuration[:command] mc.send(configuration[:command], options).each do |resp| puts "#{resp[:sender]}:" if resp[:statuscode] == 0 responses = resp[:data][:output] - puts responses if responses and ['push','pull','status','deploy','deploy_module'].include? configuration[:command] + puts responses if responses and [ + 'push','pull','status','deploy','deploy_module', 'generate_types' + ].include? configuration[:command] else puts resp[:statusmsg] end diff --git a/templates/agent/r10k.rb.erb b/templates/agent/r10k.rb.erb index 33b8885d..10a7f143 100644 --- a/templates/agent/r10k.rb.erb +++ b/templates/agent/r10k.rb.erb @@ -32,9 +32,10 @@ module MCollective 'synchronize', 'deploy', 'deploy_module', - 'sync'].each do |act| + 'sync', + 'generate_types'].each do |act| action act do - if act == 'deploy' + if ['deploy', 'generate_types'].include? act validate :environment, :shellsafe environment = request[:environment] lock do @@ -76,6 +77,7 @@ module MCollective output = '' git = ['/usr/bin/env', 'git'] r10k = ['/usr/bin/env', 'r10k'] + puppet = ['/usr/bin/env', 'puppet'] # Given most people using this are using Puppet Enterprise, add the PE Path environment = {"LC_ALL" => "C","PATH" => "#{ENV['PATH']}:<%= if @is_pe == true or @is_pe == 'true' then '/opt/puppet/bin' else '/usr/local/bin' end %>", "http_proxy" => "<%= @http_proxy %>", "https_proxy" => "<%= @http_proxy %>", "GIT_SSL_NO_VERIFY" => "<%= @git_ssl_no_verify %>" } case action @@ -94,6 +96,11 @@ module MCollective elsif action == 'deploy_module' cmd << 'deploy' << 'module' << arg end + reply[:status] = run(cmd_as_user(cmd), :stderr => :error, :stdout => :output, :chomp => true, :environment => environment) + when 'generate_types' + cmd = puppet + cmd << 'generate' << 'types' << '--environment' << arg + reply[:status] = run(cmd_as_user(cmd), :stderr => :error, :stdout => :output, :chomp => true, :environment => environment) end end From 4e1aacdfbc5ce6bf2be73a95dde4bbbc523c3edf Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Mon, 12 Nov 2018 15:33:34 +1300 Subject: [PATCH 2/7] change path to /opt/puppetlabs/bin since PE/FOSS < 4 not supprted Check manifests/params.pp for detail --- templates/agent/r10k.rb.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/agent/r10k.rb.erb b/templates/agent/r10k.rb.erb index 10a7f143..fd1deecb 100644 --- a/templates/agent/r10k.rb.erb +++ b/templates/agent/r10k.rb.erb @@ -78,8 +78,7 @@ module MCollective git = ['/usr/bin/env', 'git'] r10k = ['/usr/bin/env', 'r10k'] puppet = ['/usr/bin/env', 'puppet'] - # Given most people using this are using Puppet Enterprise, add the PE Path - environment = {"LC_ALL" => "C","PATH" => "#{ENV['PATH']}:<%= if @is_pe == true or @is_pe == 'true' then '/opt/puppet/bin' else '/usr/local/bin' end %>", "http_proxy" => "<%= @http_proxy %>", "https_proxy" => "<%= @http_proxy %>", "GIT_SSL_NO_VERIFY" => "<%= @git_ssl_no_verify %>" } + environment = {"LC_ALL" => "C","PATH" => "#{ENV['PATH']}:/opt/puppetlabs/bin", "http_proxy" => "<%= @http_proxy %>", "https_proxy" => "<%= @http_proxy %>", "GIT_SSL_NO_VERIFY" => "<%= @git_ssl_no_verify %>" } case action when 'push','pull','status' cmd = git From 161c3c8bdf639e134d44c7c5723e61ad55d42243 Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Tue, 13 Nov 2018 09:20:37 +1300 Subject: [PATCH 3/7] Add generic help doc --- files/agent/r10k.ddl | 2 +- files/application/r10k.rb | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/files/agent/r10k.ddl b/files/agent/r10k.ddl index 0cc93189..773fee48 100644 --- a/files/agent/r10k.ddl +++ b/files/agent/r10k.ddl @@ -127,7 +127,7 @@ action 'deploy_module', :description => "Deploy a specific module" do display :always end -action 'generate_types', :description => "Generate metadata files" do +action 'generate_types', :description => "Generate metadata files of resource types" do input :environment, :prompt => "Specific environment", :description => "Generate metadata files for a particular environment", diff --git a/files/application/r10k.rb b/files/application/r10k.rb index e1ee224c..ee5b734a 100644 --- a/files/application/r10k.rb +++ b/files/application/r10k.rb @@ -1,4 +1,21 @@ class MCollective::Application::R10k +Usage: mco r10k + +The ACTION can be one of the following: + + push - Git push a module + pull - Git pull a module + status - Git status of a module + deploy - Deploy a specific environment, and its Puppetfile specified modules + deploy_module - Deploy a specific module + generate_types - Generate metadata file of resource types + +END_OF_USAGE + def post_option_parser(configuration) if ARGV.length >= 1 configuration[:command] = ARGV.shift From 2f112a8abf77f38f46ce7711bfd7238abed59e1c Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Tue, 13 Nov 2018 09:38:08 +1300 Subject: [PATCH 4/7] Make generate type to fire from MCO --- templates/webhook.bin.erb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/webhook.bin.erb b/templates/webhook.bin.erb index 31e9a015..7488db79 100755 --- a/templates/webhook.bin.erb +++ b/templates/webhook.bin.erb @@ -267,7 +267,11 @@ class Server < Sinatra::Base def generate_types(environment) begin - command = "#{$command_prefix} /opt/puppetlabs/bin/puppet generate types --environment #{environment.gsub(/\W/, '_')}" + if $config['use_mcollective'] + command = "#{$command_prefix} mco r10k generate_types --environment #{environment.gsub(/\W/, '_')}" + else + command = "#{$command_prefix} /opt/puppetlabs/bin/puppet generate types --environment #{environment.gsub(/\W/, '_')}" + end message = run_command(command) $logger.info("message: #{message} environment: #{environment}") From e67b8da3bee953c0493829dfd2b37f3aafca527d Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Tue, 13 Nov 2018 14:29:00 +1300 Subject: [PATCH 5/7] revert partial of commit 90da3cf74 exec will replace current process with the shell sub-process instead of spawning separate new process all executable commands are passed to run_command, there is no much point to fork the process before exec --- templates/webhook.bin.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/webhook.bin.erb b/templates/webhook.bin.erb index 7488db79..ce32489e 100755 --- a/templates/webhook.bin.erb +++ b/templates/webhook.bin.erb @@ -121,7 +121,7 @@ class Server < Sinatra::Base module_name = sanitize_input(module_name) $logger.info("Deploying module #{module_name} in the background.") - Process.detach(fork { deploy_module(module_name) }) + deploy_module(module_name) end # Simulate a github post: @@ -209,7 +209,7 @@ class Server < Sinatra::Base return 200 else $logger.info("Deploying environment #{env} in the background") - Process.detach(fork { deploy(env, deleted) }) + deploy(env, deleted) end end @@ -258,7 +258,7 @@ class Server < Sinatra::Base file.flock(File::LOCK_EX) message = "triggered: #{command}" - exec "#{command} &" + Process.detach(fork { exec "#{command} &" }) exit_status = 0 raise "#{stdout}\n#{stderr}" if exit_status != 0 end From 3c1f9423fee35236ee86b1efcdb07e0705e6ff40 Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Tue, 13 Nov 2018 14:44:42 +1300 Subject: [PATCH 6/7] wrong command parameter to mco r10k generate_types action the usage is mco r10k generate_types ENVIRONMENT --- templates/webhook.bin.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/webhook.bin.erb b/templates/webhook.bin.erb index ce32489e..6c8befdd 100755 --- a/templates/webhook.bin.erb +++ b/templates/webhook.bin.erb @@ -268,7 +268,7 @@ class Server < Sinatra::Base def generate_types(environment) begin if $config['use_mcollective'] - command = "#{$command_prefix} mco r10k generate_types --environment #{environment.gsub(/\W/, '_')}" + command = "#{$command_prefix} mco r10k generate_types #{environment.gsub(/\W/, '_')}" else command = "#{$command_prefix} /opt/puppetlabs/bin/puppet generate types --environment #{environment.gsub(/\W/, '_')}" end From 35594e15b2819c27edf0e5458100d92f899f8243 Mon Sep 17 00:00:00 2001 From: Stanley Zhang Date: Tue, 13 Nov 2018 16:36:08 +1300 Subject: [PATCH 7/7] turn to block mode if mco r10k genenrate_types need to run --- templates/webhook.bin.erb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/webhook.bin.erb b/templates/webhook.bin.erb index 6c8befdd..8e4ad195 100755 --- a/templates/webhook.bin.erb +++ b/templates/webhook.bin.erb @@ -256,9 +256,9 @@ class Server < Sinatra::Base # more or less simultaneously. To mitigate, we just lock on a file and wait for # the other one to complete. file.flock(File::LOCK_EX) - + message = "triggered: #{command}" - Process.detach(fork { exec "#{command} &" }) + fork { exec "#{command}" } exit_status = 0 raise "#{stdout}\n#{stderr}" if exit_status != 0 end @@ -269,6 +269,9 @@ class Server < Sinatra::Base begin if $config['use_mcollective'] command = "#{$command_prefix} mco r10k generate_types #{environment.gsub(/\W/, '_')}" + # because of the r10k racing condition issue wait for other r10k processes to finish + # https://github.com/voxpupuli/puppet-r10k/pull/268 + Process.waitall else command = "#{$command_prefix} /opt/puppetlabs/bin/puppet generate types --environment #{environment.gsub(/\W/, '_')}" end