From f5d1620ca3d92dd5181e72caa07145fce2749d90 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 22 Jun 2023 18:31:33 +0200 Subject: [PATCH 01/27] Add -t argument to envify --- lib/mrsk/cli/main.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index bf0d89171..898c3dfdc 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -165,15 +165,17 @@ def init end desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)" + option :template, aliases: "-t", type: :string, default: ".env.erb", desc: "Template to use" def envify if destination = options[:destination] - env_template_path = ".env.#{destination}.erb" + env_template_path = options[:template] || ".env.#{destination}.erb" env_path = ".env.#{destination}" else - env_template_path = ".env.erb" + env_template_path = options[:template] || ".env.erb" env_path = ".env" end + ENV["MRSK_DESTINATION"] = destination.to_s if destination File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) end From 24320323c8773e2bc913f7fd4403b7045aef5924 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 22 Jun 2023 18:39:24 +0200 Subject: [PATCH 02/27] Fix default value and add tests --- lib/mrsk/cli/main.rb | 2 +- test/cli/main_test.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 898c3dfdc..18e3c7240 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -165,7 +165,7 @@ def init end desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)" - option :template, aliases: "-t", type: :string, default: ".env.erb", desc: "Template to use" + option :template, aliases: "-t", type: :string, desc: "Template to use" def envify if destination = options[:destination] env_template_path = options[:template] || ".env.#{destination}.erb" diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 728ce24ef..d55691923 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -339,6 +339,20 @@ class CliMainTest < CliTestCase run_command("envify", "-d", "staging") end + test "envify with custom template file" do + File.expects(:read).with(".env.template.erb").returns("HELLO=<%= 'world' %>") + File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600) + + run_command("envify", "-t", ".env.template.erb", "-d", "staging") + end + + test "envify with custom template file and destination" do + File.expects(:read).with(".env.template.erb").returns("HELLO=<%= ENV['MRSK_DESTINATION'] %>") + File.expects(:write).with(".env.staging", "HELLO=staging", perm: 0600) + + run_command("envify", "-t", ".env.template.erb", "-d", "staging") + end + test "remove with confirmation" do run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output| assert_match /docker container stop traefik/, output From 17571eb9ecd5671b75a1e0fe559f4784212d74fc Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Fri, 23 Jun 2023 10:41:55 +0200 Subject: [PATCH 03/27] Add option to output and then read .env file from specific path --- lib/mrsk/cli/base.rb | 5 +++-- lib/mrsk/cli/main.rb | 8 ++++---- test/cli/main_test.rb | 39 +++++++++++++++++++++++++++++++++------ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb index 44df756ea..d7bca7bf1 100644 --- a/lib/mrsk/cli/base.rb +++ b/lib/mrsk/cli/base.rb @@ -19,6 +19,7 @@ def self.exit_on_failure?() true end class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file" class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)" + class_option :env_path, aliases: "-e", default: ".env", desc: "Path to env file" class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks" @@ -31,9 +32,9 @@ def initialize(*) private def load_envs if destination = options[:destination] - Dotenv.load(".env.#{destination}", ".env") + Dotenv.load("#{options[:env_path]}.#{destination}", options[:env_path]) else - Dotenv.load(".env") + Dotenv.load(options[:env_path]) end end diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index bf0d89171..aeee0ba0b 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -137,9 +137,9 @@ def init puts "Created configuration file in config/deploy.yml" end - unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist? + unless (deploy_file = Pathname.new(File.expand_path(options[:env_path]))).exist? FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file - puts "Created .env file" + puts "Created #{options[:env_path]} file" end unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist? @@ -168,10 +168,10 @@ def init def envify if destination = options[:destination] env_template_path = ".env.#{destination}.erb" - env_path = ".env.#{destination}" + env_path = "#{options[:env_path]}.#{destination}" else env_template_path = ".env.erb" - env_path = ".env" + env_path = options[:env_path] end File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 728ce24ef..8de51c0e6 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -10,7 +10,7 @@ class CliMainTest < CliTestCase end test "deploy" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) @@ -37,7 +37,7 @@ class CliMainTest < CliTestCase end test "deploy with skip_push" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) @@ -87,7 +87,7 @@ class CliMainTest < CliTestCase end test "deploy errors during outside section leave remove lock" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke) .with("mrsk:cli:registry:login", [], invoke_options) @@ -101,7 +101,7 @@ class CliMainTest < CliTestCase end test "deploy with skipped hooks" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) @@ -123,7 +123,7 @@ class CliMainTest < CliTestCase end test "redeploy" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) @@ -145,7 +145,7 @@ class CliMainTest < CliTestCase end test "redeploy with skip_push" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "env_path" => ".env" } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) @@ -287,6 +287,19 @@ class CliMainTest < CliTestCase end end + test "init with custom env path" do + Pathname.any_instance.expects(:exist?).returns(false).times(3) + Pathname.any_instance.stubs(:mkpath) + FileUtils.stubs(:mkdir_p) + FileUtils.stubs(:cp_r) + FileUtils.stubs(:cp) + + run_command("init", "-e", ".env.out").tap do |output| + assert_match /Created configuration file in config\/deploy.yml/, output + assert_match /Created \.env.out file/, output + end + end + test "init with existing config" do Pathname.any_instance.expects(:exist?).returns(true).times(3) @@ -332,6 +345,13 @@ class CliMainTest < CliTestCase run_command("envify") end + test "envify with env output" do + File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>") + File.expects(:write).with("dist/.env", "HELLO=world", perm: 0600) + + run_command("envify", '-e', 'dist/.env') + end + test "envify with destination" do File.expects(:read).with(".env.staging.erb").returns("HELLO=<%= 'world' %>") File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600) @@ -339,6 +359,13 @@ class CliMainTest < CliTestCase run_command("envify", "-d", "staging") end + test "envify with destination and env output" do + File.expects(:read).with(".env.staging.erb").returns("HELLO=<%= 'world' %>") + File.expects(:write).with("dist/.env.staging", "HELLO=world", perm: 0600) + + run_command("envify", "-d", "staging", '-e', 'dist/.env') + end + test "remove with confirmation" do run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output| assert_match /docker container stop traefik/, output From 27ce26beeb4b7ccb12d6e683fb5a8d2c543786ab Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Mon, 26 Jun 2023 08:29:03 +0200 Subject: [PATCH 04/27] Cleanup --- lib/mrsk/cli/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb index d7bca7bf1..d0bece0cd 100644 --- a/lib/mrsk/cli/base.rb +++ b/lib/mrsk/cli/base.rb @@ -31,7 +31,7 @@ def initialize(*) private def load_envs - if destination = options[:destination] + if (destination = options[:destination]) Dotenv.load("#{options[:env_path]}.#{destination}", options[:env_path]) else Dotenv.load(options[:env_path]) From 78be2a8e010ba660f6d95c7360a6ec2a9a9ad61e Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Mon, 26 Jun 2023 08:33:10 +0200 Subject: [PATCH 05/27] Fix wrong logical assertion --- test/cli/main_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index d55691923..2d8ae0b9e 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -341,9 +341,9 @@ class CliMainTest < CliTestCase test "envify with custom template file" do File.expects(:read).with(".env.template.erb").returns("HELLO=<%= 'world' %>") - File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600) + File.expects(:write).with(".env", "HELLO=world", perm: 0600) - run_command("envify", "-t", ".env.template.erb", "-d", "staging") + run_command("envify", "-t", ".env.template.erb") end test "envify with custom template file and destination" do From abf4f5ad10b979aae752956be7e7badeef04feaf Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Mon, 26 Jun 2023 11:12:30 +0200 Subject: [PATCH 06/27] Fix --- lib/mrsk/cli/main.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 7f3ff2630..9dce0871e 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -1,3 +1,5 @@ +require 'fileutils' + class Mrsk::Cli::Main < Mrsk::Cli::Base desc "setup", "Setup all accessories and deploy app to servers" def setup @@ -167,7 +169,7 @@ def init desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)" option :template, aliases: "-t", type: :string, desc: "Template to use" def envify - if destination = options[:destination] + if (destination = options[:destination]) env_template_path = options[:template] || ".env.#{destination}.erb" env_path = "#{options[:env_path]}.#{destination}" else @@ -175,6 +177,10 @@ def envify env_path = options[:env_path] end + unless File.exist?(env_path) && env_path.include?('/') + FileUtils.mkdir_p(File.dirname(env_path)) + end + ENV["MRSK_DESTINATION"] = destination.to_s if destination File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) end From 5472370a0f4e7046d344e2254dd085ac4c933a01 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Mon, 26 Jun 2023 11:14:44 +0200 Subject: [PATCH 07/27] Make sure to create nested path if missing --- lib/mrsk/cli/main.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index aeee0ba0b..a2ded0402 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -1,3 +1,5 @@ +require 'fileutils' + class Mrsk::Cli::Main < Mrsk::Cli::Base desc "setup", "Setup all accessories and deploy app to servers" def setup @@ -174,6 +176,10 @@ def envify env_path = options[:env_path] end + unless File.exist?(env_path) && env_path.include?('/') + FileUtils.mkdir_p(File.dirname(env_path)) + end + File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) end From bd71192353dbf61590aae94b35b1b8657ea19770 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Mon, 26 Jun 2023 11:24:52 +0200 Subject: [PATCH 08/27] Remove unused import --- lib/mrsk/cli/main.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index a2ded0402..b37c98103 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -1,5 +1,3 @@ -require 'fileutils' - class Mrsk::Cli::Main < Mrsk::Cli::Base desc "setup", "Setup all accessories and deploy app to servers" def setup From bb6ec8c05ed2b43d53bcd8ba4d16c65f8b42ecfe Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Tue, 27 Jun 2023 14:10:42 +0200 Subject: [PATCH 09/27] Ignore .vscode/.idea folders --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6100dc738..80e6a5437 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ coverage/* .DS_Store gemfiles/*.lock +.vscode/ +.idea/ \ No newline at end of file From c5ec8b15159614db06905a7b396baeaf6d3b01b8 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Tue, 27 Jun 2023 14:11:06 +0200 Subject: [PATCH 10/27] Wip add base logic for optional secrets --- lib/mrsk/configuration/role.rb | 17 ++++++++++++++--- lib/mrsk/utils.rb | 14 ++++++++++++-- test/utils_test.rb | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/lib/mrsk/configuration/role.rb b/lib/mrsk/configuration/role.rb index 048efe16d..fecf2687a 100644 --- a/lib/mrsk/configuration/role.rb +++ b/lib/mrsk/configuration/role.rb @@ -139,16 +139,27 @@ def merged_env # Secrets are stored in an array, which won't merge by default, so have to do it by hand. def merged_env_with_secrets merged_env.tap do |new_env| - new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"]) - # If there's no secret/clear split, everything is clear clear_app_env = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env) clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env) - new_env["clear"] = (clear_app_env + clear_role_env).uniq + + secrets_app_env = Array(config.env["secret"]) + secrets_role_env = Array(specialized_env["secret"]) + new_env["secret"] = (secrets_app_env + secrets_role_env).uniq.filter { |secret| filter_secret_env(secret, new_env) } end end + def filter_secret_env(secret, new_env) + # allow clear to override secret + return false if new_env['clear'].include?(secret) + + # if we find FOO but FOO? exists, we keep the FOO? + return false if !secret.end_with?('?') && new_env['secret'].include?("#{secret}?") + + true + end + def http_health_check(port:, path:) "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present? end diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index e463f5478..a3c430828 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -20,12 +20,22 @@ def argumentize(argument, attributes, sensitive: false) # but redacts and expands secrets. def argumentize_env_with_secrets(env) if (secrets = env["secret"]).present? - argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"]) + argumentize("-e", handle_optional_secrets(secrets), sensitive: true) + argumentize("-e", env["clear"]) else - argumentize "-e", env.fetch("clear", env) + argumentize("-e", env.fetch("clear", env)) end end + def handle_optional_secrets(secrets) + secrets.to_h do |key| + if key.end_with? '?' + [ key.chop, ENV[key.chop] ] + else + [ key, ENV.fetch(key) ] + end + end.compact + end + # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option. def optionize(args, with: nil) options = if with diff --git a/test/utils_test.rb b/test/utils_test.rb index 24a1b96b6..0c067ae0e 100644 --- a/test/utils_test.rb +++ b/test/utils_test.rb @@ -20,6 +20,22 @@ class UtilsTest < ActiveSupport::TestCase assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args) end + test "argumentize_env_with_secrets with optional" do + ENV.expects(:[]).with("FOO").returns('secret') + + args = Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO?" ], "clear" => { BAZ: "qux" } }) + + assert_equal [ "-e", "FOO=[REDACTED]", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.redacted(args) + assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args) + end + + test "argumentize_env_with_secrets with missing optional" do + args = Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO?" ], "clear" => { BAZ: "qux" } }) + + assert_equal [ "-e", "BAZ=\"qux\"" ], Mrsk::Utils.redacted(args) + assert_equal [ "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args) + end + test "optionize" do assert_equal [ "--foo", "\"bar\"", "--baz", "\"qux\"", "--quux" ], \ Mrsk::Utils.optionize({ foo: "bar", baz: "qux", quux: true }) From e17e4a9512c8ef7f4768d20a329281d0849ba6eb Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Tue, 27 Jun 2023 16:26:34 +0200 Subject: [PATCH 11/27] Add initial delay to healtchecks --- lib/mrsk/configuration.rb | 2 +- lib/mrsk/utils/healthcheck_poller.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index ee6dce056..bf7368a73 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -158,7 +158,7 @@ def ssh_options def healthcheck - { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {}) + { "path" => "/up", "port" => 3000, "max_attempts" => 7, "initial_delay" => 0 }.merge(raw_config.healthcheck || {}) end def readiness_delay diff --git a/lib/mrsk/utils/healthcheck_poller.rb b/lib/mrsk/utils/healthcheck_poller.rb index 3ef5b7a88..c57750e8b 100644 --- a/lib/mrsk/utils/healthcheck_poller.rb +++ b/lib/mrsk/utils/healthcheck_poller.rb @@ -7,6 +7,12 @@ class << self def wait_for_healthy(pause_after_ready: false, &block) attempt = 1 max_attempts = MRSK.config.healthcheck["max_attempts"] + initial_delay = MRSK.config.healthcheck["initial_delay"] + + if initial_delay > 0 + info "Waiting #{initial_delay}s before checking container health..." + sleep initial_delay + end begin case status = block.call From 8c543596e77e15dfdb7688469951ee66d9d0ebbc Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 9 Jul 2023 13:30:07 +0200 Subject: [PATCH 12/27] Cleanup output of `envify` --- lib/mrsk/cli/main.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 795602c22..6445d8376 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -180,7 +180,10 @@ def envify end ENV["MRSK_DESTINATION"] = destination.to_s if destination - File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) + output = ERB.new(File.read(env_template_path)).result + output = output..gsub(/(?<=\n)\s+/, '').gsub(/\n{3,}/, "\n\n") + + File.write(env_path, output, perm: 0600) end desc "remove", "Remove Traefik, app, accessories, and registry session from servers" From 4a7984a45d1e6eb8c57bfc2a2328b71400323cde Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 9 Jul 2023 13:31:14 +0200 Subject: [PATCH 13/27] Fix typo --- lib/mrsk/cli/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 6445d8376..2caeef162 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -181,7 +181,7 @@ def envify ENV["MRSK_DESTINATION"] = destination.to_s if destination output = ERB.new(File.read(env_template_path)).result - output = output..gsub(/(?<=\n)\s+/, '').gsub(/\n{3,}/, "\n\n") + output = output.gsub(/(?<=\n)\s+/, '').gsub(/\n{3,}/, "\n\n") File.write(env_path, output, perm: 0600) end From 245bd786e66e31ae084aa1a000f15cb5088a1afc Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 9 Jul 2023 13:45:30 +0200 Subject: [PATCH 14/27] Fix line replacement + add tests --- lib/mrsk/cli/main.rb | 2 +- test/cli/main_test.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 2caeef162..31f7553a0 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -181,7 +181,7 @@ def envify ENV["MRSK_DESTINATION"] = destination.to_s if destination output = ERB.new(File.read(env_template_path)).result - output = output.gsub(/(?<=\n)\s+/, '').gsub(/\n{3,}/, "\n\n") + output = output.gsub(/\n{3,}/, "\n\n").gsub(/^(?![\r\n])\s+/, '') File.write(env_path, output, perm: 0600) end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index e0ad88e59..f292c6f0f 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -345,6 +345,14 @@ class CliMainTest < CliTestCase run_command("envify") end + test "envify with bad formatting in ERB" do + # should filter out more than 1 blank line + leading whitespace + File.expects(:read).with(".env.erb").returns(" HELLO=<%= 'world' %>\n HELLO2=<%= 'world2' %>\n\n\nBAR=foo") + File.expects(:write).with(".env", "HELLO=world\nHELLO2=world2\n\nBAR=foo", perm: 0600) + + run_command("envify") + end + test "envify with env output" do File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>") File.expects(:write).with("dist/.env", "HELLO=world", perm: 0600) From b0cccc1f088943d2e85f5e3d9c26af528a8ab6dd Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 9 Jul 2023 13:54:26 +0200 Subject: [PATCH 15/27] Ignore .idea folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6100dc738..9c0078dd3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ coverage/* .DS_Store gemfiles/*.lock +.idea/ From dd52a4579e19d660a93c0001a6a87e3f16d909fb Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 9 Jul 2023 13:54:34 +0200 Subject: [PATCH 16/27] Cleanup output of envify --- lib/mrsk/cli/main.rb | 5 ++++- test/cli/main_test.rb | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index bf0d89171..fef869d4a 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -174,7 +174,10 @@ def envify env_path = ".env" end - File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600) + output = ERB.new(File.read(env_template_path)).result + output = output.gsub(/\n{3,}/, "\n\n").gsub(/^(?![\r\n])\s+/, '') + + File.write(env_path, output, perm: 0600) end desc "remove", "Remove Traefik, app, accessories, and registry session from servers" diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 728ce24ef..708553061 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -332,6 +332,14 @@ class CliMainTest < CliTestCase run_command("envify") end + test "envify with bad formatting in ERB" do + # filters out more than 1 blank line + leading whitespace + File.expects(:read).with(".env.erb").returns(" HELLO=<%= 'world' %>\n HELLO2=<%= 'world2' %>\n\n\nBAR=foo") + File.expects(:write).with(".env", "HELLO=world\nHELLO2=world2\n\nBAR=foo", perm: 0600) + + run_command("envify") + end + test "envify with destination" do File.expects(:read).with(".env.staging.erb").returns("HELLO=<%= 'world' %>") File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600) From f9e9232ab5780eeb0ed1a8fd01a9aef8e7801f72 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 13 Jul 2023 12:05:58 +0200 Subject: [PATCH 17/27] Ignore .idea folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6100dc738..9c0078dd3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ coverage/* .DS_Store gemfiles/*.lock +.idea/ From 04163633de4c6cf1286cbe77fc0d25fd89b77e6b Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 13 Jul 2023 12:06:35 +0200 Subject: [PATCH 18/27] Allow usage of .yml.erb and .yml files (without breaking existing compatibility of erb templates in .yml files) --- lib/mrsk/configuration.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index ee6dce056..2e48fecb3 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -33,7 +33,12 @@ def load_config_file(file) end def destination_config_file(base_config_file, destination) - base_config_file.sub_ext(".#{destination}.yml") if destination + return unless destination + + return base_config_file.sub_ext(".#{destination}.yml") if base_config_file.extname == ".yml" + return base_config_file.sub(/\.yml.erb$/, ".#{destination}.yml.erb") if base_config_file.extname == ".erb" + + raise 'Unsupported config file extension. Please use .yml or .yml.erb' end end From a1a927b64b531fd6c28ebdbcb5be9df7f1856ec7 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 13 Jul 2023 12:12:27 +0200 Subject: [PATCH 19/27] Add tests for .yml.erb support --- test/configuration_test.rb | 10 +++++++++- test/fixtures/deploy.staging.yml.erb | 2 ++ test/fixtures/{deploy.erb.yml => deploy.yml.erb} | 0 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/deploy.staging.yml.erb rename test/fixtures/{deploy.erb.yml => deploy.yml.erb} (100%) diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 0f7e12ffe..d93e9a8ba 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -243,10 +243,18 @@ class ConfigurationTest < ActiveSupport::TestCase end test "erb evaluation of yml config" do - config = Mrsk::Configuration.create_from config_file: Pathname.new(File.expand_path("fixtures/deploy.erb.yml", __dir__)) + config = Mrsk::Configuration.create_from config_file: Pathname.new(File.expand_path("fixtures/deploy.yml.erb", __dir__)) assert_equal "my-user", config.registry["username"] end + test "erb evaluation of yml config with destinations" do + config_file = Pathname.new(File.expand_path("fixtures/deploy.yml.erb", __dir__)) + + config = Mrsk::Configuration.create_from config_file: config_file, destination: 'staging' + assert_equal "my-user", config.registry["username"] + assert_equal "my-password-override", config.registry["password"] + end + test "destination yml config merge" do dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_dest.yml", __dir__)) diff --git a/test/fixtures/deploy.staging.yml.erb b/test/fixtures/deploy.staging.yml.erb new file mode 100644 index 000000000..5b2b81f53 --- /dev/null +++ b/test/fixtures/deploy.staging.yml.erb @@ -0,0 +1,2 @@ +registry: + password: <%= "my-password-override" %> diff --git a/test/fixtures/deploy.erb.yml b/test/fixtures/deploy.yml.erb similarity index 100% rename from test/fixtures/deploy.erb.yml rename to test/fixtures/deploy.yml.erb From 9ab4fdc6170227f18c62e4b59a039197d6d7f3a3 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 13 Jul 2023 12:59:18 +0200 Subject: [PATCH 20/27] Simplify extension replacement --- lib/mrsk/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 2e48fecb3..722e51f03 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -35,7 +35,7 @@ def load_config_file(file) def destination_config_file(base_config_file, destination) return unless destination - return base_config_file.sub_ext(".#{destination}.yml") if base_config_file.extname == ".yml" + return base_config_file.sub(/\.yml$/, ".#{destination}.yml") if base_config_file.extname == ".yml" return base_config_file.sub(/\.yml.erb$/, ".#{destination}.yml.erb") if base_config_file.extname == ".erb" raise 'Unsupported config file extension. Please use .yml or .yml.erb' From 33eb19c361703b83e4a3fae7101265172d12944e Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 13 Jul 2023 13:29:15 +0200 Subject: [PATCH 21/27] Expose `MRSK_DESTINATION` when loading deploy.yml --- .gitignore | 1 + lib/mrsk/configuration.rb | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6100dc738..9c0078dd3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ coverage/* .DS_Store gemfiles/*.lock +.idea/ diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index ee6dce056..cd36f71e6 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -14,18 +14,19 @@ class Mrsk::Configuration class << self def create_from(config_file:, destination: nil, version: nil) - raw_config = load_config_files(config_file, *destination_config_file(config_file, destination)) + raw_config = load_config_files(config_file, *destination_config_file(config_file, destination), destination: destination) new raw_config, destination: destination, version: version end private - def load_config_files(*files) - files.inject({}) { |config, file| config.deep_merge! load_config_file(file) } + def load_config_files(*files, destination: nil) + files.inject({}) { |config, file| config.deep_merge! load_config_file(file, destination) } end - def load_config_file(file) + def load_config_file(file, destination) if file.exist? + ENV["MRSK_DESTINATION"] = destination.to_s if destination YAML.load(ERB.new(IO.read(file)).result).symbolize_keys else raise "Configuration file not found in #{file}" From 7d4ddf3d070db518acbc60db983dc2f20a0711ca Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Wed, 19 Jul 2023 10:21:51 +0200 Subject: [PATCH 22/27] Fix clear overrides secrets --- lib/mrsk/utils.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index a3c430828..cd8647711 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -20,16 +20,18 @@ def argumentize(argument, attributes, sensitive: false) # but redacts and expands secrets. def argumentize_env_with_secrets(env) if (secrets = env["secret"]).present? - argumentize("-e", handle_optional_secrets(secrets), sensitive: true) + argumentize("-e", env["clear"]) + argumentize("-e", handle_optional_secrets(secrets, env.fetch('clear', {})), sensitive: true) + argumentize("-e", env["clear"]) else argumentize("-e", env.fetch("clear", env)) end end - def handle_optional_secrets(secrets) + def handle_optional_secrets(secrets, clear_env) secrets.to_h do |key| if key.end_with? '?' [ key.chop, ENV[key.chop] ] + elsif clear_env.key?(key) + [ key, clear_env[key] ] else [ key, ENV.fetch(key) ] end From ac8849596858e00c6d22833bd7d9b81f6a00cccd Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Wed, 19 Jul 2023 10:27:25 +0200 Subject: [PATCH 23/27] fix wrong key --- lib/mrsk/utils.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index cd8647711..99800088d 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -30,8 +30,8 @@ def handle_optional_secrets(secrets, clear_env) secrets.to_h do |key| if key.end_with? '?' [ key.chop, ENV[key.chop] ] - elsif clear_env.key?(key) - [ key, clear_env[key] ] + elsif (index = clear_env.keys.index(key)) + [ key, clear_env[index] ] else [ key, ENV.fetch(key) ] end From 2991ca408d4f85522010775001fa0f1f91616d38 Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Wed, 19 Jul 2023 10:28:44 +0200 Subject: [PATCH 24/27] Fix again.. --- lib/mrsk/utils.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index 99800088d..dd7654c7f 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -30,7 +30,7 @@ def handle_optional_secrets(secrets, clear_env) secrets.to_h do |key| if key.end_with? '?' [ key.chop, ENV[key.chop] ] - elsif (index = clear_env.keys.index(key)) + elsif (index = clear_env.find_index { |pair| pair.first == key }) [ key, clear_env[index] ] else [ key, ENV.fetch(key) ] From 6d53e188d67e6902da82d661e6b65c0d254525df Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Thu, 24 Aug 2023 09:13:14 +0200 Subject: [PATCH 25/27] Return a lot more logs onn failure --- lib/mrsk/commands/healthcheck.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/commands/healthcheck.rb b/lib/mrsk/commands/healthcheck.rb index ad8070df6..d929685c8 100644 --- a/lib/mrsk/commands/healthcheck.rb +++ b/lib/mrsk/commands/healthcheck.rb @@ -27,7 +27,7 @@ def container_health_log end def logs - pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1")) + pipe container_id, xargs(docker(:logs, "--tail", 200, "2>&1")) end def stop From dc899d17b78bdb3772a9bca1db7525e5cede073a Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 27 Aug 2023 12:21:30 +0200 Subject: [PATCH 26/27] Try loading the bin --- bin/mrsk | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/mrsk b/bin/mrsk index f2eadfae0..a7eb0cd6d 100755 --- a/bin/mrsk +++ b/bin/mrsk @@ -3,6 +3,13 @@ # Prevent failures from being reported twice. Thread.report_on_exception = false +LOCAL_BIN = 'bin/mrsk' + +if File.exist?(LOCAL_BIN) + load LOCAL_BIN + exit +end + require "mrsk" begin From e08435b728c63730774fe52a9852700b7381ae2c Mon Sep 17 00:00:00 2001 From: Fran Zekan Date: Sun, 27 Aug 2023 17:07:25 +0200 Subject: [PATCH 27/27] Setup load bin/mrsk if possible --- bin/mrsk | 9 +++------ lib/loader.rb | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 lib/loader.rb diff --git a/bin/mrsk b/bin/mrsk index a7eb0cd6d..4fe9753f5 100755 --- a/bin/mrsk +++ b/bin/mrsk @@ -3,12 +3,9 @@ # Prevent failures from being reported twice. Thread.report_on_exception = false -LOCAL_BIN = 'bin/mrsk' - -if File.exist?(LOCAL_BIN) - load LOCAL_BIN - exit -end +require_relative '../lib/loader' +p Loader.methods - Object.methods +Loader::load_local_bin # local bin is found this command exists so the rest of the code doesn't run require "mrsk" diff --git a/lib/loader.rb b/lib/loader.rb new file mode 100644 index 000000000..f5065f78c --- /dev/null +++ b/lib/loader.rb @@ -0,0 +1,28 @@ +module Loader + LOCAL_BIN = 'bin/mrsk' + + def load_local_bin + original_cwd = Dir.pwd + + loop do + if bin_exists? + exec LOCAL_BIN, *ARGV + end + + # If we exhaust the search there is no executable, run bundled mrsk from original cwd + Dir.chdir(original_cwd) && return if Pathname.new(Dir.pwd).root? + + # Otherwise keep moving upwards in search of an executable. + Dir.chdir("..") + end + end + + private + + def bin_exists? + return unless File.exist?(LOCAL_BIN) || File.executable?(LOCAL_BIN) + return unless File.read(LOCAL_BIN, 300).include?("This file was generated by Bundler") + + true + end +end