Develop into old main

.gitignore

@@ -3,3 +3,5 @@
 coverage/*
 .DS_Store
 gemfiles/*.lock
+.vscode/
+.idea/

bin/mrsk

@@ -3,6 +3,10 @@
 # Prevent failures from being reported twice.
 Thread.report_on_exception = false
 
+require_relative '../lib/loader'
+p Loader.methods - Object.methods
+Loader::load_local_bin # local bin is found this command exists so the rest of the code doesn't run
+
 require "mrsk"
 
 begin

lib/loader.rb

@@ -0,0 +1,28 @@
+module Loader
+  LOCAL_BIN = 'bin/mrsk'
+
+  def load_local_bin
+    original_cwd = Dir.pwd
+
+    loop do
+      if bin_exists?
+        exec LOCAL_BIN, *ARGV
+      end
+
+      # If we exhaust the search there is no executable, run bundled mrsk from original cwd
+      Dir.chdir(original_cwd) && return if Pathname.new(Dir.pwd).root?
+
+      # Otherwise keep moving upwards in search of an executable.
+      Dir.chdir("..")
+    end
+  end
+
+  private
+
+  def bin_exists?
+    return unless File.exist?(LOCAL_BIN) || File.executable?(LOCAL_BIN)
+    return unless File.read(LOCAL_BIN, 300).include?("This file was generated by Bundler")
+
+    true
+  end
+end

lib/mrsk/cli/base.rb

@@ -19,6 +19,7 @@ def self.exit_on_failure?() true end
 
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
+    class_option :env_path, aliases: "-e", default: ".env", desc: "Path to env file"
 
     class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks"
 
@@ -30,10 +31,10 @@ def initialize(*)
 
     private
       def load_envs
-        if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
+        if (destination = options[:destination])
+          Dotenv.load("#{options[:env_path]}.#{destination}", options[:env_path])
         else
-          Dotenv.load(".env")
+          Dotenv.load(options[:env_path])
         end
       end
 

lib/mrsk/cli/main.rb

@@ -137,9 +137,9 @@ def init
       puts "Created configuration file in config/deploy.yml"
     end
 
-    unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
+    unless (deploy_file = Pathname.new(File.expand_path(options[:env_path]))).exist?
       FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
-      puts "Created .env file"
+      puts "Created #{options[:env_path]} file"
     end
 
     unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
@@ -165,16 +165,25 @@ def init
   end
 
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  option :template, aliases: "-t", type: :string, desc: "Template to use"
   def envify
-    if destination = options[:destination]
-      env_template_path = ".env.#{destination}.erb"
-      env_path          = ".env.#{destination}"
+    if (destination = options[:destination])
+      env_template_path = options[:template] || ".env.#{destination}.erb"
+      env_path          = "#{options[:env_path]}.#{destination}"
     else
-      env_template_path = ".env.erb"
-      env_path          = ".env"
+      env_template_path = options[:template] || ".env.erb"
+      env_path          = options[:env_path]
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+    unless File.exist?(env_path) && env_path.include?('/')
+      FileUtils.mkdir_p(File.dirname(env_path))
+    end
+
+    ENV["MRSK_DESTINATION"] = destination.to_s if destination
+    output = ERB.new(File.read(env_template_path)).result
+    output = output.gsub(/\n{3,}/, "\n\n").gsub(/^(?![\r\n])\s+/, '')
+
+    File.write(env_path, output, perm: 0600)
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"

lib/mrsk/commands/healthcheck.rb

@@ -27,7 +27,7 @@ def container_health_log
   end
 
   def logs
-    pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1"))
+    pipe container_id, xargs(docker(:logs, "--tail", 200, "2>&1"))
   end
 
   def stop

lib/mrsk/configuration.rb

@@ -14,26 +14,32 @@ class Mrsk::Configuration
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
-      raw_config = load_config_files(config_file, *destination_config_file(config_file, destination))
+      raw_config = load_config_files(config_file, *destination_config_file(config_file, destination), destination: destination)
 
       new raw_config, destination: destination, version: version
     end
 
     private
-      def load_config_files(*files)
-        files.inject({}) { |config, file| config.deep_merge! load_config_file(file) }
+      def load_config_files(*files, destination: nil)
+        files.inject({}) { |config, file| config.deep_merge! load_config_file(file, destination) }
       end
 
-      def load_config_file(file)
+      def load_config_file(file, destination)
         if file.exist?
+          ENV["MRSK_DESTINATION"] = destination.to_s if destination
           YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
       end
 
       def destination_config_file(base_config_file, destination)
-        base_config_file.sub_ext(".#{destination}.yml") if destination
+        return unless destination
+
+        return base_config_file.sub(/\.yml$/, ".#{destination}.yml") if base_config_file.extname == ".yml"
+        return base_config_file.sub(/\.yml.erb$/, ".#{destination}.yml.erb") if base_config_file.extname == ".erb"
+
+        raise 'Unsupported config file extension. Please use .yml or .yml.erb'
       end
   end
 
@@ -158,7 +164,7 @@ def ssh_options
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "initial_delay" => 0 }.merge(raw_config.healthcheck || {})
   end
 
   def readiness_delay

lib/mrsk/configuration/role.rb

@@ -139,16 +139,27 @@ def merged_env
     # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
     def merged_env_with_secrets
       merged_env.tap do |new_env|
-        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
-
         # If there's no secret/clear split, everything is clear
         clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
         clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
-
         new_env["clear"] = (clear_app_env + clear_role_env).uniq
+
+        secrets_app_env = Array(config.env["secret"])
+        secrets_role_env = Array(specialized_env["secret"])
+        new_env["secret"] = (secrets_app_env + secrets_role_env).uniq.filter { |secret| filter_secret_env(secret, new_env) }
       end
     end
 
+    def filter_secret_env(secret, new_env)
+      # allow clear to override secret
+      return false if new_env['clear'].include?(secret)
+
+      # if we find FOO but FOO? exists, we keep the FOO?
+      return false if !secret.end_with?('?') && new_env['secret'].include?("#{secret}?")
+
+      true
+    end
+
     def http_health_check(port:, path:)
       "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
     end

lib/mrsk/utils.rb

@@ -20,12 +20,24 @@ def argumentize(argument, attributes, sensitive: false)
   # but redacts and expands secrets.
   def argumentize_env_with_secrets(env)
     if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
+      argumentize("-e", handle_optional_secrets(secrets, env.fetch('clear', {})), sensitive: true) + argumentize("-e", env["clear"])
     else
-      argumentize "-e", env.fetch("clear", env)
+      argumentize("-e", env.fetch("clear", env))
     end
   end
 
+  def handle_optional_secrets(secrets, clear_env)
+    secrets.to_h do |key|
+      if key.end_with? '?'
+        [ key.chop, ENV[key.chop] ]
+      elsif (index = clear_env.find_index { |pair| pair.first == key })
+        [ key, clear_env[index] ]
+      else
+        [ key, ENV.fetch(key) ]
+      end
+    end.compact
+  end
+
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
   def optionize(args, with: nil)
     options = if with

lib/mrsk/utils/healthcheck_poller.rb

@@ -7,6 +7,12 @@ class << self
     def wait_for_healthy(pause_after_ready: false, &block)
       attempt = 1
       max_attempts = MRSK.config.healthcheck["max_attempts"]
+      initial_delay = MRSK.config.healthcheck["initial_delay"]
+
+      if initial_delay > 0
+        info "Waiting #{initial_delay}s before checking container health..."
+        sleep initial_delay
+      end
 
       begin
         case status = block.call