Asset paths

lib/kamal/cli/app.rb

@@ -10,12 +10,21 @@ def boot
           on(KAMAL.hosts) do
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
             execute *KAMAL.app.tag_current_as_latest
+
+            KAMAL.roles_on(host).each do |role|
+              app = KAMAL.app(role: role)
+              role_config = KAMAL.config.role(role)
+
+              if role_config.assets?
+                execute *app.extract_assets
+                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                execute *app.sync_asset_volumes(old_version: old_version)
+              end
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
-
-            roles.each do |role|
+            KAMAL.roles_on(host).each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
               role_config = KAMAL.config.role(role)
@@ -27,13 +36,11 @@ def boot
                 execute *app.rename_container(version: version, new_version: tmp_version)
               end
 
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
 
-              if role_config.uses_cord?
-                execute *app.tie_cord(role_config.cord_host_file)
-              end
+              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
+
+              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
@@ -47,7 +54,10 @@ def boot
                     Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
+
                 execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+
+                execute *app.clean_up_assets if role_config.assets?
               end
             end
           end

lib/kamal/commands/app.rb

@@ -20,6 +20,7 @@ def run(hostname: nil)
       *role_config.health_check_args,
       *config.logging_args,
       *config.volume_args,
+      *role_config.asset_volume_args,
       *role_config.label_args,
       *role_config.option_args,
       config.absolute_image,
@@ -105,7 +106,7 @@ def current_running_version
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -153,7 +154,7 @@ def remove_env_file
   def cord(version:)
     pipe \
       docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"]
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
   end
 
   def tie_cord(cord)
@@ -164,9 +165,43 @@ def cut_cord(cord)
     remove_directory(cord)
   end
 
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
+  end
+
   private
     def container_name(version = nil)
-      [ role_config.full_name, version || config.version ].compact.join("-")
+      [ role_config.container_prefix, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)
@@ -186,4 +221,19 @@ def filters(statuses: nil)
         end
       end
     end
+
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rn", "#{source}/*", destination, *("|| true" if continue_on_error)]
+    end
 end

lib/kamal/configuration.rb

@@ -211,6 +211,10 @@ def run_id
     @run_id ||= SecureRandom.hex(16)
   end
 
+  def asset_path
+    raw_config.asset_path
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present

lib/kamal/configuration/role.rb

@@ -48,11 +48,15 @@ def env_args
     argumentize "--env-file", host_env_file_path
   end
 
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
   def health_check_args(cord: true)
     if health_check_cmd.present?
       if cord && uses_cord?
         optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
-          .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"])
+          .concat(cord_volume.docker_args)
       else
         optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
       end
@@ -74,23 +78,31 @@ def health_check_interval
   end
 
   def uses_cord?
-    running_traefik? && cord_container_directory.present? && health_check_cmd.present?
+    running_traefik? && cord_volume && health_check_cmd.present?
   end
 
   def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-")
+    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        container_path: cord
+    end
   end
 
   def cord_host_file
-    File.join cord_host_directory, CORD_FILE
+    File.join cord_volume.host_path, CORD_FILE
   end
 
   def cord_container_directory
     health_check_options.fetch("cord", nil)
   end
 
   def cord_container_file
-    File.join cord_container_directory, CORD_FILE
+    File.join cord_volume.container_path, CORD_FILE
   end
 
 
@@ -110,10 +122,37 @@ def running_traefik?
     name.web? || specializations["traefik"]
   end
 
-  def full_name
+  def container_name(version = nil)
+    [ container_prefix, version || config.version ].compact.join("-")
+  end
+
+  def container_prefix
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end
+
   private
     attr_accessor :config
 

lib/kamal/configuration/volume.rb

@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end

test/cli/app_test.rb

@@ -55,8 +55,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -66,6 +64,34 @@ class CliAppTest < CliTestCase
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rn .kamal/assets/extracted/app-web-latest/* .kamal/assets/volumes/app-web-latest ; cp -rn .kamal/assets/extracted/app-web-latest/* .kamal/assets/volumes/app-web-123 || true ; cp -rn .kamal/assets/extracted/app-web-123/* .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output

test/commands/app_test.rb

@@ -345,8 +345,39 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
   end
 
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-998/*", ".kamal/assets/volumes/app-web-999", "|| true",
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
+  end
+
   private
-    def new_command(role: "web")
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+    def new_command(role: "web", **additional_config)
+      Kamal::Commands::App.new(Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999"), role: role)
     end
 end