Merge pull request #472 from basecamp/move-healthcheck-poller-to-cli

Healthcheck polling is a CLI concern
basecamp · Sep 16, 2023 · ec8192b · ec8192b
2 parents ff4d025 + 59ac59d
commit ec8192b
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 71 deletions.
diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
@@ -44,14 +44,14 @@ def boot
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
-              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
               if old_version.present?
                 if role_config.uses_cord?
                   cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
                   if cord.present?
                     execute *app.cut_cord(cord)
-                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
 

diff --git a/lib/kamal/cli/healthcheck.rb b/lib/kamal/cli/healthcheck.rb
@@ -6,8 +6,8 @@ def perform
     on(KAMAL.primary_host) do
       begin
         execute *KAMAL.healthcheck.run
-        Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
+        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
+      rescue Poller::HealthcheckError => e
         error capture_with_info(*KAMAL.healthcheck.logs)
         error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
         raise

diff --git a/lib/kamal/cli/healthcheck/poller.rb b/lib/kamal/cli/healthcheck/poller.rb
@@ -0,0 +1,64 @@
+module Kamal::Cli::Healthcheck::Poller
+  extend self
+
+  TRAEFIK_UPDATE_DELAY = 5
+
+  class HealthcheckError < StandardError; end
+
+  def wait_for_healthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "healthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      when "running" # No health check configured
+        sleep KAMAL.config.readiness_delay if pause_after_ready
+      else
+        raise HealthcheckError, "container not ready (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is healthy!"
+  end
+
+  def wait_for_unhealthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "unhealthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      else
+        raise HealthcheckError, "container not unhealthy (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is unhealthy!"
+  end
+
+  private
+    def info(message)
+      SSHKit.config.output.info(message)
+    end
+end
diff --git a/lib/kamal/utils/healthcheck_poller.rb b/lib/kamal/utils/healthcheck_poller.rb
diff --git a/test/cli/healthcheck_test.rb b/test/cli/healthcheck_test.rb
@@ -5,7 +5,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
     Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -35,7 +35,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)

diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
@@ -216,7 +216,7 @@ class CliMainTest < CliTestCase
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)