From ec9e45afde1f08ed8e14eec313a74b4a11324026 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 9 Jul 2024 15:05:15 +0200 Subject: [PATCH 1/6] Allow customization of rollout strategies for deployments --- Chart.yaml | 2 +- templates/deployment-sidekiq.yaml | 5 +++-- templates/deployment-streaming.yaml | 8 +++----- templates/deployment-web.yaml | 8 +++----- values.yaml | 18 ++++++++++++++++++ 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 57768b3..8bb36ad 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time # you make changes to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 5.1.3 +version: 5.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index 1870cf4..e998a3c 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -18,8 +18,9 @@ spec: {{- if (gt (int .replicas) 1) }} {{ fail "The scheduler queue should never have more than 1 replicas" }} {{- end }} - strategy: - type: Recreate + {{- end }} + {{- if $context.Values.mastodon.sidekiq.updateStrategy }} + strategy: {{- toYaml $context.Values.mastodon.sidekiq.updateStrategy | nindent 4 }} {{- end }} replicas: {{ .replicas }} {{- if (ne (toString $context.Values.mastodon.revisionHistoryLimit) "") }} diff --git a/templates/deployment-streaming.yaml b/templates/deployment-streaming.yaml index 3165fa2..89bd06a 100644 --- a/templates/deployment-streaming.yaml +++ b/templates/deployment-streaming.yaml @@ -13,11 +13,9 @@ spec: {{- if (ne (toString .Values.mastodon.revisionHistoryLimit) "") }} revisionHistoryLimit: {{ .Values.mastodon.revisionHistoryLimit }} {{- end }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 10% - maxUnavailable: 25% + {{- if .Values.mastodon.web.updateStrategy }} + strategy: {{- toYaml .Values.mastodon.web.updateStrategy | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "mastodon.selectorLabels" . | nindent 6 }} diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index 70d1c29..2956a21 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -13,11 +13,9 @@ spec: {{- if (ne (toString .Values.mastodon.revisionHistoryLimit) "") }} revisionHistoryLimit: {{ .Values.mastodon.revisionHistoryLimit }} {{- end }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 10% - maxUnavailable: 25% + {{- if .Values.mastodon.web.updateStrategy }} + strategy: {{- toYaml .Values.mastodon.web.updateStrategy | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "mastodon.selectorLabels" . | nindent 6 }} diff --git a/values.yaml b/values.yaml index 073fd2e..cd047ec 100644 --- a/values.yaml +++ b/values.yaml @@ -138,6 +138,10 @@ mastodon: resources: {} # -- Affinity for all Sidekiq Deployments unless overwritten, overwrites .Values.affinity affinity: {} + # Rollout strategy to use when updating pods + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: Recreate # -- Topology spread constraints for Sidekiq Pods, overwrites .Values.topologySpreadConstraints topologySpreadConstraints: {} # limits: @@ -227,6 +231,13 @@ mastodon: replicas: 1 # -- Affinity for Streaming Pods, overwrites .Values.affinity affinity: {} + # Rollout strategy to use when updating pods + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 10% + maxUnavailable: 25% # -- Topology spread constraints for Streaming Pods, overwrites .Values.topologySpreadConstraints topologySpreadConstraints: {} # -- Pod Security Context for Streaming Pods, overwrites .Values.podSecurityContext @@ -266,6 +277,13 @@ mastodon: replicas: 1 # -- Affinity for Web Pods, overwrites .Values.affinity affinity: {} + # Rollout strategy to use when updating pods + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 10% + maxUnavailable: 25% # -- Topology spread constraints for Web Pods, overwrites .Values.topologySpreadConstraints topologySpreadConstraints: {} # -- Pod Security Context for Web Pods, overwrites .Values.podSecurityContext From aaab4497584daddcd96f138969a057ee88dbfe0e Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 9 Jul 2024 15:51:10 +0200 Subject: [PATCH 2/6] Added additional comments preparing for sidekiq readiness probes --- templates/deployment-sidekiq.yaml | 15 ++++++++++++++- values.yaml | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index e998a3c..b10573c 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -201,7 +201,20 @@ spec: {{- with $context.Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} - resources: + # NOTE: Readiness probe will only work on versions of Mastodon built after 2024-07-10. + # Will be un-commented once this is in an official release. + # + #readinessProbe: + # failureThreshold: 10 + # exec: + # command: + # - cat + # - /var/www/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs + # initialDelaySeconds: 10 + # periodSeconds: 2 + # successThreshold: 2 + # timeoutSeconds: 1 + #resources: {{- toYaml (default (default $context.Values.resources $context.Values.mastodon.sidekiq.resources) .resources) | nindent 12 }} {{- include "mastodon.statsdExporterContainer" $ | indent 8 }} {{- with $context.Values.nodeSelector }} diff --git a/values.yaml b/values.yaml index cd047ec..827190c 100644 --- a/values.yaml +++ b/values.yaml @@ -138,7 +138,10 @@ mastodon: resources: {} # -- Affinity for all Sidekiq Deployments unless overwritten, overwrites .Values.affinity affinity: {} - # Rollout strategy to use when updating pods + # Rollout strategy to use when updating pods. + # Recreate is recommended for sidekiq pods for now because of possible data + # error between old/new versions. Please only use RollingUpdate if you are + # running a Mastodon version built after 2024-07-10. # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy updateStrategy: type: Recreate From 5d673400ab560c41538aa5580618d6e42a60c7a3 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 9 Jul 2024 16:00:32 +0200 Subject: [PATCH 3/6] Fix small type --- templates/deployment-sidekiq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index b10573c..3182bcc 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -214,7 +214,7 @@ spec: # periodSeconds: 2 # successThreshold: 2 # timeoutSeconds: 1 - #resources: + resources: {{- toYaml (default (default $context.Values.resources $context.Values.mastodon.sidekiq.resources) .resources) | nindent 12 }} {{- include "mastodon.statsdExporterContainer" $ | indent 8 }} {{- with $context.Values.nodeSelector }} From 08f9101c247930b1c52c32ba2237e3b607b029a0 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Wed, 10 Jul 2024 12:27:43 +0200 Subject: [PATCH 4/6] Allow for better customization of readiness probe in sidekiq --- templates/deployment-sidekiq.yaml | 25 ++++++++++++------------- values.yaml | 8 ++++++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index 3182bcc..cacfe7a 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -201,19 +201,18 @@ spec: {{- with $context.Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} - # NOTE: Readiness probe will only work on versions of Mastodon built after 2024-07-10. - # Will be un-commented once this is in an official release. - # - #readinessProbe: - # failureThreshold: 10 - # exec: - # command: - # - cat - # - /var/www/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs - # initialDelaySeconds: 10 - # periodSeconds: 2 - # successThreshold: 2 - # timeoutSeconds: 1 + {{- if $context.Values.mastodon.sidekiq.readinessProbe.enabled }} + readinessProbe: + failureThreshold: {{ default 10 $context.Values.mastodon.sidekiq.readinessProbe.failureThreshold }} + exec: + command: + - cat + - /var/www/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs + initialDelaySeconds: {{ default 10 $context.Values.mastodon.sidekiq.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.periodSeconds }} + successThreshold: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.successThreshold }} + timeoutSeconds: {{ default 1 $context.Values.mastodon.sidekiq.readinessProbe.timeoutSeconds }} + {{- end }} resources: {{- toYaml (default (default $context.Values.resources $context.Values.mastodon.sidekiq.resources) .resources) | nindent 12 }} {{- include "mastodon.statsdExporterContainer" $ | indent 8 }} diff --git a/values.yaml b/values.yaml index 827190c..3d66054 100644 --- a/values.yaml +++ b/values.yaml @@ -145,6 +145,14 @@ mastodon: # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy updateStrategy: type: Recreate + # Readiness probe configuration + # NOTE: Readiness probe will only work on versions of Mastodon built after 2024-07-10. + readinessProbe: + enabled: false + initialDelaySeconds: 10 + periodSeconds: 2 + successThreshold: 2 + timeoutSeconds: 1 # -- Topology spread constraints for Sidekiq Pods, overwrites .Values.topologySpreadConstraints topologySpreadConstraints: {} # limits: From 8c62763e41e440d53f580f09a3526f3d3a65dada Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Wed, 10 Jul 2024 15:15:34 +0200 Subject: [PATCH 5/6] Customize readiness path, update comments --- templates/deployment-sidekiq.yaml | 2 +- values.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index cacfe7a..e473c7d 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -207,7 +207,7 @@ spec: exec: command: - cat - - /var/www/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs + - {{ required "A valid sidekiq readiness path is required." $context.Values.mastodon.sidekiq.readinessProbe.path }} initialDelaySeconds: {{ default 10 $context.Values.mastodon.sidekiq.readinessProbe.initialDelaySeconds }} periodSeconds: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.periodSeconds }} successThreshold: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.successThreshold }} diff --git a/values.yaml b/values.yaml index 3d66054..a081750 100644 --- a/values.yaml +++ b/values.yaml @@ -139,9 +139,11 @@ mastodon: # -- Affinity for all Sidekiq Deployments unless overwritten, overwrites .Values.affinity affinity: {} # Rollout strategy to use when updating pods. - # Recreate is recommended for sidekiq pods for now because of possible data - # error between old/new versions. Please only use RollingUpdate if you are - # running a Mastodon version built after 2024-07-10. + # Recreate will help reduce the number of retried jobs when updating when + # the code introduces a new job as the pods are all replaced immediately. + # RollingUpdate can help with larger clusters if job retries aren't an + # issue, as it will reduce strain by replacing pods more slowly. It is + # strongly recommended to enable the readinessProbe when using RollingUpdate. # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy updateStrategy: type: Recreate @@ -149,6 +151,7 @@ mastodon: # NOTE: Readiness probe will only work on versions of Mastodon built after 2024-07-10. readinessProbe: enabled: false + path: /opt/mastodon/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs initialDelaySeconds: 10 periodSeconds: 2 successThreshold: 2 From 6578d4453f391b4634351e11efc054a8ec85e38c Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Wed, 10 Jul 2024 15:51:00 +0200 Subject: [PATCH 6/6] Adjust default values --- templates/deployment-sidekiq.yaml | 2 +- values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index e473c7d..9eec596 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -210,7 +210,7 @@ spec: - {{ required "A valid sidekiq readiness path is required." $context.Values.mastodon.sidekiq.readinessProbe.path }} initialDelaySeconds: {{ default 10 $context.Values.mastodon.sidekiq.readinessProbe.initialDelaySeconds }} periodSeconds: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.periodSeconds }} - successThreshold: {{ default 2 $context.Values.mastodon.sidekiq.readinessProbe.successThreshold }} + successThreshold: {{ default 1 $context.Values.mastodon.sidekiq.readinessProbe.successThreshold }} timeoutSeconds: {{ default 1 $context.Values.mastodon.sidekiq.readinessProbe.timeoutSeconds }} {{- end }} resources: diff --git a/values.yaml b/values.yaml index a081750..d9782fd 100644 --- a/values.yaml +++ b/values.yaml @@ -154,7 +154,7 @@ mastodon: path: /opt/mastodon/tmp/sidekiq_process_has_started_and_will_begin_processing_jobs initialDelaySeconds: 10 periodSeconds: 2 - successThreshold: 2 + successThreshold: 1 timeoutSeconds: 1 # -- Topology spread constraints for Sidekiq Pods, overwrites .Values.topologySpreadConstraints topologySpreadConstraints: {}