feat(dashmate): rolling restart (#656)

* feat(dashmate): rolling restart

* style: fix linter errors

* style: fix more linter errors

* style: and more linter error fixes

* chore: add changed_when

ansible/roles/dashmate/defaults/main.yml

@@ -76,3 +76,7 @@ dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_timeout:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_delta:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_vote_timeout:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_vote_delta:
+
+
+restart_chunk_size: 10
+restart_delay_minutes: 10

ansible/roles/dashmate/tasks/main.yml

@@ -255,17 +255,33 @@
 # dashmate version is changed => restart all
 # dashmate config is changed => restart all
 
-- name: Restart all dashmate services
-  ansible.builtin.command: "{{ dashmate_cmd }} restart --safe --verbose"
-  become: true
-  become_user: dashmate
-  args:
-    chdir: '{{ dashmate_cwd }}'
-  register: dashmate_restart_all
+- name: Create host chunks to restart
+  ansible.builtin.set_fact:
+    host_chunks: "{{ play_hosts | batch(restart_chunk_size) }}"
+  when:
+    - not dashmate_start_all.changed
+    - is_dashmate_package_changed or is_dashmate_config_changed
+  run_once: true
+
+- name: Print chunks info
+  ansible.builtin.debug:
+    msg: "Split {{ play_hosts | length }} into {{ host_chunks | length }} chunks to restart all services"
+  when:
+    - not dashmate_start_all.changed
+    - is_dashmate_package_changed or is_dashmate_config_changed
+  run_once: true
+
+- name: Restart all dashmate services by chunks
+  ansible.builtin.include_tasks: rolling_restart.yml
+  loop: "{{ host_chunks }}"
+  loop_control:
+    loop_var: current_chunk
+    index_var: current_chunk_index
+    label: "{{ current_chunk }}"
   when:
     - not dashmate_start_all.changed
     - is_dashmate_package_changed or is_dashmate_config_changed
-  changed_when: dashmate_restart_all.rc == 0
+  run_once: true
 
 # Force start logic
 # Force start is equivalent to `docker compose up` and will replace services that have changed

ansible/roles/dashmate/tasks/rolling_restart.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Restart all dashmate services for current chunk
+  ansible.builtin.command: "{{ dashmate_cmd }} restart --safe --verbose"
+  become: true
+  become_user: dashmate
+  args:
+    chdir: '{{ dashmate_cwd }}'
+  register: dashmate_restart_all
+  changed_when: dashmate_restart_all.rc == 0
+  delegate_to: "{{ item }}"
+  with_items: "{{ current_chunk }}"
+
+- name: Print progress
+  ansible.builtin.debug:
+    msg: "Restarted chunk {{ current_chunk_index + 1 }} of {{ host_chunks | length }}"
+
+- name: Wait for delay before restarting next chunk
+  ansible.builtin.pause:
+    minutes: "{{ restart_delay_minutes }}"
+  when: current_chunk_index + 1 < host_chunks|length