From e117484820a246fb714ae03b83ec80fbe0692371 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Mon, 5 Aug 2024 18:57:18 +0700
Subject: [PATCH] feat(dashmate): rolling restart (#656)

* feat(dashmate): rolling restart

* style: fix linter errors

* style: fix more linter errors

* style: and more linter error fixes

* chore: add changed_when
---
 ansible/roles/dashmate/defaults/main.yml      |  4 +++
 ansible/roles/dashmate/tasks/main.yml         | 32 ++++++++++++++-----
 .../roles/dashmate/tasks/rolling_restart.yml  | 21 ++++++++++++
 3 files changed, 49 insertions(+), 8 deletions(-)
 create mode 100644 ansible/roles/dashmate/tasks/rolling_restart.yml

diff --git a/ansible/roles/dashmate/defaults/main.yml b/ansible/roles/dashmate/defaults/main.yml
index d6294c96..0e228b1a 100644
--- a/ansible/roles/dashmate/defaults/main.yml
+++ b/ansible/roles/dashmate/defaults/main.yml
@@ -76,3 +76,7 @@ dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_timeout:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_delta:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_vote_timeout:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_vote_delta:
+
+
+restart_chunk_size: 10
+restart_delay_minutes: 10
diff --git a/ansible/roles/dashmate/tasks/main.yml b/ansible/roles/dashmate/tasks/main.yml
index bf1503d4..25e6ba6e 100644
--- a/ansible/roles/dashmate/tasks/main.yml
+++ b/ansible/roles/dashmate/tasks/main.yml
@@ -255,17 +255,33 @@
 # dashmate version is changed => restart all
 # dashmate config is changed => restart all
 
-- name: Restart all dashmate services
-  ansible.builtin.command: "{{ dashmate_cmd }} restart --safe --verbose"
-  become: true
-  become_user: dashmate
-  args:
-    chdir: '{{ dashmate_cwd }}'
-  register: dashmate_restart_all
+- name: Create host chunks to restart
+  ansible.builtin.set_fact:
+    host_chunks: "{{ play_hosts | batch(restart_chunk_size) }}"
+  when:
+    - not dashmate_start_all.changed
+    - is_dashmate_package_changed or is_dashmate_config_changed
+  run_once: true
+
+- name: Print chunks info
+  ansible.builtin.debug:
+    msg: "Split {{ play_hosts | length }} into {{ host_chunks | length }} chunks to restart all services"
+  when:
+    - not dashmate_start_all.changed
+    - is_dashmate_package_changed or is_dashmate_config_changed
+  run_once: true
+
+- name: Restart all dashmate services by chunks
+  ansible.builtin.include_tasks: rolling_restart.yml
+  loop: "{{ host_chunks }}"
+  loop_control:
+    loop_var: current_chunk
+    index_var: current_chunk_index
+    label: "{{ current_chunk }}"
   when:
     - not dashmate_start_all.changed
     - is_dashmate_package_changed or is_dashmate_config_changed
-  changed_when: dashmate_restart_all.rc == 0
+  run_once: true
 
 # Force start logic
 # Force start is equivalent to `docker compose up` and will replace services that have changed
diff --git a/ansible/roles/dashmate/tasks/rolling_restart.yml b/ansible/roles/dashmate/tasks/rolling_restart.yml
new file mode 100644
index 00000000..db4de66f
--- /dev/null
+++ b/ansible/roles/dashmate/tasks/rolling_restart.yml
@@ -0,0 +1,21 @@
+---
+
+- name: Restart all dashmate services for current chunk
+  ansible.builtin.command: "{{ dashmate_cmd }} restart --safe --verbose"
+  become: true
+  become_user: dashmate
+  args:
+    chdir: '{{ dashmate_cwd }}'
+  register: dashmate_restart_all
+  changed_when: dashmate_restart_all.rc == 0
+  delegate_to: "{{ item }}"
+  with_items: "{{ current_chunk }}"
+
+- name: Print progress
+  ansible.builtin.debug:
+    msg: "Restarted chunk {{ current_chunk_index + 1 }} of {{ host_chunks | length }}"
+
+- name: Wait for delay before restarting next chunk
+  ansible.builtin.pause:
+    minutes: "{{ restart_delay_minutes }}"
+  when: current_chunk_index + 1 < host_chunks|length