From 9847546fe1490abfd626a4c9a98c939a7811174a Mon Sep 17 00:00:00 2001
From: Dmitriy Rabotyagov <dmitriy.rabotyagov@cleura.com>
Date: Wed, 9 Oct 2024 16:55:14 +0200
Subject: [PATCH] Refactor rabbitmq_upgrade behavior

At the moment rabbitmq_upgrade will temporary stop all nodes in the
cluster, shortly disrupting operations.
This behavior is now changed to perform a regular rolling upgrade [1]

Community right now relying on the rabbitmq_upgrade variable to
"recover" the cluster state by basically re-bootstrapping the cluster.
Such behaviour should be implemented separately in a follow-up patch.

[1] https://www.rabbitmq.com/docs/upgrade#rolling-upgrade

Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/931801
Change-Id: Ibb3fae0d85dc31237ce169bb4cde9551dd5cabb6
---
 ...tmq_upgrade_refactor-4dfd860f76610a28.yaml |  9 +++++
 tasks/rabbitmq_upgrade_check.yml              | 37 +++++++++++++++++--
 tasks/rabbitmq_upgrade_prep.yml               | 18 ++++-----
 vars/main.yml                                 |  5 +++
 4 files changed, 56 insertions(+), 13 deletions(-)
 create mode 100644 releasenotes/notes/rabbitmq_upgrade_refactor-4dfd860f76610a28.yaml

diff --git a/releasenotes/notes/rabbitmq_upgrade_refactor-4dfd860f76610a28.yaml b/releasenotes/notes/rabbitmq_upgrade_refactor-4dfd860f76610a28.yaml
new file mode 100644
index 00000000..4f120f2f
--- /dev/null
+++ b/releasenotes/notes/rabbitmq_upgrade_refactor-4dfd860f76610a28.yaml
@@ -0,0 +1,9 @@
+---
+prelude: >
+    Behavior of ``rabbitmq_upgrade`` variable has changed. It no longer can
+    be used to reset the cluster state. It now acts solely as a flag to perform
+    a rolling-upgrade of the existing cluster.
+other:
+  - |
+    Supplying ``rabbitmq_upgrade=true`` no longer re-initialize cluster and
+    will perform regular rolling upgrade of the RabbitMQ cluster.
diff --git a/tasks/rabbitmq_upgrade_check.yml b/tasks/rabbitmq_upgrade_check.yml
index 8d39491b..0cb0660a 100644
--- a/tasks/rabbitmq_upgrade_check.yml
+++ b/tasks/rabbitmq_upgrade_check.yml
@@ -69,12 +69,41 @@
     - rabbitmq-package-rpm
     - rabbitmq-apt-packages
 
-- name: Including rabbitmq_upgrade_prep tasks
-  include_tasks: rabbitmq_upgrade_prep.yml
+- name: Including rabbitmq_cluster_state tasks
+  include_tasks: rabbitmq_cluster_state.yml
   args:
     apply:
       tags:
         - rabbitmq-upgrade
-  when: rabbitmq_upgrade | bool
+  when:
+    - rabbitmq_upgrade | bool
   tags:
-    - always
+    - rabbitmq-upgrade
+
+- name: Prepare node for upgrade
+  when:
+    - rabbitmq_upgrade | bool
+    - _cluster_state
+    - (_cluster_state.get('running_nodes', []) | length) == (groups[rabbitmq_host_group] | length)
+    - (_cluster_state.get('alarms', []) | length) == 0
+    - (_cluster_state.get('partitions', []) | length) == 0
+  block:
+    - name: Including rabbitmq_feature_flags tasks
+      include_tasks: rabbitmq_feature_flags.yml
+      args:
+        apply:
+          tags:
+            - rabbitmq-upgrade
+      when:
+        - _rabbitmq_is_first_play_host
+      tags:
+        - rabbitmq-upgrade
+
+    - name: Including rabbitmq_upgrade_prep tasks
+      include_tasks: rabbitmq_upgrade_prep.yml
+      args:
+        apply:
+          tags:
+            - rabbitmq-upgrade
+      tags:
+        - always
diff --git a/tasks/rabbitmq_upgrade_prep.yml b/tasks/rabbitmq_upgrade_prep.yml
index 234eb2f9..c97e232c 100644
--- a/tasks/rabbitmq_upgrade_prep.yml
+++ b/tasks/rabbitmq_upgrade_prep.yml
@@ -23,13 +23,13 @@
     - rabbitmq-policy-file
     - rabbitmq-upgrade
 
-- name: Including rabbitmq_stopped tasks
-  include_tasks: rabbitmq_stopped.yml
+- name: Set node for maintenance state
+  command: rabbitmq-upgrade drain --quiet
+  changed_when: False
 
-- name: Ensure erlang epmd is stopped
-  shell: |
-    pkill $(pgrep -lf "erlang" | awk '{print $2}')
-  changed_when: false
-  failed_when: false
-  tags:
-    - rabbitmq-upgrade
+- name: Pause to allow node to sync
+  pause:
+    seconds: 10
+
+- name: Including rabbitmq_stopped tasks
+  import_tasks: rabbitmq_stopped.yml
diff --git a/vars/main.yml b/vars/main.yml
index 4adb9ce3..8687cbd6 100644
--- a/vars/main.yml
+++ b/vars/main.yml
@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+_rabbitmq_is_first_play_host: >-
+  {{
+    (inventory_hostname == (groups[rabbitmq_host_group] | intersect(ansible_play_hosts)) | first) | bool
+  }}
+
 _rabbitmq_is_last_play_host: >-
   {{
     (inventory_hostname == (groups[rabbitmq_host_group] | intersect(ansible_play_hosts)) | last) | bool