Refactor rabbitmq_upgrade behavior

At the moment rabbitmq_upgrade will temporary stop all nodes in the
cluster, shortly disrupting operations.
This behavior is now changed to perform a regular rolling upgrade [1]

Community right now relying on the rabbitmq_upgrade variable to
"recover" the cluster state by basically re-bootstrapping the cluster.
Such behaviour should be implemented separately in a follow-up patch.

[1] https://www.rabbitmq.com/docs/upgrade#rolling-upgrade

Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/931801
Change-Id: Ibb3fae0d85dc31237ce169bb4cde9551dd5cabb6

releasenotes/notes/rabbitmq_upgrade_refactor-4dfd860f76610a28.yaml

@@ -0,0 +1,9 @@
+---
+prelude: >
+    Behavior of ``rabbitmq_upgrade`` variable has changed. It no longer can
+    be used to reset the cluster state. It now acts solely as a flag to perform
+    a rolling-upgrade of the existing cluster.
+other:
+  - |
+    Supplying ``rabbitmq_upgrade=true`` no longer re-initialize cluster and
+    will perform regular rolling upgrade of the RabbitMQ cluster.

tasks/rabbitmq_upgrade_check.yml

@@ -69,12 +69,41 @@
     - rabbitmq-package-rpm
     - rabbitmq-apt-packages
 
-- name: Including rabbitmq_upgrade_prep tasks
-  include_tasks: rabbitmq_upgrade_prep.yml
+- name: Including rabbitmq_cluster_state tasks
+  include_tasks: rabbitmq_cluster_state.yml
   args:
     apply:
       tags:
         - rabbitmq-upgrade
-  when: rabbitmq_upgrade | bool
+  when:
+    - rabbitmq_upgrade | bool
   tags:
-    - always
+    - rabbitmq-upgrade
+
+- name: Prepare node for upgrade
+  when:
+    - rabbitmq_upgrade | bool
+    - _cluster_state
+    - (_cluster_state.get('running_nodes', []) | length) == (groups[rabbitmq_host_group] | length)
+    - (_cluster_state.get('alarms', []) | length) == 0
+    - (_cluster_state.get('partitions', []) | length) == 0
+  block:
+    - name: Including rabbitmq_feature_flags tasks
+      include_tasks: rabbitmq_feature_flags.yml
+      args:
+        apply:
+          tags:
+            - rabbitmq-upgrade
+      when:
+        - _rabbitmq_is_first_play_host
+      tags:
+        - rabbitmq-upgrade
+
+    - name: Including rabbitmq_upgrade_prep tasks
+      include_tasks: rabbitmq_upgrade_prep.yml
+      args:
+        apply:
+          tags:
+            - rabbitmq-upgrade
+      tags:
+        - always

tasks/rabbitmq_upgrade_prep.yml

@@ -23,13 +23,13 @@
     - rabbitmq-policy-file
     - rabbitmq-upgrade
 
-- name: Including rabbitmq_stopped tasks
-  include_tasks: rabbitmq_stopped.yml
+- name: Set node for maintenance state
+  command: rabbitmq-upgrade drain --quiet
+  changed_when: False
 
-- name: Ensure erlang epmd is stopped
-  shell: |
-    pkill $(pgrep -lf "erlang" | awk '{print $2}')
-  changed_when: false
-  failed_when: false
-  tags:
-    - rabbitmq-upgrade
+- name: Pause to allow node to sync
+  pause:
+    seconds: 10
+
+- name: Including rabbitmq_stopped tasks
+  import_tasks: rabbitmq_stopped.yml

vars/main.yml

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+_rabbitmq_is_first_play_host: >-
+  {{
+    (inventory_hostname == (groups[rabbitmq_host_group] | intersect(ansible_play_hosts)) | first) | bool
+  }}
+
 _rabbitmq_is_last_play_host: >-
   {{
     (inventory_hostname == (groups[rabbitmq_host_group] | intersect(ansible_play_hosts)) | last) | bool