From 3c3a18aa5b2f0228d489566bb255912c0f0e3892 Mon Sep 17 00:00:00 2001
From: Sven Kieske <kieske@osism.tech>
Date: Thu, 29 Aug 2024 13:51:18 +0200
Subject: [PATCH] Rabbitmq: enable quorum for transient queues

This helps to improve the reliability of openstack services when
a rabbitmq node has issues.

See also: https://bugs.launchpad.net/oslo.messaging/+bug/2031497

Adjust the upgrade tests similar to what was done
during the introduction of quorum queues in I6c033d460a5c9b93c346e9e47e93b159d3c27830

Closes-Bug: #2078339
Partial-Bug: #2077448
Depends-On: https://review.opendev.org/c/openstack/oslo.messaging/+/888479
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/924623
Signed-off-by: Sven Kieske <kieske@osism.tech>
Change-Id: Idb8a8d2e560206f7697c0771c9ae3913268fa6dd
---
 ansible/group_vars/all.yml                    |  5 +++
 ansible/roles/aodh/templates/aodh.conf.j2     |  3 ++
 .../roles/barbican/templates/barbican.conf.j2 |  3 ++
 ansible/roles/blazar/templates/blazar.conf.j2 |  3 ++
 .../ceilometer/templates/ceilometer.conf.j2   |  3 ++
 ansible/roles/cinder/templates/cinder.conf.j2 |  3 ++
 .../cloudkitty/templates/cloudkitty.conf.j2   |  3 ++
 ansible/roles/cyborg/templates/cyborg.conf.j2 |  3 ++
 .../designate/templates/designate.conf.j2     |  3 ++
 .../roles/glance/templates/glance-api.conf.j2 |  3 ++
 ansible/roles/heat/templates/heat.conf.j2     |  3 ++
 .../ironic/templates/ironic-inspector.conf.j2 |  3 ++
 ansible/roles/ironic/templates/ironic.conf.j2 |  3 ++
 .../roles/keystone/templates/keystone.conf.j2 |  3 ++
 ansible/roles/magnum/templates/magnum.conf.j2 |  3 ++
 ansible/roles/manila/templates/manila.conf.j2 |  3 ++
 .../roles/masakari/templates/masakari.conf.j2 |  3 ++
 .../roles/mistral/templates/mistral.conf.j2   |  3 ++
 .../roles/neutron/templates/neutron.conf.j2   |  3 ++
 .../roles/nova-cell/templates/nova.conf.j2    |  3 ++
 ansible/roles/nova/templates/nova.conf.j2     |  3 ++
 .../roles/octavia/templates/octavia.conf.j2   |  3 ++
 ansible/roles/rabbitmq/tasks/precheck.yml     | 17 +++++++++-
 ansible/roles/tacker/templates/tacker.conf.j2 |  3 ++
 .../trove/templates/trove-guestagent.conf.j2  |  3 ++
 ansible/roles/trove/templates/trove.conf.j2   |  3 ++
 .../roles/watcher/templates/watcher.conf.j2   |  3 ++
 ansible/roles/zun/templates/zun.conf.j2       |  3 ++
 ...rum_queues_transient-020b373831acda36.yaml | 12 +++++++
 tests/upgrade.sh                              | 31 +++++++++++++++++++
 30 files changed, 142 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 7452ef953d..867feaf952 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -278,6 +278,11 @@ om_enable_rabbitmq_high_availability: false
 # Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
 om_enable_rabbitmq_quorum_queues: true
 om_enable_queue_manager: true
+# Enabling om_enable_rabbitmq_transient_quorum_queue will have no effect
+# without also enabling om_enable_rabbitmq_quorum_queues.
+# om_enable_rabbitmq_transient_quorum_queue should only be used in conunction
+# with om_enable_queue_manager.
+om_enable_rabbitmq_transient_quorum_queue: "{{ om_enable_rabbitmq_quorum_queues | bool and om_enable_queue_manager | bool }}"
 
 ####################
 # Networking options
diff --git a/ansible/roles/aodh/templates/aodh.conf.j2 b/ansible/roles/aodh/templates/aodh.conf.j2
index 9c7ca11d95..f94a491967 100644
--- a/ansible/roles/aodh/templates/aodh.conf.j2
+++ b/ansible/roles/aodh/templates/aodh.conf.j2
@@ -77,6 +77,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/barbican/templates/barbican.conf.j2 b/ansible/roles/barbican/templates/barbican.conf.j2
index 00de9baa9d..f5827446b5 100644
--- a/ansible/roles/barbican/templates/barbican.conf.j2
+++ b/ansible/roles/barbican/templates/barbican.conf.j2
@@ -97,6 +97,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/blazar/templates/blazar.conf.j2 b/ansible/roles/blazar/templates/blazar.conf.j2
index e75c918687..7ec2394402 100644
--- a/ansible/roles/blazar/templates/blazar.conf.j2
+++ b/ansible/roles/blazar/templates/blazar.conf.j2
@@ -68,6 +68,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if blazar_policy_file is defined %}
diff --git a/ansible/roles/ceilometer/templates/ceilometer.conf.j2 b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
index c39e73ddb0..a09d5125c4 100644
--- a/ansible/roles/ceilometer/templates/ceilometer.conf.j2
+++ b/ansible/roles/ceilometer/templates/ceilometer.conf.j2
@@ -49,6 +49,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ceilometer_policy_file is defined %}
diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2
index 55e137cb20..71466a7ca0 100644
--- a/ansible/roles/cinder/templates/cinder.conf.j2
+++ b/ansible/roles/cinder/templates/cinder.conf.j2
@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
index 0526e11332..409ce44c87 100644
--- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
+++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2
@@ -61,6 +61,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [collect]
diff --git a/ansible/roles/cyborg/templates/cyborg.conf.j2 b/ansible/roles/cyborg/templates/cyborg.conf.j2
index 58b1169bf5..8b74a76228 100644
--- a/ansible/roles/cyborg/templates/cyborg.conf.j2
+++ b/ansible/roles/cyborg/templates/cyborg.conf.j2
@@ -83,6 +83,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/designate/templates/designate.conf.j2 b/ansible/roles/designate/templates/designate.conf.j2
index 2b61fce59f..964ea2e5b3 100644
--- a/ansible/roles/designate/templates/designate.conf.j2
+++ b/ansible/roles/designate/templates/designate.conf.j2
@@ -106,6 +106,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2
index dcbcd6972d..66d3533ffe 100644
--- a/ansible/roles/glance/templates/glance-api.conf.j2
+++ b/ansible/roles/glance/templates/glance-api.conf.j2
@@ -126,6 +126,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if glance_policy_file is defined %}
diff --git a/ansible/roles/heat/templates/heat.conf.j2 b/ansible/roles/heat/templates/heat.conf.j2
index 2121996b29..fcae67459c 100644
--- a/ansible/roles/heat/templates/heat.conf.j2
+++ b/ansible/roles/heat/templates/heat.conf.j2
@@ -88,6 +88,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if heat_policy_file is defined %}
diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
index d45f40d5f4..73e9732b75 100644
--- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2
+++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2
@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [ironic]
diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2
index 92e4e485e1..9b3bcffa45 100644
--- a/ansible/roles/ironic/templates/ironic.conf.j2
+++ b/ansible/roles/ironic/templates/ironic.conf.j2
@@ -57,6 +57,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ironic_policy_file is defined %}
diff --git a/ansible/roles/keystone/templates/keystone.conf.j2 b/ansible/roles/keystone/templates/keystone.conf.j2
index cf44e09911..07bb7f04d5 100644
--- a/ansible/roles/keystone/templates/keystone.conf.j2
+++ b/ansible/roles/keystone/templates/keystone.conf.j2
@@ -76,6 +76,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}
diff --git a/ansible/roles/magnum/templates/magnum.conf.j2 b/ansible/roles/magnum/templates/magnum.conf.j2
index ce97de7abf..27a76d48dc 100644
--- a/ansible/roles/magnum/templates/magnum.conf.j2
+++ b/ansible/roles/magnum/templates/magnum.conf.j2
@@ -140,6 +140,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if magnum_policy_file is defined %}
diff --git a/ansible/roles/manila/templates/manila.conf.j2 b/ansible/roles/manila/templates/manila.conf.j2
index 2383380e35..45d3a69d47 100644
--- a/ansible/roles/manila/templates/manila.conf.j2
+++ b/ansible/roles/manila/templates/manila.conf.j2
@@ -70,6 +70,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/masakari/templates/masakari.conf.j2 b/ansible/roles/masakari/templates/masakari.conf.j2
index dff6040b48..3c05472215 100644
--- a/ansible/roles/masakari/templates/masakari.conf.j2
+++ b/ansible/roles/masakari/templates/masakari.conf.j2
@@ -67,6 +67,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]
diff --git a/ansible/roles/mistral/templates/mistral.conf.j2 b/ansible/roles/mistral/templates/mistral.conf.j2
index c7f03475f1..a983fcd51a 100644
--- a/ansible/roles/mistral/templates/mistral.conf.j2
+++ b/ansible/roles/mistral/templates/mistral.conf.j2
@@ -86,6 +86,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if mistral_policy_file is defined %}
diff --git a/ansible/roles/neutron/templates/neutron.conf.j2 b/ansible/roles/neutron/templates/neutron.conf.j2
index 3b7e4063b2..178312a686 100644
--- a/ansible/roles/neutron/templates/neutron.conf.j2
+++ b/ansible/roles/neutron/templates/neutron.conf.j2
@@ -145,6 +145,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if neutron_policy_file is defined %}
diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2
index a52132e31d..9db0bf050f 100644
--- a/ansible/roles/nova-cell/templates/nova.conf.j2
+++ b/ansible/roles/nova-cell/templates/nova.conf.j2
@@ -205,6 +205,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
diff --git a/ansible/roles/nova/templates/nova.conf.j2 b/ansible/roles/nova/templates/nova.conf.j2
index 0b7680f814..6be659dd06 100644
--- a/ansible/roles/nova/templates/nova.conf.j2
+++ b/ansible/roles/nova/templates/nova.conf.j2
@@ -155,6 +155,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
diff --git a/ansible/roles/octavia/templates/octavia.conf.j2 b/ansible/roles/octavia/templates/octavia.conf.j2
index 9a33e74871..71eebc4c18 100644
--- a/ansible/roles/octavia/templates/octavia.conf.j2
+++ b/ansible/roles/octavia/templates/octavia.conf.j2
@@ -139,6 +139,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if octavia_policy_file is defined %}
diff --git a/ansible/roles/rabbitmq/tasks/precheck.yml b/ansible/roles/rabbitmq/tasks/precheck.yml
index 3df9548b81..a56b0ebe4c 100644
--- a/ansible/roles/rabbitmq/tasks/precheck.yml
+++ b/ansible/roles/rabbitmq/tasks/precheck.yml
@@ -149,9 +149,24 @@
       loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
       loop_control:
         label: "{{ item.name }}"
-      # TODO(mattcress): remove skipping reply and fanout once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged.
       when: not (item.name is search('reply_') or item.name is search('_fanout'))
 
+    - name: Check if RabbitMQ quorum queues for transient queues need to be configured
+      assert:
+        that: "{{ item.type == 'quorum' }}"
+        fail_msg: >
+          om_enable_rabbitmq_transient_quorum_queue is True but {{ item.name }} is a non-quorum queue.
+          Currently the procedure to migrate to quorum queues is manual.
+          Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability.
+          Note that this process may take several hours on larger systems, and may cause a degradation in performance at large scale.
+          If you do not wish to enable this feature, set om_enable_rabbitmq_transient_quorum_queue to False.
+      loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when:
+        - item.name is search('reply_') or item.name is search('_fanout')
+        - om_enable_rabbitmq_transient_quorum_queue | bool
+
   run_once: true
   when:
     - container_facts.containers['rabbitmq'] is defined
diff --git a/ansible/roles/tacker/templates/tacker.conf.j2 b/ansible/roles/tacker/templates/tacker.conf.j2
index 8f961ac69e..ca3bb576e9 100644
--- a/ansible/roles/tacker/templates/tacker.conf.j2
+++ b/ansible/roles/tacker/templates/tacker.conf.j2
@@ -82,6 +82,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if tacker_policy_file is defined %}
diff --git a/ansible/roles/trove/templates/trove-guestagent.conf.j2 b/ansible/roles/trove/templates/trove-guestagent.conf.j2
index b9f9e6f772..7b4be3f81e 100644
--- a/ansible/roles/trove/templates/trove-guestagent.conf.j2
+++ b/ansible/roles/trove/templates/trove-guestagent.conf.j2
@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]
diff --git a/ansible/roles/trove/templates/trove.conf.j2 b/ansible/roles/trove/templates/trove.conf.j2
index bbc7c959a2..edc0ab4396 100644
--- a/ansible/roles/trove/templates/trove.conf.j2
+++ b/ansible/roles/trove/templates/trove.conf.j2
@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}
diff --git a/ansible/roles/watcher/templates/watcher.conf.j2 b/ansible/roles/watcher/templates/watcher.conf.j2
index 844a1c2b12..63cac83524 100644
--- a/ansible/roles/watcher/templates/watcher.conf.j2
+++ b/ansible/roles/watcher/templates/watcher.conf.j2
@@ -73,6 +73,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if watcher_policy_file is defined %}
diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2
index 08221cadc7..b46765f4c1 100644
--- a/ansible/roles/zun/templates/zun.conf.j2
+++ b/ansible/roles/zun/templates/zun.conf.j2
@@ -143,4 +143,7 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
diff --git a/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml b/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml
new file mode 100644
index 0000000000..40be107293
--- /dev/null
+++ b/releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml
@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Make use of quorum queues for transient/fanout queues.
+    This helps OpenStack services to be more resilient to message broker
+    failures.
+    This feature is enabled by default.
+    You can disable it by setting the following in your ``globals.yml``:
+    ``om_enable_rabbitmq_transient_quorum_queue: false``.
+    Notice that this disables quorum queues for all transient/fanout message
+    exchanges and likely reduces the resilience of the system.
+    See also `LP#2078339 <https://bugs.launchpad.net/kolla-ansible/+bug/2078339>`_.
diff --git a/tests/upgrade.sh b/tests/upgrade.sh
index 3cd218eb0d..e75699ca99 100755
--- a/tests/upgrade.sh
+++ b/tests/upgrade.sh
@@ -13,7 +13,38 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    # Skip rabbitmq-ha-precheck before the queues are migrated.
+    kolla-ansible prechecks -i ${RAW_INVENTORY} --skip-tags rabbitmq-ha-precheck -vvv &> /tmp/logs/ansible/upgrade-prechecks-pre-rabbitmq
+
+    # NOTE(SvenKieske): As om_enable_rabbitmq_transient_quorum_queue now also
+    # enables quorum_queues for fanout/reply queues in Epoxy, we need
+    # to perform a migration to durable queues.
+    # TODO(SvenKieske): Remove these steps in F Cycle.
+    SERVICE_TAGS="heat,keystone,neutron,nova"
+    if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
+        SERVICE_TAGS+=",cinder"
+    fi
+    if [[ $SCENARIO == "scenario_nfv" ]]; then
+        SERVICE_TAGS+=",barbican"
+    fi
+    if [[ $SCENARIO == "ironic" ]]; then
+        SERVICE_TAGS+=",ironic"
+    fi
+    if [[ $SCENARIO == "masakari" ]]; then
+        SERVICE_TAGS+=",masakari"
+    fi
+    if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then
+        SERVICE_TAGS+=",octavia"
+    fi
+    if [[ $SCENARIO == "magnum" ]]; then
+        SERVICE_TAGS+=",magnum,designate"
+    fi
+    kolla-ansible stop -i ${RAW_INVENTORY} -vvv --tags $SERVICE_TAGS --yes-i-really-really-mean-it --ignore-missing &> /tmp/logs/ansible/stop
+    kolla-ansible genconfig -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/genconfig
+    kolla-ansible rabbitmq-reset-state -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/rabbitmq-reset-state
+    # Include rabbitmq-ha-precheck this time to confirm all queues have migrated.
     kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-prechecks
+
     kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible upgrade -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade