Merge "Rabbitmq: enable quorum for transient queues"

ansible/group_vars/all.yml

@@ -278,6 +278,11 @@ om_enable_rabbitmq_high_availability: false
 # Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
 om_enable_rabbitmq_quorum_queues: true
 om_enable_queue_manager: true
+# Enabling om_enable_rabbitmq_transient_quorum_queue will have no effect
+# without also enabling om_enable_rabbitmq_quorum_queues.
+# om_enable_rabbitmq_transient_quorum_queue should only be used in conunction
+# with om_enable_queue_manager.
+om_enable_rabbitmq_transient_quorum_queue: "{{ om_enable_rabbitmq_quorum_queues | bool and om_enable_queue_manager | bool }}"
 
 ####################
 # Networking options

ansible/roles/aodh/templates/aodh.conf.j2

@@ -77,6 +77,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]

ansible/roles/barbican/templates/barbican.conf.j2

@@ -97,6 +97,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]

ansible/roles/blazar/templates/blazar.conf.j2

@@ -68,6 +68,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if blazar_policy_file is defined %}

ansible/roles/ceilometer/templates/ceilometer.conf.j2

@@ -49,6 +49,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ceilometer_policy_file is defined %}

ansible/roles/cinder/templates/cinder.conf.j2

@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]

ansible/roles/cloudkitty/templates/cloudkitty.conf.j2

@@ -61,6 +61,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [collect]

ansible/roles/cyborg/templates/cyborg.conf.j2

@@ -83,6 +83,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]

ansible/roles/designate/templates/designate.conf.j2

@@ -106,6 +106,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]

ansible/roles/glance/templates/glance-api.conf.j2

@@ -126,6 +126,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if glance_policy_file is defined %}

ansible/roles/heat/templates/heat.conf.j2

@@ -88,6 +88,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if heat_policy_file is defined %}

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [ironic]

ansible/roles/ironic/templates/ironic.conf.j2

@@ -57,6 +57,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if ironic_policy_file is defined %}

ansible/roles/keystone/templates/keystone.conf.j2

@@ -76,6 +76,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}

ansible/roles/magnum/templates/magnum.conf.j2

@@ -140,6 +140,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if magnum_policy_file is defined %}

ansible/roles/manila/templates/manila.conf.j2

@@ -70,6 +70,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]

ansible/roles/masakari/templates/masakari.conf.j2

@@ -67,6 +67,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_middleware]

ansible/roles/mistral/templates/mistral.conf.j2

@@ -86,6 +86,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if mistral_policy_file is defined %}

ansible/roles/neutron/templates/neutron.conf.j2

@@ -145,6 +145,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if neutron_policy_file is defined %}

ansible/roles/nova-cell/templates/nova.conf.j2

@@ -205,6 +205,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}

ansible/roles/nova/templates/nova.conf.j2

@@ -155,6 +155,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}

ansible/roles/octavia/templates/octavia.conf.j2

@@ -140,6 +140,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if octavia_policy_file is defined %}

ansible/roles/rabbitmq/tasks/precheck.yml

@@ -149,9 +149,24 @@
       loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
       loop_control:
         label: "{{ item.name }}"
-      # TODO(mattcress): remove skipping reply and fanout once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged.
       when: not (item.name is search('reply_') or item.name is search('_fanout'))
 
+    - name: Check if RabbitMQ quorum queues for transient queues need to be configured
+      assert:
+        that: "{{ item.type == 'quorum' }}"
+        fail_msg: >
+          om_enable_rabbitmq_transient_quorum_queue is True but {{ item.name }} is a non-quorum queue.
+          Currently the procedure to migrate to quorum queues is manual.
+          Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability.
+          Note that this process may take several hours on larger systems, and may cause a degradation in performance at large scale.
+          If you do not wish to enable this feature, set om_enable_rabbitmq_transient_quorum_queue to False.
+      loop: "{{ (rabbitmq_queues.stdout | from_json) if rabbitmq_queues is not skipped else [] }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when:
+        - item.name is search('reply_') or item.name is search('_fanout')
+        - om_enable_rabbitmq_transient_quorum_queue | bool
+
   run_once: true
   when:
     - container_facts.containers['rabbitmq'] is defined

ansible/roles/tacker/templates/tacker.conf.j2

@@ -82,6 +82,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if tacker_policy_file is defined %}

ansible/roles/trove/templates/trove-guestagent.conf.j2

@@ -26,6 +26,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 [oslo_concurrency]

ansible/roles/trove/templates/trove.conf.j2

@@ -87,6 +87,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if enable_osprofiler | bool %}

ansible/roles/watcher/templates/watcher.conf.j2

@@ -73,6 +73,9 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}
 
 {% if watcher_policy_file is defined %}

ansible/roles/zun/templates/zun.conf.j2

@@ -143,4 +143,7 @@ amqp_durable_queues = true
 {% endif %}
 {% if om_enable_rabbitmq_quorum_queues | bool %}
 rabbit_quorum_queue = true
+{% if om_enable_rabbitmq_transient_quorum_queue | bool %}
+rabbit_transient_quorum_queue = true
+{% endif %}
 {% endif %}

releasenotes/notes/enable_quorum_queues_transient-020b373831acda36.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Make use of quorum queues for transient/fanout queues.
+    This helps OpenStack services to be more resilient to message broker
+    failures.
+    This feature is enabled by default.
+    You can disable it by setting the following in your ``globals.yml``:
+    ``om_enable_rabbitmq_transient_quorum_queue: false``.
+    Notice that this disables quorum queues for all transient/fanout message
+    exchanges and likely reduces the resilience of the system.
+    See also `LP#2078339 <https://bugs.launchpad.net/kolla-ansible/+bug/2078339>`_.

tests/upgrade.sh

@@ -13,7 +13,38 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    # Skip rabbitmq-ha-precheck before the queues are migrated.
+    kolla-ansible prechecks -i ${RAW_INVENTORY} --skip-tags rabbitmq-ha-precheck -vvv &> /tmp/logs/ansible/upgrade-prechecks-pre-rabbitmq
+
+    # NOTE(SvenKieske): As om_enable_rabbitmq_transient_quorum_queue now also
+    # enables quorum_queues for fanout/reply queues in Epoxy, we need
+    # to perform a migration to durable queues.
+    # TODO(SvenKieske): Remove these steps in F Cycle.
+    SERVICE_TAGS="heat,keystone,neutron,nova"
+    if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
+        SERVICE_TAGS+=",cinder"
+    fi
+    if [[ $SCENARIO == "scenario_nfv" ]]; then
+        SERVICE_TAGS+=",barbican"
+    fi
+    if [[ $SCENARIO == "ironic" ]]; then
+        SERVICE_TAGS+=",ironic"
+    fi
+    if [[ $SCENARIO == "masakari" ]]; then
+        SERVICE_TAGS+=",masakari"
+    fi
+    if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then
+        SERVICE_TAGS+=",octavia"
+    fi
+    if [[ $SCENARIO == "magnum" ]]; then
+        SERVICE_TAGS+=",magnum,designate"
+    fi
+    kolla-ansible stop -i ${RAW_INVENTORY} -vvv --tags $SERVICE_TAGS --yes-i-really-really-mean-it --ignore-missing &> /tmp/logs/ansible/stop
+    kolla-ansible genconfig -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/genconfig
+    kolla-ansible rabbitmq-reset-state -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/rabbitmq-reset-state
+    # Include rabbitmq-ha-precheck this time to confirm all queues have migrated.
     kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-prechecks
+
     kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible upgrade -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade