enable quorum queues

This implements a global toggle `om_enable_rabbitmq_quorum_queues`
to enable quorum queues for each service in RabbitMQ, similar to
what was done for HA[0].

Quorum Queues are enabled by default.

Quorum queues are more reliable, safer, simpler and faster than
replicated mirrored classic queues[1].

Mirrored classic queues are deprecated and scheduled for removal
in RabbitMQ 4.0[2].

Notice, that we do not need a new policy in the RabbitMQ definitions
template, because their usage is enabled on the client side and can't
be set using a policy[3].

Notice also, that quorum queues are not yet enabled in oslo.messaging
for the usage of reply_ and fanout_ queues (transient queues).
This will change once[4] is merged.

[0]: https://review.opendev.org/c/openstack/kolla-ansible/+/867771
[1]: https://www.rabbitmq.com/quorum-queues.html
[2]: https://blog.rabbitmq.com/posts/2021/08/4.0-deprecation-announcements/
[3]: https://www.rabbitmq.com/quorum-queues.html#declaring
[4]: https://review.opendev.org/c/openstack/oslo.messaging/+/888479

Signed-off-by: Sven Kieske <kieske@osism.tech>
Change-Id: I6c033d460a5c9b93c346e9e47e93b159d3c27830
This commit is contained in:
Sven Kieske 2023-10-17 11:38:23 +02:00 committed by Michal Nasiadka
parent 9b6a70e441
commit 64575519aa
33 changed files with 151 additions and 4 deletions

View File

@ -272,6 +272,8 @@ om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}"
om_rabbitmq_cacert: "{{ rabbitmq_cacert }}" om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
om_enable_rabbitmq_high_availability: false om_enable_rabbitmq_high_availability: false
# Only enable quorum queues if you disable om_enable_rabbitmq_high_availability
om_enable_rabbitmq_quorum_queues: true
#################### ####################
# Networking options # Networking options

View File

@ -69,3 +69,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}

View File

@ -87,6 +87,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_middleware] [oslo_middleware]
enable_proxy_headers_parsing = True enable_proxy_headers_parsing = True

View File

@ -62,6 +62,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if blazar_policy_file is defined %} {% if blazar_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if ceilometer_policy_file is defined %} {% if ceilometer_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_middleware] [oslo_middleware]
enable_proxy_headers_parsing = True enable_proxy_headers_parsing = True

View File

@ -53,6 +53,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[collect] [collect]
collector = {{ cloudkitty_collector_backend }} collector = {{ cloudkitty_collector_backend }}

View File

@ -66,3 +66,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}

View File

@ -101,6 +101,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_concurrency] [oslo_concurrency]
lock_path = /var/lib/designate/tmp lock_path = /var/lib/designate/tmp

View File

@ -131,6 +131,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if glance_policy_file is defined %} {% if glance_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if heat_policy_file is defined %} {% if heat_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -21,6 +21,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[ironic] [ironic]
{% if ironic_enable_keystone_integration | bool %} {% if ironic_enable_keystone_integration | bool %}

View File

@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if ironic_policy_file is defined %} {% if ironic_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if enable_osprofiler | bool %} {% if enable_osprofiler | bool %}
[profiler] [profiler]

View File

@ -135,6 +135,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if magnum_policy_file is defined %} {% if magnum_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -65,6 +65,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_middleware] [oslo_middleware]
enable_proxy_headers_parsing = True enable_proxy_headers_parsing = True

View File

@ -81,6 +81,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if mistral_policy_file is defined %} {% if mistral_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -70,6 +70,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_middleware] [oslo_middleware]
enable_proxy_headers_parsing = True enable_proxy_headers_parsing = True

View File

@ -145,6 +145,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if neutron_policy_file is defined %} {% if neutron_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -193,6 +193,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %} {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -147,6 +147,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if service_name in nova_services_require_policy_json and nova_policy_file is defined %} {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -72,3 +72,10 @@
Please consult the release notes. Please consult the release notes.
when: when:
- (enable_zun | default()) | bool - (enable_zun | default()) | bool
- name: Validate rabbitmq variables
run_once: True
fail:
msg: |
Please set only one of om_enable_rabbitmq_high_availability or om_enable_rabbitmq_quorum_queues
when: (om_enable_rabbitmq_high_availability | bool) == (om_enable_rabbitmq_quorum_queues | bool)

View File

@ -49,6 +49,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if sahara_policy_file is defined %} {% if sahara_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if senlin_policy_file is defined %} {% if senlin_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -76,3 +76,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}

View File

@ -77,6 +77,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if tacker_policy_file is defined %} {% if tacker_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -83,6 +83,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if enable_osprofiler | bool %} {% if enable_osprofiler | bool %}
[profiler] [profiler]

View File

@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
[oslo_concurrency] [oslo_concurrency]
lock_path = /var/lib/vitrage/tmp lock_path = /var/lib/vitrage/tmp

View File

@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}
{% if watcher_policy_file is defined %} {% if watcher_policy_file is defined %}
[oslo_policy] [oslo_policy]

View File

@ -135,3 +135,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }}
{% if om_enable_rabbitmq_high_availability | bool %} {% if om_enable_rabbitmq_high_availability | bool %}
amqp_durable_queues = true amqp_durable_queues = true
{% endif %} {% endif %}
{% if om_enable_rabbitmq_quorum_queues | bool %}
rabbit_quorum_queue = true
{% endif %}

View File

@ -113,15 +113,23 @@ https://www.rabbitmq.com/runtime.html#busy-waiting.
High Availability High Availability
~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~
RabbitMQ offers two features that, when used together, allow for high RabbitMQ offers two options to configure HA:
availability. These are durable queues and classic queue mirroring. Setting the * Quorum queues (enabled by default and controlled by
flag ``om_enable_rabbitmq_high_availability`` to ``true`` will enable both of ``om_enable_rabbitmq_quorum_queues`` variable)
these features. There are some queue types which are intentionally not mirrored * Classic queue mirroring and durable queues (deprecated in RabbitMQ and to
be dropped in 4.0, controlled by ``om_enable_rabbitmq_high_availability``)
There are some queue types which are intentionally not mirrored
using the exclusionary pattern ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``. using the exclusionary pattern ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``.
After enabling this value on a running system, there are some additional steps After enabling this value on a running system, there are some additional steps
needed to migrate from transient to durable queues. needed to migrate from transient to durable queues.
.. warning::
Since the default changed from non-HA to Quorum queues in Bobcat release,
following procedure is required to be carried out before an upgrade.
1. Stop all OpenStack services which use RabbitMQ, so that they will not 1. Stop all OpenStack services which use RabbitMQ, so that they will not
attempt to recreate any queues yet. attempt to recreate any queues yet.

View File

@ -0,0 +1,18 @@
---
features:
- |
You can now enable the usage of quorum queues in RabbitMQ for all services
by setting the variable ``om_enable_rabbitmq_quorum_queues`` to ``true``.
Notice that you can't use quorum queues and high availability at the same
time. This is caught by a precheck.
This feature is enabled by default to improve reliability of the messaging
queues.
upgrade:
- |
Quorum queues in RabbitMQ (controlled by
``om_enable_rabbitmq_quorum_queues`` variable) is enabled by default from
now on.
Support for non-HA RabbitMQ queues is dropped. Either quorum queues that
are enabled by default, or classic mirrored queues are required now.
`Migration procedure from non-HA to HA
<https://docs.openstack.org/kolla-ansible/zed/reference/message-queues/rabbitmq.html#high-availability>`__

View File

@ -13,6 +13,34 @@ function upgrade {
source $KOLLA_ANSIBLE_VENV_PATH/bin/activate source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
# NOTE(mattcrees): As om_enable_rabbitmq_quorum_queues now defaults to
# true in Bobcat, we need to perform a migration to durable queues.
# TODO(mattcrees): Remove these steps in Caracal.
SERVICE_TAGS="heat,keystone,neutron,nova"
if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
SERVICE_TAGS+=",cinder"
fi
if [[ $SCENARIO == "scenario_nfv" ]]; then
SERVICE_TAGS+=",barbican"
fi
if [[ $SCENARIO == "ironic" ]]; then
SERVICE_TAGS+=",ironic"
fi
if [[ $SCENARIO == "masakari" ]]; then
SERVICE_TAGS+=",masakari"
fi
if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then
SERVICE_TAGS+=",octavia"
fi
if [[ $SCENARIO == "magnum" ]]; then
SERVICE_TAGS+=",magnum,designate"
fi
kolla-ansible -i ${RAW_INVENTORY} -vvv stop --tags $SERVICE_TAGS --yes-i-really-really-mean-it &> /tmp/logs/ansible/stop
kolla-ansible -i ${RAW_INVENTORY} -vvv genconfig &> /tmp/logs/ansible/genconfig
kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure --tags rabbitmq &> /tmp/logs/ansible/reconfigure-rabbitmq
kolla-ansible -i ${RAW_INVENTORY} -vvv rabbitmq-reset-state &> /tmp/logs/ansible/rabbitmq-reset-state
kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade