diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index a546e6e8c3..f512f5c03c 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -272,6 +272,8 @@ om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}" om_rabbitmq_cacert: "{{ rabbitmq_cacert }}" om_enable_rabbitmq_high_availability: false +# Only enable quorum queues if you disable om_enable_rabbitmq_high_availability +om_enable_rabbitmq_quorum_queues: true #################### # Networking options diff --git a/ansible/roles/aodh/templates/aodh.conf.j2 b/ansible/roles/aodh/templates/aodh.conf.j2 index 5ec0f60595..dfb4e049d3 100644 --- a/ansible/roles/aodh/templates/aodh.conf.j2 +++ b/ansible/roles/aodh/templates/aodh.conf.j2 @@ -69,3 +69,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} diff --git a/ansible/roles/barbican/templates/barbican.conf.j2 b/ansible/roles/barbican/templates/barbican.conf.j2 index 891508571b..fa7c3e0d5c 100644 --- a/ansible/roles/barbican/templates/barbican.conf.j2 +++ b/ansible/roles/barbican/templates/barbican.conf.j2 @@ -87,6 +87,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_middleware] enable_proxy_headers_parsing = True diff --git a/ansible/roles/blazar/templates/blazar.conf.j2 b/ansible/roles/blazar/templates/blazar.conf.j2 index 2cf03ed53f..a37b899f4c 100644 --- a/ansible/roles/blazar/templates/blazar.conf.j2 +++ b/ansible/roles/blazar/templates/blazar.conf.j2 @@ -62,6 +62,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if blazar_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/ceilometer/templates/ceilometer.conf.j2 b/ansible/roles/ceilometer/templates/ceilometer.conf.j2 index 4a0ab9e03a..309c5379b7 100644 --- a/ansible/roles/ceilometer/templates/ceilometer.conf.j2 +++ b/ansible/roles/ceilometer/templates/ceilometer.conf.j2 @@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if ceilometer_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index e89e2008e0..69cde541a3 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_middleware] enable_proxy_headers_parsing = True diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 index 1fdb8b5c3b..b2f209c2ba 100644 --- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 +++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 @@ -53,6 +53,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [collect] collector = {{ cloudkitty_collector_backend }} diff --git a/ansible/roles/cyborg/templates/cyborg.conf.j2 b/ansible/roles/cyborg/templates/cyborg.conf.j2 index 1bd325906c..8915a1e2b5 100644 --- a/ansible/roles/cyborg/templates/cyborg.conf.j2 +++ b/ansible/roles/cyborg/templates/cyborg.conf.j2 @@ -66,3 +66,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} diff --git a/ansible/roles/designate/templates/designate.conf.j2 b/ansible/roles/designate/templates/designate.conf.j2 index 82db00614e..b74be50182 100644 --- a/ansible/roles/designate/templates/designate.conf.j2 +++ b/ansible/roles/designate/templates/designate.conf.j2 @@ -101,6 +101,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_concurrency] lock_path = /var/lib/designate/tmp diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2 index 4590b8c04c..84856cd7d8 100644 --- a/ansible/roles/glance/templates/glance-api.conf.j2 +++ b/ansible/roles/glance/templates/glance-api.conf.j2 @@ -131,6 +131,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if glance_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/heat/templates/heat.conf.j2 b/ansible/roles/heat/templates/heat.conf.j2 index 43f8c686a3..9584b73ac8 100644 --- a/ansible/roles/heat/templates/heat.conf.j2 +++ b/ansible/roles/heat/templates/heat.conf.j2 @@ -80,6 +80,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if heat_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 index 8781919efe..fd52f1e00b 100644 --- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 +++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 @@ -21,6 +21,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [ironic] {% if ironic_enable_keystone_integration | bool %} diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2 index a48152d3d8..425c936a49 100644 --- a/ansible/roles/ironic/templates/ironic.conf.j2 +++ b/ansible/roles/ironic/templates/ironic.conf.j2 @@ -44,6 +44,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if ironic_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/keystone/templates/keystone.conf.j2 b/ansible/roles/keystone/templates/keystone.conf.j2 index e57ee251d9..ffccd4abec 100644 --- a/ansible/roles/keystone/templates/keystone.conf.j2 +++ b/ansible/roles/keystone/templates/keystone.conf.j2 @@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if enable_osprofiler | bool %} [profiler] diff --git a/ansible/roles/magnum/templates/magnum.conf.j2 b/ansible/roles/magnum/templates/magnum.conf.j2 index 2761486c50..8422a8ce39 100644 --- a/ansible/roles/magnum/templates/magnum.conf.j2 +++ b/ansible/roles/magnum/templates/magnum.conf.j2 @@ -135,6 +135,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if magnum_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/manila/templates/manila.conf.j2 b/ansible/roles/manila/templates/manila.conf.j2 index e959e2ebdf..ae21fdecc7 100644 --- a/ansible/roles/manila/templates/manila.conf.j2 +++ b/ansible/roles/manila/templates/manila.conf.j2 @@ -65,6 +65,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_middleware] enable_proxy_headers_parsing = True diff --git a/ansible/roles/mistral/templates/mistral.conf.j2 b/ansible/roles/mistral/templates/mistral.conf.j2 index 6e621b7b76..97c57eb778 100644 --- a/ansible/roles/mistral/templates/mistral.conf.j2 +++ b/ansible/roles/mistral/templates/mistral.conf.j2 @@ -81,6 +81,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if mistral_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/murano/templates/murano.conf.j2 b/ansible/roles/murano/templates/murano.conf.j2 index 0db2da6c6c..fbebb12f9d 100644 --- a/ansible/roles/murano/templates/murano.conf.j2 +++ b/ansible/roles/murano/templates/murano.conf.j2 @@ -70,6 +70,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_middleware] enable_proxy_headers_parsing = True diff --git a/ansible/roles/neutron/templates/neutron.conf.j2 b/ansible/roles/neutron/templates/neutron.conf.j2 index 7cac74be67..a95106143c 100644 --- a/ansible/roles/neutron/templates/neutron.conf.j2 +++ b/ansible/roles/neutron/templates/neutron.conf.j2 @@ -145,6 +145,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if neutron_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/nova-cell/templates/nova.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.j2 index 4b3d0f1cfe..41cd9a1046 100644 --- a/ansible/roles/nova-cell/templates/nova.conf.j2 +++ b/ansible/roles/nova-cell/templates/nova.conf.j2 @@ -193,6 +193,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/nova/templates/nova.conf.j2 b/ansible/roles/nova/templates/nova.conf.j2 index b7a4fbf058..30288da69a 100644 --- a/ansible/roles/nova/templates/nova.conf.j2 +++ b/ansible/roles/nova/templates/nova.conf.j2 @@ -147,6 +147,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/prechecks/tasks/service_checks.yml b/ansible/roles/prechecks/tasks/service_checks.yml index 0400e07271..1c8d9af56d 100644 --- a/ansible/roles/prechecks/tasks/service_checks.yml +++ b/ansible/roles/prechecks/tasks/service_checks.yml @@ -72,3 +72,10 @@ Please consult the release notes. when: - (enable_zun | default()) | bool + +- name: Validate rabbitmq variables + run_once: True + fail: + msg: | + Please set only one of om_enable_rabbitmq_high_availability or om_enable_rabbitmq_quorum_queues + when: (om_enable_rabbitmq_high_availability | bool) == (om_enable_rabbitmq_quorum_queues | bool) diff --git a/ansible/roles/sahara/templates/sahara.conf.j2 b/ansible/roles/sahara/templates/sahara.conf.j2 index 4a20e24b52..938e9dec53 100644 --- a/ansible/roles/sahara/templates/sahara.conf.j2 +++ b/ansible/roles/sahara/templates/sahara.conf.j2 @@ -49,6 +49,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if sahara_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/senlin/templates/senlin.conf.j2 b/ansible/roles/senlin/templates/senlin.conf.j2 index 0af30c982a..83913ba2fa 100644 --- a/ansible/roles/senlin/templates/senlin.conf.j2 +++ b/ansible/roles/senlin/templates/senlin.conf.j2 @@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if senlin_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/solum/templates/solum.conf.j2 b/ansible/roles/solum/templates/solum.conf.j2 index f5b6edc6c5..d63a74933a 100644 --- a/ansible/roles/solum/templates/solum.conf.j2 +++ b/ansible/roles/solum/templates/solum.conf.j2 @@ -76,3 +76,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} diff --git a/ansible/roles/tacker/templates/tacker.conf.j2 b/ansible/roles/tacker/templates/tacker.conf.j2 index b7613ea2bc..152715d791 100644 --- a/ansible/roles/tacker/templates/tacker.conf.j2 +++ b/ansible/roles/tacker/templates/tacker.conf.j2 @@ -77,6 +77,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if tacker_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/trove/templates/trove.conf.j2 b/ansible/roles/trove/templates/trove.conf.j2 index 4f6db853c0..22d559ca28 100644 --- a/ansible/roles/trove/templates/trove.conf.j2 +++ b/ansible/roles/trove/templates/trove.conf.j2 @@ -83,6 +83,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if enable_osprofiler | bool %} [profiler] diff --git a/ansible/roles/vitrage/templates/vitrage.conf.j2 b/ansible/roles/vitrage/templates/vitrage.conf.j2 index 7326ff11ca..6d7129ea20 100644 --- a/ansible/roles/vitrage/templates/vitrage.conf.j2 +++ b/ansible/roles/vitrage/templates/vitrage.conf.j2 @@ -78,6 +78,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} [oslo_concurrency] lock_path = /var/lib/vitrage/tmp diff --git a/ansible/roles/watcher/templates/watcher.conf.j2 b/ansible/roles/watcher/templates/watcher.conf.j2 index ab0a9daffe..fb436f37ff 100644 --- a/ansible/roles/watcher/templates/watcher.conf.j2 +++ b/ansible/roles/watcher/templates/watcher.conf.j2 @@ -68,6 +68,9 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} {% if watcher_policy_file is defined %} [oslo_policy] diff --git a/ansible/roles/zun/templates/zun.conf.j2 b/ansible/roles/zun/templates/zun.conf.j2 index daf0d1a3e0..214f984b9f 100644 --- a/ansible/roles/zun/templates/zun.conf.j2 +++ b/ansible/roles/zun/templates/zun.conf.j2 @@ -135,3 +135,6 @@ ssl_ca_file = {{ om_rabbitmq_cacert }} {% if om_enable_rabbitmq_high_availability | bool %} amqp_durable_queues = true {% endif %} +{% if om_enable_rabbitmq_quorum_queues | bool %} +rabbit_quorum_queue = true +{% endif %} diff --git a/doc/source/reference/message-queues/rabbitmq.rst b/doc/source/reference/message-queues/rabbitmq.rst index eaa17e2ba8..5652d958f7 100644 --- a/doc/source/reference/message-queues/rabbitmq.rst +++ b/doc/source/reference/message-queues/rabbitmq.rst @@ -113,15 +113,23 @@ https://www.rabbitmq.com/runtime.html#busy-waiting. High Availability ~~~~~~~~~~~~~~~~~ -RabbitMQ offers two features that, when used together, allow for high -availability. These are durable queues and classic queue mirroring. Setting the -flag ``om_enable_rabbitmq_high_availability`` to ``true`` will enable both of -these features. There are some queue types which are intentionally not mirrored +RabbitMQ offers two options to configure HA: + * Quorum queues (enabled by default and controlled by + ``om_enable_rabbitmq_quorum_queues`` variable) + * Classic queue mirroring and durable queues (deprecated in RabbitMQ and to + be dropped in 4.0, controlled by ``om_enable_rabbitmq_high_availability``) + +There are some queue types which are intentionally not mirrored using the exclusionary pattern ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``. After enabling this value on a running system, there are some additional steps needed to migrate from transient to durable queues. +.. warning:: + + Since the default changed from non-HA to Quorum queues in Bobcat release, + following procedure is required to be carried out before an upgrade. + 1. Stop all OpenStack services which use RabbitMQ, so that they will not attempt to recreate any queues yet. diff --git a/releasenotes/notes/make-quorum-queues-usable-552bfb2e8975f83f.yaml b/releasenotes/notes/make-quorum-queues-usable-552bfb2e8975f83f.yaml new file mode 100644 index 0000000000..5ce4089159 --- /dev/null +++ b/releasenotes/notes/make-quorum-queues-usable-552bfb2e8975f83f.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + You can now enable the usage of quorum queues in RabbitMQ for all services + by setting the variable ``om_enable_rabbitmq_quorum_queues`` to ``true``. + Notice that you can't use quorum queues and high availability at the same + time. This is caught by a precheck. + This feature is enabled by default to improve reliability of the messaging + queues. +upgrade: + - | + Quorum queues in RabbitMQ (controlled by + ``om_enable_rabbitmq_quorum_queues`` variable) is enabled by default from + now on. + Support for non-HA RabbitMQ queues is dropped. Either quorum queues that + are enabled by default, or classic mirrored queues are required now. + `Migration procedure from non-HA to HA + `__ diff --git a/tests/upgrade.sh b/tests/upgrade.sh index 45ea07dd54..78b945ce49 100755 --- a/tests/upgrade.sh +++ b/tests/upgrade.sh @@ -13,6 +13,34 @@ function upgrade { source $KOLLA_ANSIBLE_VENV_PATH/bin/activate kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks + + # NOTE(mattcrees): As om_enable_rabbitmq_quorum_queues now defaults to + # true in Bobcat, we need to perform a migration to durable queues. + # TODO(mattcrees): Remove these steps in Caracal. + SERVICE_TAGS="heat,keystone,neutron,nova" + if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then + SERVICE_TAGS+=",cinder" + fi + if [[ $SCENARIO == "scenario_nfv" ]]; then + SERVICE_TAGS+=",barbican" + fi + if [[ $SCENARIO == "ironic" ]]; then + SERVICE_TAGS+=",ironic" + fi + if [[ $SCENARIO == "masakari" ]]; then + SERVICE_TAGS+=",masakari" + fi + if [[ $SCENARIO == "ovn" ]] || [[ $SCENARIO == "octavia" ]]; then + SERVICE_TAGS+=",octavia" + fi + if [[ $SCENARIO == "magnum" ]]; then + SERVICE_TAGS+=",magnum,designate" + fi + kolla-ansible -i ${RAW_INVENTORY} -vvv stop --tags $SERVICE_TAGS --yes-i-really-really-mean-it &> /tmp/logs/ansible/stop + kolla-ansible -i ${RAW_INVENTORY} -vvv genconfig &> /tmp/logs/ansible/genconfig + kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure --tags rabbitmq &> /tmp/logs/ansible/reconfigure-rabbitmq + kolla-ansible -i ${RAW_INVENTORY} -vvv rabbitmq-reset-state &> /tmp/logs/ansible/rabbitmq-reset-state + kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade