Add a flag to handle RabbitMQ high availability

A combination of durable queues and classic queue mirroring can be used
to provide high availability of RabbitMQ. However, these options should
only be used together, otherwise the system will become unstable. Using
the flag ``om_enable_rabbitmq_high_availability`` will either enable
both options at once, or neither of them.

There are some queues that should not be mirrored:
* ``reply`` queues (these have a single consumer and TTL policy)
* ``fanout`` queues (these have a TTL policy)
* ``amq`` queues (these are auto-delete queues, with a single consumer)
An exclusionary pattern is used in the classic mirroring policy. This
pattern is ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``

Change-Id: I51c8023b260eb40b2eaa91bd276b46890c215c25

ansible/group_vars/all.yml

@@ -234,6 +234,8 @@ om_enable_rabbitmq_tls: "{{ rabbitmq_enable_tls | bool }}"
 # CA certificate bundle in containers using oslo.messaging with RabbitMQ TLS.
 om_rabbitmq_cacert: "{{ rabbitmq_cacert }}"
 
+om_enable_rabbitmq_high_availability: false
+
 ####################
 # Networking options
 ####################

ansible/roles/aodh/templates/aodh.conf.j2

@@ -66,3 +66,6 @@ heartbeat_in_pthread = {{ service_name == 'aodh-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}

ansible/roles/barbican/templates/barbican.conf.j2

@@ -84,6 +84,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/blazar/templates/blazar.conf.j2

@@ -59,6 +59,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if blazar_policy_file is defined %}
 [oslo_policy]

ansible/roles/ceilometer/templates/ceilometer.conf.j2

@@ -41,6 +41,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if ceilometer_policy_file is defined %}
 [oslo_policy]

ansible/roles/cinder/templates/cinder.conf.j2

@@ -71,6 +71,9 @@ heartbeat_in_pthread = {{ service_name == 'cinder-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/cloudkitty/templates/cloudkitty.conf.j2

@@ -50,6 +50,9 @@ heartbeat_in_pthread = {{ service_name == 'cloudkitty-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [collect]
 collector = {{ cloudkitty_collector_backend }}

ansible/roles/cyborg/templates/cyborg.conf.j2

@@ -63,3 +63,6 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}

ansible/roles/designate/templates/designate.conf.j2

@@ -98,6 +98,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_concurrency]
 lock_path = /var/lib/designate/tmp

ansible/roles/glance/templates/glance-api.conf.j2

@@ -126,6 +126,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if glance_policy_file is defined %}
 [oslo_policy]

ansible/roles/heat/templates/heat.conf.j2

@@ -78,6 +78,9 @@ heartbeat_in_pthread = {{ service_name == 'heat-api' or service_name == 'heat-ap
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if heat_policy_file is defined %}
 [oslo_policy]

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -18,6 +18,9 @@ heartbeat_in_pthread = true
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [ironic]
 {% if ironic_enable_keystone_integration | bool %}

ansible/roles/ironic/templates/ironic.conf.j2

@@ -33,6 +33,9 @@ heartbeat_in_pthread = {{ service_name == 'ironic-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if ironic_policy_file is defined %}
 [oslo_policy]

ansible/roles/keystone/templates/keystone.conf.j2

@@ -65,6 +65,9 @@ heartbeat_in_pthread = {{ service_name == 'keystone' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/magnum/templates/magnum.conf.j2

@@ -132,6 +132,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if magnum_policy_file is defined %}
 [oslo_policy]

ansible/roles/manila/templates/manila.conf.j2

@@ -62,6 +62,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/masakari/templates/masakari.conf.j2

@@ -56,6 +56,9 @@ heartbeat_in_pthread = {{ service_name == 'masakari-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/mistral/templates/mistral.conf.j2

@@ -78,6 +78,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if mistral_policy_file is defined %}
 [oslo_policy]

ansible/roles/murano/templates/murano.conf.j2

@@ -67,6 +67,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

ansible/roles/neutron/templates/neutron.conf.j2

@@ -142,6 +142,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if neutron_policy_file is defined %}
 [oslo_policy]

ansible/roles/nova-cell/templates/nova.conf.j2

@@ -190,6 +190,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if service_name in nova_cell_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]

ansible/roles/nova/templates/nova.conf.j2

@@ -144,6 +144,9 @@ heartbeat_in_pthread = {{ service_name == 'nova-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if service_name in nova_services_require_policy_json and nova_policy_file is defined %}
 [oslo_policy]

ansible/roles/octavia/templates/octavia.conf.j2

@@ -128,6 +128,9 @@ heartbeat_in_pthread = {{ service_name == 'octavia-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if octavia_policy_file is defined %}
 [oslo_policy]

ansible/roles/rabbitmq/templates/definitions.json.j2

@@ -16,5 +16,13 @@
     {"user": "{{ murano_agent_rabbitmq_user }}", "vhost": "{{ murano_agent_rabbitmq_vhost }}", "configure": ".*", "write": ".*", "read": ".*"}
     {% endif %}
   ],
+{% if om_enable_rabbitmq_high_availability | bool %}
+  "policies":[
+    {"vhost": "/", "name": "ha-all", "pattern": "^(?!(amq\\.)|(.*_fanout_)|(reply_)).*", "apply-to": "all", "definition": {"ha-mode":"all"}, "priority":0}{% if project_name == 'outward_rabbitmq' %},
+    {"vhost": "{{ murano_agent_rabbitmq_vhost }}", "name": "ha-all", "pattern": "^(?!(amq\\.)|(.*_fanout_)|(reply_)).*", "apply-to": "all", "definition": {"ha-mode":"all"}, "priority":0}
+    {% endif %}
+  ]
+{% else %}
   "policies":[]
+{% endif %}
 }

ansible/roles/sahara/templates/sahara.conf.j2

@@ -46,6 +46,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if sahara_policy_file is defined %}
 [oslo_policy]

ansible/roles/senlin/templates/senlin.conf.j2

@@ -74,6 +74,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if senlin_policy_file is defined %}
 [oslo_policy]

ansible/roles/solum/templates/solum.conf.j2

@@ -73,3 +73,6 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}

ansible/roles/tacker/templates/tacker.conf.j2

@@ -74,6 +74,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if tacker_policy_file is defined %}
 [oslo_policy]

ansible/roles/trove/templates/trove.conf.j2

@@ -77,6 +77,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if enable_osprofiler | bool %}
 [profiler]

ansible/roles/vitrage/templates/vitrage.conf.j2

@@ -75,6 +75,9 @@ heartbeat_in_pthread = {{ service_name == 'vitrage-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 [oslo_concurrency]
 lock_path = /var/lib/vitrage/tmp

ansible/roles/watcher/templates/watcher.conf.j2

@@ -65,6 +65,9 @@ heartbeat_in_pthread = false
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}
 
 {% if watcher_policy_file is defined %}
 [oslo_policy]

ansible/roles/zun/templates/zun.conf.j2

@@ -130,3 +130,6 @@ heartbeat_in_pthread = {{ service_name == 'zun-api' }}
 ssl = true
 ssl_ca_file = {{ om_rabbitmq_cacert }}
 {% endif %}
+{% if om_enable_rabbitmq_high_availability | bool %}
+amqp_durable_queues = true
+{% endif %}

doc/source/reference/message-queues/rabbitmq.rst

@@ -109,3 +109,12 @@ https://erlang.org/doc/man/erl.html#emulator-flags
 The ``+sbwt none +sbwtdcpu none +sbwtdio none`` arguments prevent busy waiting
 of the scheduler, for more details see:
 https://www.rabbitmq.com/runtime.html#busy-waiting.
+
+High Availability
+~~~~~~~~~~~~~~~~~
+
+RabbitMQ offers two features that, when used together, allow for high
+availability. These are durable queues and classic queue mirroring. Setting the
+flag ``om_enable_rabbitmq_high_availability`` to ``true`` will enable both of
+these features. There are some queue types which are intentionally not mirrored
+using the exclusionary pattern ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``.

releasenotes/notes/add-a-flag-to-handle-rabbitmq-high-availability-44c709318be6cb7b.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds the flag ``om_enable_rabbitmq_high_availablity``. Setting this to
+    ``true`` will enable both durable queues and classic mirrored queues in
+    RabbitMQ. Note that classic queue mirroring and transient (aka non-durable)
+    queues are deprecated and subject to removal in RabbitMQ version 4.0 (date
+    of release unknown).
+    Changes the pattern used in classic mirroring to exclude some queue types.
+    This pattern is ``^(?!(amq\\.)|(.*_fanout_)|(reply_)).*``.