octavia: enable jobboard

Enable the jobboard feature for the Octavia amphora provider. This
requires Redis as a dependency, a precheck is added to ensure proper
configuration.

https://docs.openstack.org/octavia/latest/install/install-amphorav2.html

Change-Id: Iec3c8a4b4e257557dc8ec995c41d0ad7e88e13e2

ansible/group_vars/all.yml

@@ -736,6 +736,7 @@ enable_nova_serialconsole_proxy: "no"
 enable_nova_ssh: "yes"
 enable_octavia: "no"
 enable_octavia_driver_agent: "{{ enable_octavia | bool and neutron_plugin_agent == 'ovn' }}"
+enable_octavia_jobboard: "{{ enable_octavia | bool and 'amphora' in octavia_provider_drivers }}"
 enable_openvswitch: "{{ enable_neutron | bool and neutron_plugin_agent != 'linuxbridge' }}"
 enable_ovn: "{{ enable_neutron | bool and neutron_plugin_agent == 'ovn' }}"
 enable_ovs_dpdk: "no"

ansible/roles/octavia/defaults/main.yml

@@ -76,6 +76,10 @@ octavia_database_name: "octavia"
 octavia_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}octavia{% endif %}"
 octavia_database_address: "{{ database_address | put_address_in_context('url') }}:{{ database_port }}"
 
+octavia_persistence_database_name: "octavia_persistence"
+octavia_persistence_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}octavia_persistence{% endif %}"
+octavia_persistence_database_address: "{{ octavia_database_address }}"
+
 ####################
 # Database sharding
 ####################
@@ -89,6 +93,16 @@ octavia_database_shard:
     - schema: "{{ octavia_database_name }}"
       shard_id: "{{ octavia_database_shard_id }}"
 
+octavia_persistence_database_shard_root_user: "{{ octavia_database_shard_root_user }}"
+octavia_persistence_database_shard_id: "{{ octavia_database_shard_id }}"
+octavia_persistence_database_shard:
+  users:
+    - user: "{{ octavia_persistence_database_user }}"
+      password: "{{ octavia_persistence_database_password }}"
+  rules:
+    - schema: "{{ octavia_persistence_database_name }}"
+      shard_id: "{{ octavia_persistence_database_shard_id }}"
+
 
 ####################
 # Docker

ansible/roles/octavia/tasks/bootstrap.yml

@@ -15,6 +15,22 @@
   when:
     - not use_preconfigured_databases | bool
 
+- name: Creating Octavia persistence database
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: mysql_db
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ octavia_persistence_database_shard_root_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ octavia_persistence_database_name }}"
+  run_once: True
+  delegate_to: "{{ groups['octavia-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
 - name: Creating Octavia database user and setting permissions
   become: true
   kolla_toolbox:
@@ -35,4 +51,24 @@
   when:
     - not use_preconfigured_databases | bool
 
+- name: Creating Octavia persistence database user and setting permissions
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: mysql_user
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ octavia_persistence_database_shard_root_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ octavia_persistence_database_user }}"
+      password: "{{ octavia_persistence_database_password }}"
+      host: "%"
+      priv: "{{ octavia_persistence_database_name }}.*:ALL"
+      append_privs: "yes"
+  run_once: True
+  delegate_to: "{{ groups['octavia-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
 - import_tasks: bootstrap_service.yml

ansible/roles/octavia/tasks/precheck.yml

@@ -63,3 +63,11 @@
     - octavia_auto_configure | bool
     - octavia_network_type == "tenant"
     - neutron_plugin_agent != 'openvswitch'
+
+- name: Checking whether Redis is enabled for octavia jobboard
+  assert:
+    that: enable_redis | bool
+    fail_msg: "Redis must be enabled when using octavia jobboard"
+  run_once: True
+  when:
+    - enable_octavia_jobboard | bool

ansible/roles/octavia/templates/octavia.conf.j2

@@ -151,3 +151,12 @@ ca_certificates_file = {{ openstack_cacert }}
 region_name = {{ openstack_region_name }}
 endpoint_type = internal
 ca_certificates_file = {{ openstack_cacert }}
+{% if enable_octavia_jobboard %}
+
+[task_flow]
+persistence_connection = mysql+pymysql://{{ octavia_persistence_database_user }}:{{ octavia_persistence_database_password }}@{{ octavia_persistence_database_address }}/{{ octavia_persistence_database_name }}
+jobboard_enabled = true
+jobboard_backend_password = "{{ redis_master_password }}"
+jobboard_backend_port = "{{ redis_port }}"
+jobboard_backend_hosts = {% for host in groups['redis'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}{% if not loop.last %},{% endif %}{% endfor %}
+{% endif %}

etc/kolla/globals.yml

@@ -393,6 +393,7 @@ workaround_ansible_issue_8743: yes
 #enable_nova_ssh: "yes"
 #enable_octavia: "no"
 #enable_octavia_driver_agent: "{{ enable_octavia | bool and neutron_plugin_agent == 'ovn' }}"
+#enable_octavia_jobboard: "{{ enable_octavia | bool and 'amphora' in octavia_provider_drivers }}"
 #enable_opensearch: "{{ enable_central_logging | bool or enable_osprofiler | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') }}"
 #enable_opensearch_dashboards: "{{ enable_opensearch | bool }}"
 #enable_opensearch_dashboards_external: "{{ enable_opensearch_dashboards | bool }}"

etc/kolla/passwords.yml

@@ -153,6 +153,7 @@ manila_database_password:
 manila_keystone_password:
 
 octavia_database_password:
+octavia_persistence_database_password:
 octavia_keystone_password:
 octavia_ca_password:
 octavia_client_ca_password:

releasenotes/notes/octavia_jobboard-823f44393f3e109e.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    The Octavia amphora provider driver improves control plane resiliency.
+    Should a control plane host go down during a load balancer provisioning
+    operation, an alternate controller can resume the in-process provisioning
+    and complete the request. This solves the issue with resources stuck in
+    PENDING_* states by writing info about task states in persistent backend
+    and monitoring job claims via jobboard. The jobboard feature is now
+    enabled by default. It requires the Redis service to be enabled as a
+    dependency. Use the ``enable_octavia_jobboard`` variable to override
+    if needed.
+upgrade:
+  - |
+    The Octavia amphora provider by default is now deployed with the jobboard
+    feature enabled.  This requires the Redis service to be enabled as a
+    dependency, please update your configuration accordingly if needed.
+    For futher information see
+    `Amphorav2 docs <https://docs.openstack.org/octavia/latest/install/install-amphorav2.html>`_

tests/templates/globals-default.j2

@@ -169,6 +169,7 @@ neutron_enable_ovn_agent: "yes"
 enable_octavia: "yes"
 octavia_provider_drivers: "ovn:OVN provider"
 octavia_provider_agents: "ovn"
+enable_redis: "yes"
 {% endif %}
 
 {% if scenario == "prometheus-opensearch" %}
@@ -195,6 +196,7 @@ octavia_amp_flavor:
   ram: 1024
   disk: 5
 octavia_network_type: "tenant"
+enable_redis: "yes"
 {% endif %}
 
 {% if scenario == "venus" %}