Apply Nova rolling upgrade logic

This patchset apply Nova rolling upgrade logic [1]

[1] https://docs.openstack.org/nova/latest/user/upgrade.html

Implements: blueprint apply-service-upgrade-procedure

Co-Authored-By: Ha Manh Dong <donghm@vn.fujitsu.com>
Depends-On: https://review.openstack.org/#/c/558765/
Change-Id: I20531a25dc7f5b05dc70ba771cf4c4222ade33e3

ansible/roles/nova/defaults/main.yml

@@ -290,6 +290,8 @@ openstack_nova_auth: "{{ openstack_auth }}"
 openstack_placement_auth: "{{ openstack_auth }}"
 
 nova_compute_host_rp_filter_mode: 0
+nova_enable_rolling_upgrade: "yes"
+nova_safety_upgrade: "no"
 
 nova_libvirt_port: "16509"
 nova_ssh_port: "8022"

ansible/roles/nova/handlers/main.yml

@@ -1,4 +1,56 @@
 ---
+- name: Restart placement-api container
+  vars:
+    service_name: "placement-api"
+    service: "{{ nova_services[service_name] }}"
+    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or nova_conf.changed | bool
+      or policy_overwriting.changed | bool
+      or placement_api_wsgi_conf | changed
+      or placement_api_container.changed | bool
+
+- name: Restart nova-conductor container
+  vars:
+    service_name: "nova-conductor"
+    service: "{{ nova_services[service_name] }}"
+    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    privileged: "{{ service.privileged | default(False) }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or nova_conf.changed | bool
+      or policy_overwriting.changed | bool
+      or nova_conductor_container.changed | bool
+
 - name: Restart nova-ssh container
   vars:
     service_name: "nova-ssh"
@@ -52,58 +104,6 @@
       or nova_libvirt_confs.changed | bool
       or nova_libvirt_container.changed | bool
 
-- name: Restart placement-api container
-  vars:
-    service_name: "placement-api"
-    service: "{{ nova_services[service_name] }}"
-    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
-    - config_json.changed | bool
-      or nova_conf.changed | bool
-      or policy_overwriting.changed | bool
-      or placement_api_wsgi_conf | changed
-      or placement_api_container.changed | bool
-
-- name: Restart nova-api container
-  vars:
-    service_name: "nova-api"
-    service: "{{ nova_services[service_name] }}"
-    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    privileged: "{{ service.privileged | default(False) }}"
-    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
-    - config_json.changed | bool
-      or nova_conf.changed | bool
-      or policy_overwriting.changed | bool
-      or nova_api_container.changed | bool
-
 - name: Restart nova-scheduler container
   vars:
     service_name: "nova-scheduler"
@@ -130,32 +130,6 @@
       or policy_overwriting.changed | bool
       or nova_scheduler_container.changed | bool
 
-- name: Restart nova-conductor container
-  vars:
-    service_name: "nova-conductor"
-    service: "{{ nova_services[service_name] }}"
-    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    privileged: "{{ service.privileged | default(False) }}"
-    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
-    - config_json.changed | bool
-      or nova_conf.changed | bool
-      or policy_overwriting.changed | bool
-      or nova_conductor_container.changed | bool
-
 - name: Restart nova-consoleauth container
   vars:
     service_name: "nova-consoleauth"
@@ -260,6 +234,32 @@
       or policy_overwriting.changed | bool
       or nova_serialproxy_container.changed | bool
 
+- name: Restart nova-api container
+  vars:
+    service_name: "nova-api"
+    service: "{{ nova_services[service_name] }}"
+    config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    privileged: "{{ service.privileged | default(False) }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or nova_conf.changed | bool
+      or policy_overwriting.changed | bool
+      or nova_api_container.changed | bool
+
 - name: Restart nova-compute container
   vars:
     service_name: "nova-compute"

ansible/roles/nova/tasks/bootstrap_service.yml

@@ -2,14 +2,18 @@
 - name: Running Nova bootstrap container
   vars:
     nova_api: "{{ nova_services['nova-api'] }}"
+    bootstrap_environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    upgrade_environment:
+      KOLLA_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
   become: true
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     detach: False
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    environment: "{{ upgrade_environment if nova_enable_rolling_upgrade|bool else bootstrap_environment }}"
     image: "{{ nova_api.image }}"
     labels:
       BOOTSTRAP:

ansible/roles/nova/tasks/legacy_upgrade.yml

@@ -0,0 +1,30 @@
+---
+- include_tasks: config.yml
+
+- include_tasks: bootstrap_service.yml
+
+- name: Checking if conductor container needs upgrading
+  become: true
+  kolla_docker:
+    action: "compare_image"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+    image: "{{ nova_conductor_image_full }}"
+  when: inventory_hostname in groups['nova-conductor']
+  register: conductor_differs
+
+# Short downtime here, but from user perspective his call will just timeout or execute later
+- name: Stopping all nova_conductor containers
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+  when:
+    - inventory_hostname in groups['nova-conductor']
+    - conductor_differs['result']
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- include_tasks: reload.yml

ansible/roles/nova/tasks/rolling_upgrade.yml

@@ -0,0 +1,50 @@
+---
+# Create new set of configs on nodes
+- include_tasks: pull.yml
+
+- include_tasks: config.yml
+
+- include_tasks: bootstrap_service.yml
+
+- name: Stopping all nova services except nova-compute
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+  with_dict: "{{ nova_services }}"
+  when:
+    - "'nova-compute' not in item.key"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+    - nova_safety_upgrade | bool
+
+# TODO(donghm): Flush_handlers to restart nova services
+# should be run in serial nodes to decrease downtime if
+# the previous task did not run. Update when the
+# Ansible strategy module for rolling upgrade is finished.
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- include_tasks: reload.yml
+
+- name: Migrate Nova database
+  vars:
+    nova_api: "{{ nova_services['nova-api'] }}"
+  become: true
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_OSM:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ nova_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_nova"
+    restart_policy: "never"
+    volumes: "{{ nova_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[nova_api.group][0] }}"

ansible/roles/nova/tasks/upgrade.yml

@@ -1,31 +1,21 @@
 ---
-# Create new set of configs on nodes
-- include_tasks: config.yml
-
-- include_tasks: bootstrap_service.yml
-
-- name: Checking if conductor container needs upgrading
+- name: Check nova upgrade status
   become: true
-  kolla_docker:
-    action: "compare_image"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_conductor"
-    image: "{{ nova_conductor_image_full }}"
-  when: inventory_hostname in groups['nova-conductor']
-  register: conductor_differs
+  command: docker exec -t nova_api nova-status upgrade check
+  register: nova_upgrade_check_stdout
+  when: inventory_hostname == groups['nova-api'][0]
 
-# Short downtime here, but from user perspective his call will just timeout or execute later
-- name: Stopping all nova_conductor containers
-  become: true
-  kolla_docker:
-    action: "stop_container"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_conductor"
-  when:
-    - inventory_hostname in groups['nova-conductor']
-    - conductor_differs['result']
+- name: Upgrade status check result
+  fail:
+    msg:
+      - "There was an upgrade status check warning or failure!"
+      - "See the detail at https://docs.openstack.org/nova/latest/cli/nova-status.html#nova-status-checks"
+  vars:
+    first_nova_api_host: "{{ groups['nova-api'][0] }}"
+  when: hostvars[first_nova_api_host]['nova_upgrade_check_stdout']['rc'] != 0
 
-- name: Flush handlers
-  meta: flush_handlers
+- include_tasks: legacy_upgrade.yml
+  when: not nova_enable_rolling_upgrade | bool
 
-- include_tasks: reload.yml
+- include_tasks: rolling_upgrade.yml
+  when: nova_enable_rolling_upgrade | bool

etc/kolla/globals.yml

@@ -401,6 +401,21 @@ glance_enable_rolling_upgrade: "no"
 # The number of fake driver per compute node
 #num_nova_fake_per_node: 5
 
+# Configure nova upgrade option, due to currently kolla support
+# two upgrade ways for nova: legacy_upgrade and rolling_upgrade
+# The variable "nova_enable_rolling_upgrade: yes" is meaning
+# rolling_upgrade were enabled and opposite
+#nova_enable_rolling_upgrade: "yes"
+
+# The flag "nova_safety_upgrade" need to be consider when
+# "nova_enable_rolling_upgrade" is enabled. The "nova_safety_upgrade"
+# controls whether the nova services are all stopped before rolling
+# upgrade to the new version, for the safety and availability.
+# If "nova_safety_upgrade" is "yes", that will stop all nova services (except
+# nova-compute) for no failed API operations before upgrade to the
+# new version. And opposite.
+#nova_safety_upgrade: "no"
+
 #################
 # Hyper-V options
 #################

releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Implement Nova rolling upgrade logic