From c79c5d1f36d5c786c96b9935265376faf297754d Mon Sep 17 00:00:00 2001 From: Duong Ha-Quang Date: Wed, 4 Apr 2018 16:52:51 +0700 Subject: [PATCH] Apply Nova rolling upgrade logic This patchset apply Nova rolling upgrade logic [1] [1] https://docs.openstack.org/nova/latest/user/upgrade.html Implements: blueprint apply-service-upgrade-procedure Co-Authored-By: Ha Manh Dong Depends-On: https://review.openstack.org/#/c/558765/ Change-Id: I20531a25dc7f5b05dc70ba771cf4c4222ade33e3 --- ansible/roles/nova/defaults/main.yml | 2 + ansible/roles/nova/handlers/main.yml | 156 +++++++++--------- .../roles/nova/tasks/bootstrap_service.yml | 10 +- ansible/roles/nova/tasks/legacy_upgrade.yml | 30 ++++ ansible/roles/nova/tasks/rolling_upgrade.yml | 50 ++++++ ansible/roles/nova/tasks/upgrade.yml | 42 ++--- etc/kolla/globals.yml | 15 ++ ...nova-rolling-upgrade-f3b2d8382f725cb2.yaml | 4 + 8 files changed, 202 insertions(+), 107 deletions(-) create mode 100644 ansible/roles/nova/tasks/legacy_upgrade.yml create mode 100644 ansible/roles/nova/tasks/rolling_upgrade.yml create mode 100644 releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml index 9b64ff812a..f57e37fad2 100644 --- a/ansible/roles/nova/defaults/main.yml +++ b/ansible/roles/nova/defaults/main.yml @@ -290,6 +290,8 @@ openstack_nova_auth: "{{ openstack_auth }}" openstack_placement_auth: "{{ openstack_auth }}" nova_compute_host_rp_filter_mode: 0 +nova_enable_rolling_upgrade: "yes" +nova_safety_upgrade: "no" nova_libvirt_port: "16509" nova_ssh_port: "8022" diff --git a/ansible/roles/nova/handlers/main.yml b/ansible/roles/nova/handlers/main.yml index 1bf939f87d..e34c6ad37f 100644 --- a/ansible/roles/nova/handlers/main.yml +++ b/ansible/roles/nova/handlers/main.yml @@ -1,4 +1,56 @@ --- +- name: Restart placement-api container + vars: + service_name: "placement-api" + service: "{{ nova_services[service_name] }}" + config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" + placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or nova_conf.changed | bool + or policy_overwriting.changed | bool + or placement_api_wsgi_conf | changed + or placement_api_container.changed | bool + +- name: Restart nova-conductor container + vars: + service_name: "nova-conductor" + service: "{{ nova_services[service_name] }}" + config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" + nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + privileged: "{{ service.privileged | default(False) }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or nova_conf.changed | bool + or policy_overwriting.changed | bool + or nova_conductor_container.changed | bool + - name: Restart nova-ssh container vars: service_name: "nova-ssh" @@ -52,58 +104,6 @@ or nova_libvirt_confs.changed | bool or nova_libvirt_container.changed | bool -- name: Restart placement-api container - vars: - service_name: "placement-api" - service: "{{ nova_services[service_name] }}" - config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" - nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" - policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" - placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes|reject('equalto', '')|list }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - inventory_hostname in groups[service.group] - - service.enabled | bool - - config_json.changed | bool - or nova_conf.changed | bool - or policy_overwriting.changed | bool - or placement_api_wsgi_conf | changed - or placement_api_container.changed | bool - -- name: Restart nova-api container - vars: - service_name: "nova-api" - service: "{{ nova_services[service_name] }}" - config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" - nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" - policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" - nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - privileged: "{{ service.privileged | default(False) }}" - volumes: "{{ service.volumes|reject('equalto', '')|list }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - inventory_hostname in groups[service.group] - - service.enabled | bool - - config_json.changed | bool - or nova_conf.changed | bool - or policy_overwriting.changed | bool - or nova_api_container.changed | bool - - name: Restart nova-scheduler container vars: service_name: "nova-scheduler" @@ -130,32 +130,6 @@ or policy_overwriting.changed | bool or nova_scheduler_container.changed | bool -- name: Restart nova-conductor container - vars: - service_name: "nova-conductor" - service: "{{ nova_services[service_name] }}" - config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" - nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" - policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" - nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - privileged: "{{ service.privileged | default(False) }}" - volumes: "{{ service.volumes|reject('equalto', '')|list }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - inventory_hostname in groups[service.group] - - service.enabled | bool - - config_json.changed | bool - or nova_conf.changed | bool - or policy_overwriting.changed | bool - or nova_conductor_container.changed | bool - - name: Restart nova-consoleauth container vars: service_name: "nova-consoleauth" @@ -260,6 +234,32 @@ or policy_overwriting.changed | bool or nova_serialproxy_container.changed | bool +- name: Restart nova-api container + vars: + service_name: "nova-api" + service: "{{ nova_services[service_name] }}" + config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}" + nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + privileged: "{{ service.privileged | default(False) }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or nova_conf.changed | bool + or policy_overwriting.changed | bool + or nova_api_container.changed | bool + - name: Restart nova-compute container vars: service_name: "nova-compute" diff --git a/ansible/roles/nova/tasks/bootstrap_service.yml b/ansible/roles/nova/tasks/bootstrap_service.yml index 4728975e85..eed9d2cbea 100644 --- a/ansible/roles/nova/tasks/bootstrap_service.yml +++ b/ansible/roles/nova/tasks/bootstrap_service.yml @@ -2,14 +2,18 @@ - name: Running Nova bootstrap container vars: nova_api: "{{ nova_services['nova-api'] }}" + bootstrap_environment: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + upgrade_environment: + KOLLA_UPGRADE: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" become: true kolla_docker: action: "start_container" common_options: "{{ docker_common_options }}" detach: False - environment: - KOLLA_BOOTSTRAP: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + environment: "{{ upgrade_environment if nova_enable_rolling_upgrade|bool else bootstrap_environment }}" image: "{{ nova_api.image }}" labels: BOOTSTRAP: diff --git a/ansible/roles/nova/tasks/legacy_upgrade.yml b/ansible/roles/nova/tasks/legacy_upgrade.yml new file mode 100644 index 0000000000..bd931d5282 --- /dev/null +++ b/ansible/roles/nova/tasks/legacy_upgrade.yml @@ -0,0 +1,30 @@ +--- +- include_tasks: config.yml + +- include_tasks: bootstrap_service.yml + +- name: Checking if conductor container needs upgrading + become: true + kolla_docker: + action: "compare_image" + common_options: "{{ docker_common_options }}" + name: "nova_conductor" + image: "{{ nova_conductor_image_full }}" + when: inventory_hostname in groups['nova-conductor'] + register: conductor_differs + +# Short downtime here, but from user perspective his call will just timeout or execute later +- name: Stopping all nova_conductor containers + become: true + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "nova_conductor" + when: + - inventory_hostname in groups['nova-conductor'] + - conductor_differs['result'] + +- name: Flush handlers + meta: flush_handlers + +- include_tasks: reload.yml diff --git a/ansible/roles/nova/tasks/rolling_upgrade.yml b/ansible/roles/nova/tasks/rolling_upgrade.yml new file mode 100644 index 0000000000..1d016009b2 --- /dev/null +++ b/ansible/roles/nova/tasks/rolling_upgrade.yml @@ -0,0 +1,50 @@ +--- +# Create new set of configs on nodes +- include_tasks: pull.yml + +- include_tasks: config.yml + +- include_tasks: bootstrap_service.yml + +- name: Stopping all nova services except nova-compute + become: true + kolla_docker: + action: "stop_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + with_dict: "{{ nova_services }}" + when: + - "'nova-compute' not in item.key" + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + - nova_safety_upgrade | bool + +# TODO(donghm): Flush_handlers to restart nova services +# should be run in serial nodes to decrease downtime if +# the previous task did not run. Update when the +# Ansible strategy module for rolling upgrade is finished. + +- name: Flush handlers + meta: flush_handlers + +- include_tasks: reload.yml + +- name: Migrate Nova database + vars: + nova_api: "{{ nova_services['nova-api'] }}" + become: true + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + environment: + KOLLA_OSM: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + image: "{{ nova_api.image }}" + labels: + BOOTSTRAP: + name: "bootstrap_nova" + restart_policy: "never" + volumes: "{{ nova_api.volumes }}" + run_once: True + delegate_to: "{{ groups[nova_api.group][0] }}" diff --git a/ansible/roles/nova/tasks/upgrade.yml b/ansible/roles/nova/tasks/upgrade.yml index 58dc7a9e42..bc04f78868 100644 --- a/ansible/roles/nova/tasks/upgrade.yml +++ b/ansible/roles/nova/tasks/upgrade.yml @@ -1,31 +1,21 @@ --- -# Create new set of configs on nodes -- include_tasks: config.yml - -- include_tasks: bootstrap_service.yml - -- name: Checking if conductor container needs upgrading +- name: Check nova upgrade status become: true - kolla_docker: - action: "compare_image" - common_options: "{{ docker_common_options }}" - name: "nova_conductor" - image: "{{ nova_conductor_image_full }}" - when: inventory_hostname in groups['nova-conductor'] - register: conductor_differs + command: docker exec -t nova_api nova-status upgrade check + register: nova_upgrade_check_stdout + when: inventory_hostname == groups['nova-api'][0] -# Short downtime here, but from user perspective his call will just timeout or execute later -- name: Stopping all nova_conductor containers - become: true - kolla_docker: - action: "stop_container" - common_options: "{{ docker_common_options }}" - name: "nova_conductor" - when: - - inventory_hostname in groups['nova-conductor'] - - conductor_differs['result'] +- name: Upgrade status check result + fail: + msg: + - "There was an upgrade status check warning or failure!" + - "See the detail at https://docs.openstack.org/nova/latest/cli/nova-status.html#nova-status-checks" + vars: + first_nova_api_host: "{{ groups['nova-api'][0] }}" + when: hostvars[first_nova_api_host]['nova_upgrade_check_stdout']['rc'] != 0 -- name: Flush handlers - meta: flush_handlers +- include_tasks: legacy_upgrade.yml + when: not nova_enable_rolling_upgrade | bool -- include_tasks: reload.yml +- include_tasks: rolling_upgrade.yml + when: nova_enable_rolling_upgrade | bool diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 6b77d4210d..a39c6fe184 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -401,6 +401,21 @@ glance_enable_rolling_upgrade: "no" # The number of fake driver per compute node #num_nova_fake_per_node: 5 +# Configure nova upgrade option, due to currently kolla support +# two upgrade ways for nova: legacy_upgrade and rolling_upgrade +# The variable "nova_enable_rolling_upgrade: yes" is meaning +# rolling_upgrade were enabled and opposite +#nova_enable_rolling_upgrade: "yes" + +# The flag "nova_safety_upgrade" need to be consider when +# "nova_enable_rolling_upgrade" is enabled. The "nova_safety_upgrade" +# controls whether the nova services are all stopped before rolling +# upgrade to the new version, for the safety and availability. +# If "nova_safety_upgrade" is "yes", that will stop all nova services (except +# nova-compute) for no failed API operations before upgrade to the +# new version. And opposite. +#nova_safety_upgrade: "no" + ################# # Hyper-V options ################# diff --git a/releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml b/releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml new file mode 100644 index 0000000000..c84b91412a --- /dev/null +++ b/releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Implement Nova rolling upgrade logic