From e34fbb174c7042f61fc7742363ee7e81569aaba5 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 20 Jan 2023 13:32:11 +0000 Subject: [PATCH] Use the upgraded image to run Nova upgrade checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When upgrading Nova, we sometimes hit an error where an old hypervisor that hasn’t been upgraded recently (for example due to broken hardware) is preventing Nova API from starting properly. This can be detected using the tool ``nova-status upgrade check`` to make sure that there are no ``nova-compute`` that are older than N-1 releases. This is already used in the Kolla Ansible upgrade task for Nova. However, this task uses the current ``nova-api`` container, so computes which will be too old after the upgrade are not caught. This patch changes Kolla Ansible so that the upgraded ``nova-api`` image is used to run the upgrade checks, allowing computes that will be too old to be detected before the upgrades are performed. Depends-On: https://review.opendev.org/c/openstack/kolla/+/878744 Closes-Bug: #1957080 Co-Authored-By: Pierre Riteau Change-Id: I3a899411001834a0c88e37f45a756247ee11563d --- ansible/roles/nova/tasks/upgrade.yml | 21 ++++++++++++++++--- ...-with-upgraded-image-dca9c515bcd89ec8.yaml | 10 +++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml diff --git a/ansible/roles/nova/tasks/upgrade.yml b/ansible/roles/nova/tasks/upgrade.yml index d6f3f70c42..07f91379af 100644 --- a/ansible/roles/nova/tasks/upgrade.yml +++ b/ansible/roles/nova/tasks/upgrade.yml @@ -1,9 +1,24 @@ --- -- name: Check nova upgrade status +- name: Run Nova upgrade checks become: true - command: "{{ kolla_container_engine }} exec -t nova_api nova-status upgrade check" + vars: + nova_api: "{{ nova_services['nova-api'] }}" + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + environment: + KOLLA_UPGRADE_CHECK: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + image: "{{ nova_api.image }}" + labels: + UPGRADE: + name: "nova_upgrade_checks" + restart_policy: no + volumes: "{{ nova_api_default_volumes + nova_api_extra_volumes }}" + run_once: True register: nova_upgrade_check_stdout - when: inventory_hostname == groups['nova-api'][0] + delegate_to: "{{ groups['nova-api'][0] }}" failed_when: false - name: Upgrade status check result diff --git a/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml b/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml new file mode 100644 index 0000000000..746bd9988d --- /dev/null +++ b/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + When upgrading Nova to a new release, we use the tool ``nova-status upgrade + check`` to make sure that there are no ``nova-compute`` that are older than + N-1 releases. This was performed using the current ``nova-api`` container, + so computes which will be too old after the upgrade were not caught. Now + the upgraded ``nova-api`` container image is used, so older computes are + identified correctly. + `LP#1957080 `__