diff --git a/dev/functions b/dev/functions index 18ec53a11..dee3a375a 100644 --- a/dev/functions +++ b/dev/functions @@ -104,6 +104,19 @@ function install_venv { fi } +function upgrade_venv { + echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}" + virtualenv "${KAYOBE_VENV_PATH}" + # NOTE: Virtualenv's activate and deactivate scripts reference an + # unbound variable. + set +u + source "${KAYOBE_VENV_PATH}/bin/activate" + pip install -U pip + pip install -U "${KAYOBE_SOURCE_PATH}" + deactivate + set -u +} + # Deployment function is_deploy_image_built_locally { @@ -183,7 +196,18 @@ function overcloud_deploy { environment_setup echo "Bootstrapping the Ansible control host" - run_kayobe control host bootstrap + for i in $(seq 1 3); do + if run_kayobe control host bootstrap; then + chb_success=1 + break + fi + echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chb_success+x} ]]; then + echo "Failed to bootstrap control host" + exit 1 + fi + echo "Bootstrapped control host after $i attempts" echo "Configuring the controller host" run_kayobe overcloud host configure @@ -216,6 +240,61 @@ function overcloud_deploy { echo "Control plane deployment complete" } +function overcloud_upgrade { + # Upgrade a kayobe control plane. + echo "Upgrading a kayobe development environment. This consists of a " + echo "single node OpenStack control plane." + + echo "Upgrading Kayobe" + upgrade_venv + + environment_setup + + echo "Upgrading the Ansible control host" + for i in $(seq 1 3); do + if run_kayobe control host upgrade; then + chu_success=1 + break + fi + echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chu_success+x} ]]; then + echo "Failed to upgrade control host" + exit 1 + fi + echo "Upgraded control host after $i attempts" + + echo "Upgrading the controller host" + run_kayobe overcloud host upgrade + + if is_deploy_image_built_locally; then + echo "Building overcloud deployment images" + run_kayobe overcloud deployment image build --force-rebuild + else + echo "Not building overcloud deployment images" + fi + + if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then + echo "Building overcloud container images" + run_kayobe overcloud container image build + else + echo "Pulling overcloud container images" + run_kayobe overcloud container image pull + fi + + echo "Saving overcloud service configuration" + if ! run_kayobe overcloud service configuration save; then + # NOTE(mgoddard): This fails in CI due to a memory error while copying + # the IPA deployment images. + echo "FIXME: Saving service configuration failed. Ignoring for now" + fi + + echo "Deploying containerised overcloud services" + run_kayobe overcloud service upgrade + + echo "Control plane upgrade complete" +} + function overcloud_test { # Perform a simple smoke test against the cloud. echo "Performing a simple smoke test" @@ -224,9 +303,13 @@ function overcloud_test { pip install python-openstackclient - echo "Running kolla-ansible init-runonce" source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" - ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce + if ! openstack image show cirros >/dev/null 2>&1; then + echo "Running kolla-ansible init-runonce" + ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce + else + echo "Not running kolla-ansible init-runonce - resources exist" + fi echo "Creating a VM" openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1 diff --git a/dev/overcloud-upgrade.sh b/dev/overcloud-upgrade.sh new file mode 100755 index 000000000..cbb15855b --- /dev/null +++ b/dev/overcloud-upgrade.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to upgrade a development environment for an OpenStack +# controller in a Vagrant VM using kayobe. This should be executed from within +# the VM. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + overcloud_upgrade +} + +main diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst index b3bbf6243..59e6bf9dd 100644 --- a/doc/source/development/automated.rst +++ b/doc/source/development/automated.rst @@ -93,6 +93,18 @@ plane:: Upon successful completion of this script, the control plane will be active. +The control plane can be tested by running the ``dev/overcloud-test.sh`` +script. This will run the ``init-runonce`` setup script provided by Kolla +Ansible that registers images, networks, flavors etc. It will then deploy a +virtual server instance, and delete it once it becomes active:: + + ./dev/overcloud-test.sh + +It is possible to test an upgrade by running the ``dev/overcloud-upgrade.sh`` +script:: + + ./dev/overcloud-upgrade.sh + Seed Hypervisor =============== diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml index bed29c078..8e2811f62 100644 --- a/playbooks/kayobe-overcloud-base/pre.yml +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -10,28 +10,9 @@ - role: kayobe-diagnostics kayobe_diagnostics_phase: "pre" kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep tasks: - - name: Install dbus for debian system - apt: - name: dbus - when: - - ansible_os_family == 'Debian' - become: true - - - block: - # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, - # but the configure-mirrors role used by Zuul disables epel. Since we - # install epel-release and expect epel to be enabled, enable it here. - - name: Ensure yum-utils is installed - yum: - name: yum-utils - state: installed - - - name: Enable the EPEL yum repository - command: yum-config-manager --enable epel - when: ansible_os_family == 'RedHat' - become: true - # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both # the zuul and stack users. - name: Ensure upper-contraints.txt exists diff --git a/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 new file mode 100644 index 000000000..f7201a37a --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 @@ -0,0 +1,10 @@ +--- +# The Queens kayobe configuration uses the fernet token provider. To avoid +# issues during upgrade, set the provider to fernet in the Pike deployment +# also. +keystone_token_provider: 'fernet' + +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu \ No newline at end of file diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 new file mode 100644 index 000000000..4f91d4edd --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 @@ -0,0 +1,16 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +{% if not previous_release | default(false) %} +kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}" +kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}" +kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}" +kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" +kolla_upper_constraints_file: "/tmp/upper-constraints.txt" +{% endif %} + +# Use the CI infra's PyPI mirror. +pip_local_mirror: true +pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-overcloud-upgrade-base/post.yml b/playbooks/kayobe-overcloud-upgrade-base/post.yml new file mode 100644 index 000000000..31dccb283 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/post.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-overcloud-upgrade-base/pre.yml b/playbooks/kayobe-overcloud-upgrade-base/pre.yml new file mode 100644 index 000000000..7e78ea0f7 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/pre.yml @@ -0,0 +1,69 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" + kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep + tasks: + # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both + # the zuul and stack users. + - name: Ensure upper-contraints.txt exists + copy: + src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}" + dest: "/tmp" + mode: 0644 + remote_src: true + + - name: Ensure previous kayobe directory exists + file: + path: "{{ previous_kayobe_src_dir }}" + state: directory + + - name: Ensure previous kayobe repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe + dest: "{{ previous_kayobe_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + - name: Ensure previous kayobe-config directory exists + file: + path: "{{ previous_kayobe_config_src_dir }}" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe-config-dev + dest: "{{ previous_kayobe_config_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + vars: + previous_release: true + + - name: Ensure kayobe-config globals.yml config file exists + template: + src: globals.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist with an IP address of 192.168.33.3. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.3/24 dev breth1" diff --git a/playbooks/kayobe-overcloud-upgrade-base/run.yml b/playbooks/kayobe-overcloud-upgrade-base/run.yml new file mode 100644 index 000000000..8a3413f55 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/run.yml @@ -0,0 +1,83 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" + kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + logs_dir: "/tmp/logs" + tasks: + + # Install the previous release of Kayobe, and use it to deploy a control + # plane. + + - block: + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + + - name: Ensure overcloud is deployed + shell: + cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}" + + # Update the Kayobe configuration to the current release. + + - name: Ensure kolla config directory exists + file: + path: "{{ kayobe_config_src_dir }}/etc/kolla" + state: directory + + - name: Copy across relevant kayobe-config files + copy: + src: "{{ previous_kayobe_config_src_dir }}/{{ item }}" + dest: "{{ kayobe_config_src_dir }}/{{ item }}" + remote_src: true + with_items: + - etc/kayobe/kolla/passwords.yml + - etc/kolla/admin-openrc.sh + - etc/kolla/public-openrc.sh + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + # Perform a smoke test against the previous release. + + - block: + - name: Perform testing of the overcloud prior to upgrade + shell: + cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade + # NOTE(mgoddard): Currently need to use the new kayobe repo for + # testing, since the overcloud-test.sh script is not available in Pike. + chdir: "{{ kayobe_src_dir }}" + + # Upgrade Kayobe, and use it to perform an upgrade of the control plane. + + - name: Ensure overcloud is upgraded + shell: + cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade + chdir: "{{ kayobe_src_dir }}" + + # FIXME(mgoddard): The nova-compute service does not seem to be correctly + # handling the SIGHUP after being upgraded, leading to "In shutdown, no new + # events can be scheduled" errors when booting an instance. + - name: Workaround for SIGHUP issue - restart nova-compute service + shell: + cmd: docker restart nova_compute + become: true + + # Perform a smoke test against the upgraded current release. + + - name: Perform testing of the upgraded overcloud + shell: + cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade + chdir: "{{ kayobe_src_dir }}" + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" diff --git a/playbooks/kayobe-seed-base/pre.yml b/playbooks/kayobe-seed-base/pre.yml index e7a6d3dd8..3fd9bbc1f 100644 --- a/playbooks/kayobe-seed-base/pre.yml +++ b/playbooks/kayobe-seed-base/pre.yml @@ -10,28 +10,9 @@ - role: kayobe-diagnostics kayobe_diagnostics_phase: "pre" kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep tasks: - - name: Install dbus for debian system - apt: - name: dbus - when: - - ansible_os_family == 'Debian' - become: true - - - block: - # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, - # but the configure-mirrors role used by Zuul disables epel. Since we - # install epel-release and expect epel to be enabled, enable it here. - - name: Ensure yum-utils is installed - yum: - name: yum-utils - state: installed - - - name: Enable the EPEL yum repository - command: yum-config-manager --enable epel - when: ansible_os_family == 'RedHat' - become: true - # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both # the zuul and stack users. - name: Ensure upper-contraints.txt exists diff --git a/roles/kayobe-ci-prep/tasks/main.yml b/roles/kayobe-ci-prep/tasks/main.yml new file mode 100644 index 000000000..cfe5c4c2a --- /dev/null +++ b/roles/kayobe-ci-prep/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Install dbus for debian system + apt: + name: dbus + when: + - ansible_os_family == 'Debian' + become: true + +- block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh index a3f797f37..f3999ab17 100644 --- a/roles/kayobe-diagnostics/files/get_logs.sh +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -34,6 +34,7 @@ copy_logs() { df -h > ${LOG_DIR}/system_logs/df.txt free > ${LOG_DIR}/system_logs/free.txt + cat /etc/hosts > ${LOG_DIR}/system_logs/hosts.txt parted -l > ${LOG_DIR}/system_logs/parted-l.txt mount > ${LOG_DIR}/system_logs/mount.txt env > ${LOG_DIR}/system_logs/env.txt diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 081a6a8de..30670518c 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -77,6 +77,44 @@ parent: kayobe-overcloud-base nodeset: kayobe-centos +- job: + name: kayobe-overcloud-upgrade-base + description: | + Base job for testing overcloud upgrades. + + Configures the primary VM as an overcloud controller using the previous + OpenStack release, and upgrades it to the current release. + pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml + run: playbooks/kayobe-overcloud-upgrade-base/run.yml + post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml + attempts: 1 + timeout: 7200 + required-projects: + # Include kayobe to ensure other projects can use this job. + - name: openstack/kayobe + - name: openstack/kayobe-config-dev + - name: openstack/kolla + override-checkout: stable/queens + - name: openstack/kolla-ansible + override-checkout: stable/queens + - name: openstack/requirements + override-checkout: stable/queens + vars: + # Name of the release to upgrade from. + previous_release: pike + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + - ^releasenotes/.* + - ^setup.cfg$ + - ^tools/.*$ + - ^tox.ini$ + +- job: + name: kayobe-overcloud-upgrade-centos + parent: kayobe-overcloud-upgrade-base + nodeset: kayobe-centos + - job: name: kayobe-seed-base description: | diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 59a06e9b6..b9a69e8cd 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -15,6 +15,7 @@ - kayobe-tox-ansible - kayobe-tox-molecule - kayobe-overcloud-centos + - kayobe-overcloud-upgrade-centos - kayobe-seed-centos gate: @@ -25,4 +26,5 @@ - kayobe-tox-ansible - kayobe-tox-molecule - kayobe-overcloud-centos + - kayobe-overcloud-upgrade-centos - kayobe-seed-centos