From 7593a8b9259a95d26f773be12b8aab0ab1c3b714 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 8 Nov 2018 19:09:02 +0000 Subject: [PATCH] Test upgrading seed services in CI Adds the kayobe-seed-upgrade-centos job, which performs an upgrade of the seed services from the previous release to the current release. Change-Id: Ia3eb39cf81cb3618fd94c4456bd576b52098c946 Story: 2004308 Task: 27873 --- dev/functions | 96 +++++++++++++------ dev/seed-upgrade.sh | 19 ++++ doc/source/development/automated.rst | 5 + .../bifrost-overrides.yml.j2 | 7 ++ .../kayobe-seed-upgrade-base/overrides.yml.j2 | 33 +++++++ playbooks/kayobe-seed-upgrade-base/post.yml | 9 ++ playbooks/kayobe-seed-upgrade-base/pre.yml | 85 ++++++++++++++++ playbooks/kayobe-seed-upgrade-base/run.yml | 64 +++++++++++++ zuul.d/jobs.yaml | 38 ++++++++ zuul.d/project.yaml | 2 + 10 files changed, 329 insertions(+), 29 deletions(-) create mode 100755 dev/seed-upgrade.sh create mode 100644 playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 create mode 100644 playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 create mode 100644 playbooks/kayobe-seed-upgrade-base/post.yml create mode 100644 playbooks/kayobe-seed-upgrade-base/pre.yml create mode 100644 playbooks/kayobe-seed-upgrade-base/run.yml diff --git a/dev/functions b/dev/functions index beec19087..60565bc49 100644 --- a/dev/functions +++ b/dev/functions @@ -153,6 +153,38 @@ function run_kayobe { kayobe ${KAYOBE_EXTRA_ARGS} $* } +function control_host_bootstrap { + echo "Bootstrapping the Ansible control host" + for i in $(seq 1 3); do + if run_kayobe control host bootstrap; then + chb_success=1 + break + fi + echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chb_success+x} ]]; then + die $LINENO "Failed to bootstrap control host" + exit 1 + fi + echo "Bootstrapped control host after $i attempts" +} + +function control_host_upgrade { + echo "Upgrading the Ansible control host" + for i in $(seq 1 3); do + if run_kayobe control host upgrade; then + chu_success=1 + break + fi + echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chu_success+x} ]]; then + die $LINENO "Failed to upgrade control host" + exit 1 + fi + echo "Upgraded control host after $i attempts" +} + function seed_hypervisor_deploy { # Deploy a seed hypervisor. environment_setup @@ -168,8 +200,7 @@ function seed_deploy { # Deploy a kayobe seed in a VM. environment_setup - echo "Bootstrapping the Ansible control host" - run_kayobe control host bootstrap + control_host_bootstrap if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then echo "Provisioning the seed VM" @@ -179,7 +210,7 @@ function seed_deploy { echo "Configuring the seed host" run_kayobe seed host configure - # Note: This must currently be before host configure, because host + # Note: This must currently be done before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. if is_deploy_image_built_locally; then @@ -201,6 +232,37 @@ function seed_deploy { run_kayobe seed service deploy } +function seed_upgrade { + # Upgrade a kayobe seed in a VM. + echo "Upgrading Kayobe" + upgrade_kayobe_venv + + environment_setup + + control_host_upgrade + + echo "Upgrading the seed host" + run_kayobe seed host upgrade + + if is_deploy_image_built_locally; then + echo "Building seed deployment images" + run_kayobe seed deployment image build --force-rebuild + else + echo "Not building seed deployment images" + fi + + if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then + echo "Building seed container images" + run_kayobe seed container image build + else + echo "Not pulling seed container images - no such command yet" + #run_kayobe seed container image pull + fi + + echo "Upgrading containerised seed services" + run_kayobe seed service upgrade +} + function overcloud_deploy { # Deploy a kayobe control plane. echo "Deploying a kayobe development environment. This consists of a " @@ -208,19 +270,7 @@ function overcloud_deploy { environment_setup - echo "Bootstrapping the Ansible control host" - for i in $(seq 1 3); do - if run_kayobe control host bootstrap; then - chb_success=1 - break - fi - echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" - done - if [[ -z ${chb_success+x} ]]; then - echo "Failed to bootstrap control host" - exit 1 - fi - echo "Bootstrapped control host after $i attempts" + control_host_bootstrap echo "Configuring the controller host" run_kayobe overcloud host configure @@ -269,19 +319,7 @@ function overcloud_upgrade { environment_setup - echo "Upgrading the Ansible control host" - for i in $(seq 1 3); do - if run_kayobe control host upgrade; then - chu_success=1 - break - fi - echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" - done - if [[ -z ${chu_success+x} ]]; then - echo "Failed to upgrade control host" - exit 1 - fi - echo "Upgraded control host after $i attempts" + control_host_upgrade echo "Upgrading the controller host" run_kayobe overcloud host upgrade diff --git a/dev/seed-upgrade.sh b/dev/seed-upgrade.sh new file mode 100755 index 000000000..7f51cf5a8 --- /dev/null +++ b/dev/seed-upgrade.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to upgrade a development environment for a seed VM using +# kayobe. This should be executed from the hypervisor. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + seed_upgrade +} + +main diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst index 59e6bf9dd..aa9b4e01c 100644 --- a/doc/source/development/automated.rst +++ b/doc/source/development/automated.rst @@ -193,3 +193,8 @@ Upon successful completion of this script, the seed VM will be active. The seed VM may be accessed via SSH as the ``stack`` user:: ssh stack@192.168.33.5 + +It is possible to test an upgrade by running the ``dev/seed-upgrade.sh`` +script:: + + ./dev/seed-upgrade.sh diff --git a/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 new file mode 100644 index 000000000..0ada56a2f --- /dev/null +++ b/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 @@ -0,0 +1,7 @@ +--- +# Don't build an IPA deployment image, instead download upstream images. +create_ipa_image: false +download_ipa: true + +# Don't build a disk image. It takes time and can be unreliable. +use_cirros: true diff --git a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 new file mode 100644 index 000000000..a9dd6ea76 --- /dev/null +++ b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 @@ -0,0 +1,33 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +{% if not is_previous_release | default(false) %} +kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}" +kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}" +kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}" +kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" +kolla_upper_constraints_file: "/tmp/upper-constraints.txt" +{% endif %} + +# NOTE(mgoddard): We're using a cirros image, which doesn't require the +# resolv.conf work around used for CentOS. +overcloud_host_image_workaround_resolv_enabled: false + +# Use the CI infra's PyPI mirror. +pip_local_mirror: true +pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" + +{% if previous_release == "pike" %} +# kayobe-config-dev in queens changed to use overlay by default. Specify +# devicemapper explicitly to avoid changing. +docker_storage_driver: devicemapper +docker_storage_volume_thinpool_size: 45%VG + +# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since +# the overlay driver doesn't work with the ansible template module until +# ansible 2.4.0, and this is required by bifrost. +seed_lvm_group_data_disks: + - /dev/loop0 +{% endif %} diff --git a/playbooks/kayobe-seed-upgrade-base/post.yml b/playbooks/kayobe-seed-upgrade-base/post.yml new file mode 100644 index 000000000..be286660b --- /dev/null +++ b/playbooks/kayobe-seed-upgrade-base/post.yml @@ -0,0 +1,9 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + kayobe_diagnostics_previous_config_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-seed-upgrade-base/pre.yml b/playbooks/kayobe-seed-upgrade-base/pre.yml new file mode 100644 index 000000000..dc472ab4b --- /dev/null +++ b/playbooks/kayobe-seed-upgrade-base/pre.yml @@ -0,0 +1,85 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep + tasks: + # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both + # the zuul and stack users. + - name: Ensure upper-contraints.txt exists + copy: + src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}" + dest: "/tmp" + mode: 0644 + remote_src: true + + - name: Ensure previous kayobe directory exists + file: + path: "{{ previous_kayobe_src_dir }}" + state: directory + + - name: Ensure previous kayobe repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe + dest: "{{ previous_kayobe_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + - name: Ensure previous kayobe-config directory exists + file: + path: "{{ previous_kayobe_config_src_dir }}" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe-config-dev + dest: "{{ previous_kayobe_config_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + vars: + is_previous_release: true + + - name: Ensure bifrost overrides directory exists + file: + path: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost" + state: "directory" + + - name: Ensure bifrost overrides file exists + template: + src: bifrost-overrides.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml" + + - block: + # NOTE(mgoddard): Create a loopback device backed by a file for docker + # storage. We do this since the overlay driver doesn't work with the + # ansible template module until ansible 2.4.0, and this is required by + # bifrost. + - name: Ensure a docker storage backing file exists + command: truncate -s 20G /tmp/docker-storage + + - name: Ensure the docker storage loopback device is created + command: losetup /dev/loop0 /tmp/docker-storage + become: true + when: previous_release == "pike" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist on the seed with an IP address of 192.168.33.5. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.5/24 dev breth1" diff --git a/playbooks/kayobe-seed-upgrade-base/run.yml b/playbooks/kayobe-seed-upgrade-base/run.yml new file mode 100644 index 000000000..d1d960f32 --- /dev/null +++ b/playbooks/kayobe-seed-upgrade-base/run.yml @@ -0,0 +1,64 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" + kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + logs_dir: "/tmp/logs" + tasks: + + # Install the previous release of Kayobe, and use it to deploy a control + # plane. + + - block: + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + + - name: Ensure seed is deployed + shell: + cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}" + # Don't provision a seed VM - use the Zuul VM as the seed host. + KAYOBE_SEED_VM_PROVISION: 0 + + # Update the Kayobe configuration to the current release. + + - name: Ensure bifrost overrides directory exists + file: + path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost" + state: "directory" + + - name: Copy across relevant kayobe-config files + copy: + src: "{{ previous_kayobe_config_src_dir }}/{{ item }}" + dest: "{{ kayobe_config_src_dir }}/{{ item }}" + remote_src: true + with_items: + - etc/kayobe/kolla/passwords.yml + - etc/kayobe/kolla/config/bifrost/bifrost.yml + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + # TODO(mgoddard): Perform a smoke test against the previous release. + + - block: + # Upgrade Kayobe, and use it to perform an upgrade of the seed. + + - name: Ensure seed is upgraded + shell: + cmd: dev/seed-upgrade.sh > {{ logs_dir }}/ansible/seed-upgrade + chdir: "{{ kayobe_src_dir }}" + + # TODO(mgoddard): Perform a smoke test against the upgraded current release. + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index e4689dcc8..23fb52e11 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -150,3 +150,41 @@ name: kayobe-seed-centos parent: kayobe-seed-base nodeset: kayobe-centos + +- job: + name: kayobe-seed-upgrade-base + description: | + Base job for testing seed upgrades. + + Configures the primary VM as a seed using the previous OpenStack release, + and upgrades it to the current release. + pre-run: playbooks/kayobe-seed-upgrade-base/pre.yml + run: playbooks/kayobe-seed-upgrade-base/run.yml + post-run: playbooks/kayobe-seed-upgrade-base/post.yml + attempts: 1 + timeout: 5400 + required-projects: + # Include kayobe to ensure other projects can use this job. + - name: openstack/kayobe + - name: openstack/kayobe-config-dev + - name: openstack/kolla + override-checkout: stable/queens + - name: openstack/kolla-ansible + override-checkout: stable/queens + - name: openstack/requirements + override-checkout: stable/queens + vars: + # Name of the release to upgrade from. + previous_release: pike + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + - ^releasenotes/.* + - ^setup.cfg$ + - ^tools/.*$ + - ^tox.ini$ + +- job: + name: kayobe-seed-upgrade-centos + parent: kayobe-seed-upgrade-base + nodeset: kayobe-centos diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index b9a69e8cd..76394b653 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -17,6 +17,7 @@ - kayobe-overcloud-centos - kayobe-overcloud-upgrade-centos - kayobe-seed-centos + - kayobe-seed-upgrade-centos gate: queue: kayobe @@ -28,3 +29,4 @@ - kayobe-overcloud-centos - kayobe-overcloud-upgrade-centos - kayobe-seed-centos + - kayobe-seed-upgrade-centos