Merge "Test upgrades in CI"

This commit is contained in:
Zuul 2018-11-06 11:10:20 +00:00 committed by Gerrit Code Review
commit 60ecee2b25
14 changed files with 370 additions and 45 deletions

View File

@ -104,6 +104,19 @@ function install_venv {
fi
}
function upgrade_venv {
echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}"
virtualenv "${KAYOBE_VENV_PATH}"
# NOTE: Virtualenv's activate and deactivate scripts reference an
# unbound variable.
set +u
source "${KAYOBE_VENV_PATH}/bin/activate"
pip install -U pip
pip install -U "${KAYOBE_SOURCE_PATH}"
deactivate
set -u
}
# Deployment
function is_deploy_image_built_locally {
@ -183,7 +196,18 @@ function overcloud_deploy {
environment_setup
echo "Bootstrapping the Ansible control host"
run_kayobe control host bootstrap
for i in $(seq 1 3); do
if run_kayobe control host bootstrap; then
chb_success=1
break
fi
echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
done
if [[ -z ${chb_success+x} ]]; then
echo "Failed to bootstrap control host"
exit 1
fi
echo "Bootstrapped control host after $i attempts"
echo "Configuring the controller host"
run_kayobe overcloud host configure
@ -216,6 +240,61 @@ function overcloud_deploy {
echo "Control plane deployment complete"
}
function overcloud_upgrade {
# Upgrade a kayobe control plane.
echo "Upgrading a kayobe development environment. This consists of a "
echo "single node OpenStack control plane."
echo "Upgrading Kayobe"
upgrade_venv
environment_setup
echo "Upgrading the Ansible control host"
for i in $(seq 1 3); do
if run_kayobe control host upgrade; then
chu_success=1
break
fi
echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
done
if [[ -z ${chu_success+x} ]]; then
echo "Failed to upgrade control host"
exit 1
fi
echo "Upgraded control host after $i attempts"
echo "Upgrading the controller host"
run_kayobe overcloud host upgrade
if is_deploy_image_built_locally; then
echo "Building overcloud deployment images"
run_kayobe overcloud deployment image build --force-rebuild
else
echo "Not building overcloud deployment images"
fi
if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
echo "Building overcloud container images"
run_kayobe overcloud container image build
else
echo "Pulling overcloud container images"
run_kayobe overcloud container image pull
fi
echo "Saving overcloud service configuration"
if ! run_kayobe overcloud service configuration save; then
# NOTE(mgoddard): This fails in CI due to a memory error while copying
# the IPA deployment images.
echo "FIXME: Saving service configuration failed. Ignoring for now"
fi
echo "Deploying containerised overcloud services"
run_kayobe overcloud service upgrade
echo "Control plane upgrade complete"
}
function overcloud_test {
# Perform a simple smoke test against the cloud.
echo "Performing a simple smoke test"
@ -224,9 +303,13 @@ function overcloud_test {
pip install python-openstackclient
echo "Running kolla-ansible init-runonce"
source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce
if ! openstack image show cirros >/dev/null 2>&1; then
echo "Running kolla-ansible init-runonce"
${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce
else
echo "Not running kolla-ansible init-runonce - resources exist"
fi
echo "Creating a VM"
openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1

20
dev/overcloud-upgrade.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -eu
set -o pipefail
# Simple script to upgrade a development environment for an OpenStack
# controller in a Vagrant VM using kayobe. This should be executed from within
# the VM.
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
config_init
overcloud_upgrade
}
main

View File

@ -93,6 +93,18 @@ plane::
Upon successful completion of this script, the control plane will be active.
The control plane can be tested by running the ``dev/overcloud-test.sh``
script. This will run the ``init-runonce`` setup script provided by Kolla
Ansible that registers images, networks, flavors etc. It will then deploy a
virtual server instance, and delete it once it becomes active::
./dev/overcloud-test.sh
It is possible to test an upgrade by running the ``dev/overcloud-upgrade.sh``
script::
./dev/overcloud-upgrade.sh
Seed Hypervisor
===============

View File

@ -10,28 +10,9 @@
- role: kayobe-diagnostics
kayobe_diagnostics_phase: "pre"
kayobe_diagnostics_log_dir: "{{ logs_dir }}"
- role: kayobe-ci-prep
tasks:
- name: Install dbus for debian system
apt:
name: dbus
when:
- ansible_os_family == 'Debian'
become: true
- block:
# NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
# but the configure-mirrors role used by Zuul disables epel. Since we
# install epel-release and expect epel to be enabled, enable it here.
- name: Ensure yum-utils is installed
yum:
name: yum-utils
state: installed
- name: Enable the EPEL yum repository
command: yum-config-manager --enable epel
when: ansible_os_family == 'RedHat'
become: true
# NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
# the zuul and stack users.
- name: Ensure upper-contraints.txt exists

View File

@ -0,0 +1,10 @@
---
# The Queens kayobe configuration uses the fernet token provider. To avoid
# issues during upgrade, set the provider to fernet in the Pike deployment
# also.
keystone_token_provider: 'fernet'
# Most development environments will use nested virtualisation, and we can't
# guarantee that nested KVM support is available. Use QEMU as a lowest common
# denominator.
nova_compute_virt_type: qemu

View File

@ -0,0 +1,16 @@
---
# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
# Ansible is run directly on the controller.
disable_selinux_do_reboot: false
{% if not previous_release | default(false) %}
kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}"
kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}"
kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}"
kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}"
kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
{% endif %}
# Use the CI infra's PyPI mirror.
pip_local_mirror: true
pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"

View File

@ -0,0 +1,8 @@
---
- hosts: all
roles:
- role: kayobe-diagnostics
kayobe_diagnostics_phase: "post"
kayobe_diagnostics_log_dir: "/tmp/logs"
kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

View File

@ -0,0 +1,69 @@
---
- hosts: primary
vars:
logs_dir: "/tmp/logs"
kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
roles:
- role: kayobe-diagnostics
kayobe_diagnostics_phase: "pre"
kayobe_diagnostics_log_dir: "{{ logs_dir }}"
- role: kayobe-ci-prep
tasks:
# NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
# the zuul and stack users.
- name: Ensure upper-contraints.txt exists
copy:
src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
dest: "/tmp"
mode: 0644
remote_src: true
- name: Ensure previous kayobe directory exists
file:
path: "{{ previous_kayobe_src_dir }}"
state: directory
- name: Ensure previous kayobe repository is cloned
git:
repo: https://git.openstack.org/openstack/kayobe
dest: "{{ previous_kayobe_src_dir }}"
version: "stable/{{ previous_release | lower }}"
- name: Ensure previous kayobe-config directory exists
file:
path: "{{ previous_kayobe_config_src_dir }}"
state: directory
- name: Ensure kayobe-config repository is cloned
git:
repo: https://git.openstack.org/openstack/kayobe-config-dev
dest: "{{ previous_kayobe_config_src_dir }}"
version: "stable/{{ previous_release | lower }}"
# NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
# precedence over the standard config files.
- name: Ensure kayobe-config override config file exists
template:
src: overrides.yml.j2
dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
vars:
previous_release: true
- name: Ensure kayobe-config globals.yml config file exists
template:
src: globals.yml.j2
dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
# NOTE(mgoddard): The kayobe dev config by default expects a bridge -
# breth1 - to exist with an IP address of 192.168.33.3.
- name: Ensure all-in-one network bridge interface exists
command: "{{ item }}"
become: true
with_items:
- "ip l add breth1 type bridge"
- "ip l set breth1 up"
- "ip a add 192.168.33.3/24 dev breth1"

View File

@ -0,0 +1,83 @@
---
- hosts: primary
vars:
kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
logs_dir: "/tmp/logs"
tasks:
# Install the previous release of Kayobe, and use it to deploy a control
# plane.
- block:
- name: Ensure kayobe is installed
shell:
cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
chdir: "{{ previous_kayobe_src_dir }}"
- name: Ensure overcloud is deployed
shell:
cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
chdir: "{{ previous_kayobe_src_dir }}"
environment:
KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
# Update the Kayobe configuration to the current release.
- name: Ensure kolla config directory exists
file:
path: "{{ kayobe_config_src_dir }}/etc/kolla"
state: directory
- name: Copy across relevant kayobe-config files
copy:
src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
dest: "{{ kayobe_config_src_dir }}/{{ item }}"
remote_src: true
with_items:
- etc/kayobe/kolla/passwords.yml
- etc/kolla/admin-openrc.sh
- etc/kolla/public-openrc.sh
# NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
# precedence over the standard config files.
- name: Ensure kayobe-config override config file exists
template:
src: overrides.yml.j2
dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
# Perform a smoke test against the previous release.
- block:
- name: Perform testing of the overcloud prior to upgrade
shell:
cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade
# NOTE(mgoddard): Currently need to use the new kayobe repo for
# testing, since the overcloud-test.sh script is not available in Pike.
chdir: "{{ kayobe_src_dir }}"
# Upgrade Kayobe, and use it to perform an upgrade of the control plane.
- name: Ensure overcloud is upgraded
shell:
cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade
chdir: "{{ kayobe_src_dir }}"
# FIXME(mgoddard): The nova-compute service does not seem to be correctly
# handling the SIGHUP after being upgraded, leading to "In shutdown, no new
# events can be scheduled" errors when booting an instance.
- name: Workaround for SIGHUP issue - restart nova-compute service
shell:
cmd: docker restart nova_compute
become: true
# Perform a smoke test against the upgraded current release.
- name: Perform testing of the upgraded overcloud
shell:
cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade
chdir: "{{ kayobe_src_dir }}"
environment:
KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

View File

@ -10,28 +10,9 @@
- role: kayobe-diagnostics
kayobe_diagnostics_phase: "pre"
kayobe_diagnostics_log_dir: "{{ logs_dir }}"
- role: kayobe-ci-prep
tasks:
- name: Install dbus for debian system
apt:
name: dbus
when:
- ansible_os_family == 'Debian'
become: true
- block:
# NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
# but the configure-mirrors role used by Zuul disables epel. Since we
# install epel-release and expect epel to be enabled, enable it here.
- name: Ensure yum-utils is installed
yum:
name: yum-utils
state: installed
- name: Enable the EPEL yum repository
command: yum-config-manager --enable epel
when: ansible_os_family == 'RedHat'
become: true
# NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
# the zuul and stack users.
- name: Ensure upper-contraints.txt exists

View File

@ -0,0 +1,21 @@
---
- name: Install dbus for debian system
apt:
name: dbus
when:
- ansible_os_family == 'Debian'
become: true
- block:
# NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
# but the configure-mirrors role used by Zuul disables epel. Since we
# install epel-release and expect epel to be enabled, enable it here.
- name: Ensure yum-utils is installed
yum:
name: yum-utils
state: installed
- name: Enable the EPEL yum repository
command: yum-config-manager --enable epel
when: ansible_os_family == 'RedHat'
become: true

View File

@ -34,6 +34,7 @@ copy_logs() {
df -h > ${LOG_DIR}/system_logs/df.txt
free > ${LOG_DIR}/system_logs/free.txt
cat /etc/hosts > ${LOG_DIR}/system_logs/hosts.txt
parted -l > ${LOG_DIR}/system_logs/parted-l.txt
mount > ${LOG_DIR}/system_logs/mount.txt
env > ${LOG_DIR}/system_logs/env.txt

View File

@ -77,6 +77,44 @@
parent: kayobe-overcloud-base
nodeset: kayobe-centos
- job:
name: kayobe-overcloud-upgrade-base
description: |
Base job for testing overcloud upgrades.
Configures the primary VM as an overcloud controller using the previous
OpenStack release, and upgrades it to the current release.
pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml
run: playbooks/kayobe-overcloud-upgrade-base/run.yml
post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml
attempts: 1
timeout: 7200
required-projects:
# Include kayobe to ensure other projects can use this job.
- name: openstack/kayobe
- name: openstack/kayobe-config-dev
- name: openstack/kolla
override-checkout: stable/queens
- name: openstack/kolla-ansible
override-checkout: stable/queens
- name: openstack/requirements
override-checkout: stable/queens
vars:
# Name of the release to upgrade from.
previous_release: pike
irrelevant-files:
- ^.*\.rst$
- ^doc/.*
- ^releasenotes/.*
- ^setup.cfg$
- ^tools/.*$
- ^tox.ini$
- job:
name: kayobe-overcloud-upgrade-centos
parent: kayobe-overcloud-upgrade-base
nodeset: kayobe-centos
- job:
name: kayobe-seed-base
description: |

View File

@ -15,6 +15,7 @@
- kayobe-tox-ansible
- kayobe-tox-molecule
- kayobe-overcloud-centos
- kayobe-overcloud-upgrade-centos
- kayobe-seed-centos
gate:
@ -25,4 +26,5 @@
- kayobe-tox-ansible
- kayobe-tox-molecule
- kayobe-overcloud-centos
- kayobe-overcloud-upgrade-centos
- kayobe-seed-centos