Merge "Test upgrades in CI"

dev/functions

@@ -104,6 +104,19 @@ function install_venv {
     fi
 }
 
+function upgrade_venv {
+    echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}"
+    virtualenv "${KAYOBE_VENV_PATH}"
+    # NOTE: Virtualenv's activate and deactivate scripts reference an
+    # unbound variable.
+    set +u
+    source "${KAYOBE_VENV_PATH}/bin/activate"
+    pip install -U pip
+    pip install -U "${KAYOBE_SOURCE_PATH}"
+    deactivate
+    set -u
+}
+
 # Deployment
 
 function is_deploy_image_built_locally {
@@ -183,7 +196,18 @@ function overcloud_deploy {
     environment_setup
 
     echo "Bootstrapping the Ansible control host"
-    run_kayobe control host bootstrap
+    for i in $(seq 1 3); do
+        if run_kayobe control host bootstrap; then
+            chb_success=1
+            break
+        fi
+        echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chb_success+x} ]]; then
+        echo "Failed to bootstrap control host"
+        exit 1
+    fi
+    echo "Bootstrapped control host after $i attempts"
 
     echo "Configuring the controller host"
     run_kayobe overcloud host configure
@@ -216,6 +240,61 @@ function overcloud_deploy {
     echo "Control plane deployment complete"
 }
 
+function overcloud_upgrade {
+    # Upgrade a kayobe control plane.
+    echo "Upgrading a kayobe development environment. This consists of a "
+    echo "single node OpenStack control plane."
+
+    echo "Upgrading Kayobe"
+    upgrade_venv
+
+    environment_setup
+
+    echo "Upgrading the Ansible control host"
+    for i in $(seq 1 3); do
+        if run_kayobe control host upgrade; then
+            chu_success=1
+            break
+        fi
+        echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chu_success+x} ]]; then
+        echo "Failed to upgrade control host"
+        exit 1
+    fi
+    echo "Upgraded control host after $i attempts"
+
+    echo "Upgrading the controller host"
+    run_kayobe overcloud host upgrade
+
+    if is_deploy_image_built_locally; then
+        echo "Building overcloud deployment images"
+        run_kayobe overcloud deployment image build --force-rebuild
+    else
+        echo "Not building overcloud deployment images"
+    fi
+
+    if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+        echo "Building overcloud container images"
+        run_kayobe overcloud container image build
+    else
+        echo "Pulling overcloud container images"
+        run_kayobe overcloud container image pull
+    fi
+
+    echo "Saving overcloud service configuration"
+    if ! run_kayobe overcloud service configuration save; then
+        # NOTE(mgoddard): This fails in CI due to a memory error while copying
+        # the IPA deployment images.
+        echo "FIXME: Saving service configuration failed. Ignoring for now"
+    fi
+
+    echo "Deploying containerised overcloud services"
+    run_kayobe overcloud service upgrade
+
+    echo "Control plane upgrade complete"
+}
+
 function overcloud_test {
     # Perform a simple smoke test against the cloud.
     echo "Performing a simple smoke test"
@@ -224,9 +303,13 @@ function overcloud_test {
 
     pip install python-openstackclient
 
-    echo "Running kolla-ansible init-runonce"
     source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
+    if ! openstack image show cirros >/dev/null 2>&1; then
+        echo "Running kolla-ansible init-runonce"
         ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce
+    else
+        echo "Not running kolla-ansible init-runonce - resources exist"
+    fi
 
     echo "Creating a VM"
     openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1

dev/overcloud-upgrade.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to upgrade a development environment for an OpenStack
+# controller in a Vagrant VM using kayobe.  This should be executed from within
+# the VM.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    overcloud_upgrade
+}
+
+main

doc/source/development/automated.rst

@@ -93,6 +93,18 @@ plane::
 
 Upon successful completion of this script, the control plane will be active.
 
+The control plane can be tested by running the ``dev/overcloud-test.sh``
+script. This will run the ``init-runonce`` setup script provided by Kolla
+Ansible that registers images, networks, flavors etc. It will then deploy a
+virtual server instance, and delete it once it becomes active::
+
+    ./dev/overcloud-test.sh
+
+It is possible to test an upgrade by running the ``dev/overcloud-upgrade.sh``
+script::
+
+    ./dev/overcloud-upgrade.sh
+
 Seed Hypervisor
 ===============
 

playbooks/kayobe-overcloud-base/pre.yml

@@ -10,28 +10,9 @@
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
       kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
   tasks:
-    - name: Install dbus for debian system
-      apt:
-        name: dbus
-      when:
-        - ansible_os_family == 'Debian'
-      become: true
-
-    - block:
-        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
-        # but the configure-mirrors role used by Zuul disables epel. Since we
-        # install epel-release and expect epel to be enabled, enable it here.
-        - name: Ensure yum-utils is installed
-          yum:
-            name: yum-utils
-            state: installed
-
-        - name: Enable the EPEL yum repository
-          command: yum-config-manager --enable epel
-      when: ansible_os_family == 'RedHat'
-      become: true
-
     # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
     # the zuul and stack users.
     - name: Ensure upper-contraints.txt exists

playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2

@@ -0,0 +1,10 @@
+---
+# The Queens kayobe configuration uses the fernet token provider. To avoid
+# issues during upgrade, set the provider to fernet in the Pike deployment
+# also.
+keystone_token_provider: 'fernet'
+
+# Most development environments will use nested virtualisation, and we can't
+# guarantee that nested KVM support is available. Use QEMU as a lowest common
+# denominator.
+nova_compute_virt_type: qemu

playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2

@@ -0,0 +1,16 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+{% if not previous_release | default(false) %}
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}"
+kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+{% endif %}
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"

playbooks/kayobe-overcloud-upgrade-base/post.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "post"
+      kayobe_diagnostics_log_dir: "/tmp/logs"
+      kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-overcloud-upgrade-base/pre.yml

@@ -0,0 +1,69 @@
+---
+- hosts: primary
+  vars:
+    logs_dir: "/tmp/logs"
+    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "pre"
+      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
+  tasks:
+    # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
+    # the zuul and stack users.
+    - name: Ensure upper-contraints.txt exists
+      copy:
+        src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+        dest: "/tmp"
+        mode: 0644
+        remote_src: true
+
+    - name: Ensure previous kayobe directory exists
+      file:
+        path: "{{ previous_kayobe_src_dir }}"
+        state: directory
+
+    - name: Ensure previous kayobe repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe
+        dest: "{{ previous_kayobe_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    - name: Ensure previous kayobe-config directory exists
+      file:
+        path: "{{ previous_kayobe_config_src_dir }}"
+        state: directory
+
+    - name: Ensure kayobe-config repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe-config-dev
+        dest: "{{ previous_kayobe_config_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+      vars:
+        previous_release: true
+
+    - name: Ensure kayobe-config globals.yml config file exists
+      template:
+        src: globals.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.3.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
+        - "ip a add 192.168.33.3/24 dev breth1"

playbooks/kayobe-overcloud-upgrade-base/run.yml

@@ -0,0 +1,83 @@
+---
+- hosts: primary
+  vars:
+    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+    logs_dir: "/tmp/logs"
+  tasks:
+
+    # Install the previous release of Kayobe, and use it to deploy a control
+    # plane.
+
+    - block:
+        - name: Ensure kayobe is installed
+          shell:
+            cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+
+        - name: Ensure overcloud is deployed
+          shell:
+            cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
+
+    # Update the Kayobe configuration to the current release.
+
+    - name: Ensure kolla config directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}/etc/kolla"
+        state: directory
+
+    - name: Copy across relevant kayobe-config files
+      copy:
+        src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
+        dest: "{{ kayobe_config_src_dir }}/{{ item }}"
+        remote_src: true
+      with_items:
+        - etc/kayobe/kolla/passwords.yml
+        - etc/kolla/admin-openrc.sh
+        - etc/kolla/public-openrc.sh
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    # Perform a smoke test against the previous release.
+
+    - block:
+        - name: Perform testing of the overcloud prior to upgrade
+          shell:
+            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade
+            # NOTE(mgoddard): Currently need to use the new kayobe repo for
+            # testing, since the overcloud-test.sh script is not available in Pike.
+            chdir: "{{ kayobe_src_dir }}"
+
+        # Upgrade Kayobe, and use it to perform an upgrade of the control plane.
+
+        - name: Ensure overcloud is upgraded
+          shell:
+            cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade
+            chdir: "{{ kayobe_src_dir }}"
+
+        # FIXME(mgoddard): The nova-compute service does not seem to be correctly
+        # handling the SIGHUP after being upgraded, leading to "In shutdown, no new
+        # events can be scheduled" errors when booting an instance.
+        - name: Workaround for SIGHUP issue - restart nova-compute service
+          shell:
+            cmd: docker restart nova_compute
+          become: true
+
+        # Perform a smoke test against the upgraded current release.
+
+        - name: Perform testing of the upgraded overcloud
+          shell:
+            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade
+            chdir: "{{ kayobe_src_dir }}"
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

playbooks/kayobe-seed-base/pre.yml

@@ -10,28 +10,9 @@
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
       kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
   tasks:
-    - name: Install dbus for debian system
-      apt:
-        name: dbus
-      when:
-        - ansible_os_family == 'Debian'
-      become: true
-
-    - block:
-        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
-        # but the configure-mirrors role used by Zuul disables epel. Since we
-        # install epel-release and expect epel to be enabled, enable it here.
-        - name: Ensure yum-utils is installed
-          yum:
-            name: yum-utils
-            state: installed
-
-        - name: Enable the EPEL yum repository
-          command: yum-config-manager --enable epel
-      when: ansible_os_family == 'RedHat'
-      become: true
-
     # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
     # the zuul and stack users.
     - name: Ensure upper-contraints.txt exists

roles/kayobe-ci-prep/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Install dbus for debian system
+  apt:
+    name: dbus
+  when:
+    - ansible_os_family == 'Debian'
+  become: true
+
+- block:
+    # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+    # but the configure-mirrors role used by Zuul disables epel. Since we
+    # install epel-release and expect epel to be enabled, enable it here.
+    - name: Ensure yum-utils is installed
+      yum:
+        name: yum-utils
+        state: installed
+
+    - name: Enable the EPEL yum repository
+      command: yum-config-manager --enable epel
+  when: ansible_os_family == 'RedHat'
+  become: true

roles/kayobe-diagnostics/files/get_logs.sh

@@ -34,6 +34,7 @@ copy_logs() {
 
     df -h > ${LOG_DIR}/system_logs/df.txt
     free  > ${LOG_DIR}/system_logs/free.txt
+    cat /etc/hosts  > ${LOG_DIR}/system_logs/hosts.txt
     parted -l > ${LOG_DIR}/system_logs/parted-l.txt
     mount > ${LOG_DIR}/system_logs/mount.txt
     env > ${LOG_DIR}/system_logs/env.txt

zuul.d/jobs.yaml

@@ -77,6 +77,44 @@
     parent: kayobe-overcloud-base
     nodeset: kayobe-centos
 
+- job:
+    name: kayobe-overcloud-upgrade-base
+    description: |
+      Base job for testing overcloud upgrades.
+
+      Configures the primary VM as an overcloud controller using the previous
+      OpenStack release, and upgrades it to the current release.
+    pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml
+    run: playbooks/kayobe-overcloud-upgrade-base/run.yml
+    post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml
+    attempts: 1
+    timeout: 7200
+    required-projects:
+      # Include kayobe to ensure other projects can use this job.
+      - name: openstack/kayobe
+      - name: openstack/kayobe-config-dev
+      - name: openstack/kolla
+        override-checkout: stable/queens
+      - name: openstack/kolla-ansible
+        override-checkout: stable/queens
+      - name: openstack/requirements
+        override-checkout: stable/queens
+    vars:
+      # Name of the release to upgrade from.
+      previous_release: pike
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+
+- job:
+    name: kayobe-overcloud-upgrade-centos
+    parent: kayobe-overcloud-upgrade-base
+    nodeset: kayobe-centos
+
 - job:
     name: kayobe-seed-base
     description: |

zuul.d/project.yaml

@@ -15,6 +15,7 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
+        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
 
     gate:
@@ -25,4 +26,5 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
+        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos