From c23c9b2c25d4c58aca1db1b0c210662298e6b50d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 8 Nov 2018 17:18:53 +0000 Subject: [PATCH] Test upgrades in CI This patch adds two new jobs: * kolla-ansible-centos-source-upgrade * kolla-ansible-ubuntu-source-upgrade These jobs first deploy a control plane using the previous release of Kolla Ansible, then upgrade to the current release. Because we can't change the branch of the git repository on the Zuul executor, we change the branch of the kolla-ansible repository on the primary node to the branch of the previous release, in this case stable/rocky. A new remote-template role has been added that supports generating templates using a remote template source, to generate config files using the previous kolla-ansible branch. If the change being tested depends on a kolla change for the current branch, then we build images. Rather than using the current kolla-ansible version to tag the images, we now tag them with change_. This is because the version of kolla-ansible will change from the previous release to the current one as we upgrade the system. Finally, it should be noted that the 'previous_release' variable in the Zuul config needs to be updated with each release, since this sets the release of kolla-ansible that is installed initially. Depends-On: https://review.openstack.org/645089/ Depends-On: https://review.openstack.org/644250/ Depends-On: https://review.openstack.org/645816/ Depends-On: https://review.openstack.org/645840/ Change-Id: If301e0affcd55360fefe3b105f023ae5c47b0853 --- tests/roles/remote-template/defaults/main.yml | 12 + tests/roles/remote-template/tasks/main.yml | 22 ++ tests/run.yml | 269 +++++++++++------- tests/templates/globals-default.j2 | 15 +- tools/setup_gate.sh | 19 +- zuul.d/base.yaml | 7 + zuul.d/jobs.yaml | 16 ++ zuul.d/project.yaml | 6 + 8 files changed, 246 insertions(+), 120 deletions(-) create mode 100644 tests/roles/remote-template/defaults/main.yml create mode 100644 tests/roles/remote-template/tasks/main.yml diff --git a/tests/roles/remote-template/defaults/main.yml b/tests/roles/remote-template/defaults/main.yml new file mode 100644 index 0000000000..cb638bcfbf --- /dev/null +++ b/tests/roles/remote-template/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# Path to template file. +remote_template_src: + +# Path to destination. +remote_template_dest: + +# Whether to use 'become'. +remote_template_become: false + +# Path on localhost to store a copy of the template. +remote_template_temp_path: "{{ zuul.executor.work_root }}/{{ remote_template_src | basename }}" diff --git a/tests/roles/remote-template/tasks/main.yml b/tests/roles/remote-template/tasks/main.yml new file mode 100644 index 0000000000..7f0590c46c --- /dev/null +++ b/tests/roles/remote-template/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Ensure the parent directory exists. +- name: "ensure {{ remote_template_dest | dirname }} exists" + file: + path: "{{ remote_template_dest | dirname }}" + state: "directory" + mode: 0777 + become: "{{ remote_template_become }}" + +# Template sources must be on localhost, so first fetch the remote template +# file. +- name: "fetch remote template {{ remote_template_src | basename }}" + fetch: + src: "{{ remote_template_src }}" + dest: "{{ remote_template_temp_path }}" + flat: true + +- name: "template {{ remote_template_src | basename }}" + template: + src: "{{ remote_template_temp_path }}" + dest: "{{ remote_template_dest }}" + become: "{{ remote_template_become }}" diff --git a/tests/run.yml b/tests/run.yml index f5d7c4dcb9..bec3915f50 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -1,11 +1,10 @@ --- - hosts: all vars: - kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" - kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}" + kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" tasks: - name: Prepare disks for Ceph or LVM - script: "{{ kolla_ansible_full_src_dir }}/tests/setup_disks.sh {{ disk_type }}" + script: "setup_disks.sh {{ disk_type }}" when: scenario in ["ceph", "cinder-lvm"] become: true vars: @@ -16,23 +15,10 @@ vars: kolla_inventory_path: "/etc/kolla/inventory" logs_dir: "/tmp/logs" - kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" - kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}" + kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" + kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" need_build_image: false tasks: - - name: ensure /etc/kolla exists - file: - path: "/etc/kolla" - state: "directory" - mode: 0777 - become: true - - - name: copy default ansible kolla-ansible inventory - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/inventory.j2" - dest: "{{ kolla_inventory_path }}" - delegate_to: "primary" - # FIXME: in multi node env, api_interface may be different on each node. - name: detect api_interface_name variable vars: @@ -45,77 +31,78 @@ - hostvars[inventory_hostname][ansible_interface_name]['ipv4'] is defined - hostvars[inventory_hostname][ansible_interface_name]['ipv4']['address'] == api_interface_address with_items: "{{ ansible_interfaces }}" - delegate_to: "primary" - name: detect whether need build images set_fact: need_build_image: true when: - item.project.short_name == "kolla" + - item.branch == zuul.branch with_items: "{{ zuul['items'] }}" - - name: generate global.yml file - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/globals-default.j2" - dest: /etc/kolla/globals.yml - delegate_to: "primary" + # NOTE(mgoddard): This only affects the remote copy of the repo, not the + # one on the executor. + - name: checkout the previous kolla-ansible branch + command: + cmd: "git checkout stable/{{ previous_release | lower }}" + chdir: "{{ kolla_ansible_src_dir }}" + when: scenario == "upgrade" - - name: ensure nova conf overrides dir exists + - name: ensure /etc/kolla exists file: - path: "/etc/kolla/config/nova" - state: "directory" - mode: 0777 - when: scenario != "bifrost" - become: true - delegate_to: "primary" - - - name: generate nova config overrides - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2" - dest: /etc/kolla/config/nova/nova-compute.conf - when: scenario != "bifrost" - delegate_to: "primary" - - - name: ensure bifrost conf overrides dir exists - file: - path: "/etc/kolla/config/bifrost" - state: "directory" - mode: 0777 - when: scenario == "bifrost" - become: true - delegate_to: "primary" - - - name: generate bifrost DIB config overrides - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2" - dest: /etc/kolla/config/bifrost/dib.yml - when: scenario == "bifrost" - delegate_to: "primary" - - - name: ensure /etc/docker exists - file: - path: "/etc/docker" + path: "/etc/kolla" state: "directory" mode: 0777 become: true - - name: create deamon.json for nodepool cache + # Use the initial repo to generate config files. For upgrade jobs, this + # repo is only available on the remote node, so use the remote-template + # role. + - name: generate configuration files + include_role: + role: remote-template vars: + is_previous_release: "{{ scenario == 'upgrade' }}" infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/" - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/docker_daemon.json.j2" - dest: "/etc/docker/daemon.json" - become: true + # Role variables. + remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}" + remote_template_dest: "{{ item.dest }}" + remote_template_become: "{{ item.become | default(false) }}" + with_items: + # Docker daemon.json + - src: "tests/templates/docker_daemon.json.j2" + dest: "/etc/docker/daemon.json" + become: true + # Ansible inventory + - src: "tests/templates/inventory.j2" + dest: "{{ kolla_inventory_path }}" + # globals.yml + - src: "tests/templates/globals-default.j2" + dest: /etc/kolla/globals.yml + # nova-compute.conf + - src: "tests/templates/nova-compute-overrides.j2" + dest: /etc/kolla/config/nova/nova-compute.conf + when: scenario != "bifrost" + # ceph.conf + - src: "tests/templates/ceph-overrides.j2" + dest: /etc/kolla/config/ceph.conf + when: scenario == "ceph" + # bifrost/dib.yml + - src: "tests/templates/bifrost-dib-overrides.j2" + dest: /etc/kolla/config/bifrost/dib.yml + when: scenario == "bifrost" + when: item.when | default(true) - name: install kolla-ansible requirements pip: - requirements: "{{ ansible_env.HOME }}/{{ kolla_ansible_src_dir }}/requirements.txt" + requirements: "{{ kolla_ansible_src_dir }}/requirements.txt" become: true - name: copy passwords.yml file copy: - src: "{{ kolla_ansible_full_src_dir }}/etc/kolla/passwords.yml" + src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml" dest: /etc/kolla/passwords.yml + remote_src: true - name: generate passwords shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py" @@ -141,30 +128,133 @@ - all delegate_to: "{{ item }}" - - name: generate ceph config overrides - template: - src: "{{ kolla_ansible_full_src_dir }}/tests/templates/ceph-overrides.j2" - dest: /etc/kolla/config/ceph.conf - when: scenario == "ceph" - delegate_to: "primary" - + # NOTE(mgoddard): We are using the script module here and later to ensure + # we use the local copy of these scripts, rather than the one on the remote + # host, which could be checked out to a previous release (in an upgrade + # job). - name: Run setup_gate.sh script - shell: - cmd: tools/setup_gate.sh + script: + cmd: ../tools/setup_gate.sh executable: /bin/bash chdir: "{{ kolla_ansible_src_dir }}" environment: BASE_DISTRO: "{{ base_distro }}" INSTALL_TYPE: "{{ install_type }}" - NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs" BUILD_IMAGE: "{{ need_build_image }}" + TAG: "change_{{ zuul.change }}" KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla" ACTION: "{{ scenario }}" + # At this point we have generated all necessary configuration, and are + # ready to deploy the control plane services. Control flow now depends on + # the scenario being exercised. + + # Deploy control plane. For upgrade jobs this is the previous release. - block: - name: Run deploy.sh script + script: + cmd: deploy.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + + - name: Run test-openstack.sh script + script: + cmd: test-openstack.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + when: scenario not in ['scenario_nfv'] + + - name: Run test-scenario-nfv.sh script + script: + cmd: test-scenario-nfv.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + when: scenario == "scenario_nfv" + + - name: Run reconfigure.sh script + script: + cmd: reconfigure.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + when: scenario != "upgrade" + when: scenario != "bifrost" + + # Upgrade: update config. + - block: + - name: Run check-failure.sh script + script: + cmd: check-failure.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + + # NOTE(mgoddard): This only affects the remote copy of the repo, not the + # one on the executor. + - name: checkout the current kolla-ansible branch + command: + cmd: "git checkout {{ zuul.branch }}" + chdir: "{{ kolla_ansible_src_dir }}" + + # Use the new kolla-ansible repo to generate config files. + # This is the branch checked out on the executor, so we can use + # template directly rather than the remote-template role. + - name: Generate configuration files + template: + src: "{{ kolla_ansible_local_src_dir }}/{{ item.src }}" + dest: "{{ item.dest }}" + vars: + is_previous_release: false + with_items: + # Ansible inventory + - src: "tests/templates/inventory.j2" + dest: "{{ kolla_inventory_path }}" + # globals.yml + - src: "tests/templates/globals-default.j2" + dest: /etc/kolla/globals.yml + # nova-compute.conf + - src: "tests/templates/nova-compute-overrides.j2" + dest: /etc/kolla/config/nova/nova-compute.conf + # ceph.conf + - src: "tests/templates/ceph-overrides.j2" + dest: /etc/kolla/config/ceph.conf + when: "'ceph' in scenario" + when: item.when | default(true) + + - name: upgrade kolla-ansible requirements + pip: + requirements: "{{ kolla_ansible_src_dir }}/requirements.txt" + become: true + + # Update passwords.yml to include any new passwords added in this + # release. + - name: move passwords.yml to passwords.yml.old + command: mv /etc/kolla/passwords.yml /etc/kolla/passwords.yml.old + + - name: copy passwords.yml file + copy: + src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml" + dest: /etc/kolla/passwords.yml + remote_src: true + + - name: generate new passwords + shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py" + + - name: merge old and new passwords + shell: >- + {{ kolla_ansible_src_dir }}/tools/merge_passwords.py + --old /etc/kolla/passwords.yml.old + --new /etc/kolla/passwords.yml + --final /etc/kolla/passwords.yml + + # Perform an upgrade to the in-development code. + - name: Run upgrade.sh script shell: - cmd: tests/deploy.sh + cmd: tests/upgrade.sh executable: /bin/bash chdir: "{{ kolla_ansible_src_dir }}" environment: @@ -177,32 +267,9 @@ chdir: "{{ kolla_ansible_src_dir }}" environment: ACTION: "{{ scenario }}" - when: scenario not in ['scenario_nfv'] - - - name: Run test-scenario-nfv.sh script - shell: - cmd: tests/test-scenario-nfv.sh - executable: /bin/bash - chdir: "{{ kolla_ansible_src_dir }}" - when: scenario == "scenario_nfv" - - - name: Run reconfigure.sh script - shell: - cmd: tests/reconfigure.sh - executable: /bin/bash - chdir: "{{ kolla_ansible_src_dir }}" - environment: - ACTION: "{{ scenario }}" - - - name: Run upgrade.sh script - shell: - cmd: tests/upgrade.sh - executable: /bin/bash - chdir: "{{ kolla_ansible_src_dir }}" - environment: - ACTION: "{{ scenario }}" - when: scenario != "bifrost" + when: scenario == "upgrade" + # Bifrost testing. - block: - name: Run deploy-bifrost.sh script shell: diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index aca0b31158..5a2a3c753d 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -11,19 +11,32 @@ keepalived_virtual_router_id: "{{ 250 | random(1) }}" {% if enable_core_openstack | bool %} kolla_internal_vip_address: "{{ api_interface_address if hostvars | length > 2 else '169.254.169.10' }}" enable_haproxy: "{{ 'no' if hostvars | length > 2 else 'yes' }}" +# TODO(mgoddard): Remove this in the Train cycle when heat is enabled in the +# initial deployment. +enable_heat: "{{ scenario != 'upgrade' or previous_release != 'rocky' }}" neutron_external_interface: "fake_interface" openstack_logging_debug: "True" openstack_service_workers: "1" {% endif %} -{% if need_build_image %} +{% if need_build_image and not is_previous_release %} # NOTE(Jeffrey4l): use different a docker namespace name in case it pull image from hub.docker.io when deplying docker_namespace: "lokolla" docker_registry: "{{ api_interface_address }}:4000" +openstack_release: "change_{{ zuul.change }}" {% else %} # use docker hub images docker_namespace: "kolla" +{% if need_build_image and is_previous_release %} +# NOTE(mgoddard): Ensure that the insecure local registry is trusted, since it +# will be the source of images during the upgrade. +docker_custom_option: "--insecure-registry {{ api_interface_address }}:4000" +{% endif %} +{% if not is_previous_release %} openstack_release: "{{ zuul.branch | basename }}" +{% else %} +openstack_release: "{{ previous_release }}" +{% endif %} {% endif %} {% if scenario == "ceph" %} diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 87b57d685e..2b92842626 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -8,19 +8,6 @@ export PYTHONUNBUFFERED=1 GIT_PROJECT_DIR=$(mktemp -d) -function clone_repos { - cat > /tmp/clonemap <