From f92de48c287136074046476b7bb9888d9af23b3d Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 10 Jul 2018 15:38:40 +0100
Subject: [PATCH] Refactor CI testing scripts
This change is intended to ease the addition of new test scenarios, by
splitting the existing setup_gate.sh script up into several smaller
scripts. We may wish to further break these scripts up in future,
depending on the requirements of new test jobs.
Change-Id: Ifa442ac79a54f749c56d20f5f7383c3852d95268
---
tests/check-failure.sh | 27 +++++++++
tests/deploy-bifrost.sh | 21 +++++++
tests/deploy.sh | 28 +++++++++
tests/reconfigure.sh | 21 +++++++
tests/run.yml | 84 ++++++++++++++++++++++-----
tests/test-bifrost.sh | 29 ++++++++++
tests/test-openstack.sh | 38 +++++++++++++
tests/upgrade-bifrost.sh | 22 +++++++
tests/upgrade.sh | 22 +++++++
tools/setup_gate.sh | 120 ---------------------------------------
10 files changed, 278 insertions(+), 134 deletions(-)
create mode 100755 tests/check-failure.sh
create mode 100755 tests/deploy-bifrost.sh
create mode 100755 tests/deploy.sh
create mode 100755 tests/reconfigure.sh
create mode 100755 tests/test-bifrost.sh
create mode 100755 tests/test-openstack.sh
create mode 100755 tests/upgrade-bifrost.sh
create mode 100755 tests/upgrade.sh
diff --git a/tests/check-failure.sh b/tests/check-failure.sh
new file mode 100755
index 0000000000..6689d86a68
--- /dev/null
+++ b/tests/check-failure.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+check_failure() {
+ # All docker container's status are created, restarting, running, removing,
+ # paused, exited and dead. Containers without running status are treated as
+ # failure. removing is added in docker 1.13, just ignore it now.
+ failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
+ --filter status=created \
+ --filter status=restarting \
+ --filter status=paused \
+ --filter status=exited \
+ --filter status=dead)
+
+ if [[ -n "$failed_containers" ]]; then
+ exit 1;
+ fi
+}
+
+
+check_failure
diff --git a/tests/deploy-bifrost.sh b/tests/deploy-bifrost.sh
new file mode 100755
index 0000000000..f45e40c4bb
--- /dev/null
+++ b/tests/deploy-bifrost.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function deploy_bifrost {
+ RAW_INVENTORY=/etc/kolla/inventory
+
+ # TODO(mgoddard): run prechecks.
+ # Deploy the bifrost container.
+ # TODO(mgoddard): add pull action when we have a local registry service in
+ # CI.
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
+}
+
+
+deploy_bifrost
diff --git a/tests/deploy.sh b/tests/deploy.sh
new file mode 100755
index 0000000000..c06ef46534
--- /dev/null
+++ b/tests/deploy.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function deploy {
+ RAW_INVENTORY=/etc/kolla/inventory
+
+ # Create dummy interface for neutron
+ ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
+
+ #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+ sudo chmod -R 777 /etc/kolla
+ # Actually do the deployment
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/deploy-prechecks
+ # TODO(jeffrey4l): add pull action when we have a local registry
+ # service in CI
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
+}
+
+
+deploy
diff --git a/tests/reconfigure.sh b/tests/reconfigure.sh
new file mode 100755
index 0000000000..f8b2c04f40
--- /dev/null
+++ b/tests/reconfigure.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function reconfigure {
+ RAW_INVENTORY=/etc/kolla/inventory
+
+ # TODO(jeffrey4l): make some configure file change and
+ # trigger a real reconfigure
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/reconfigure-prechecks
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
+}
+
+
+reconfigure
diff --git a/tests/run.yml b/tests/run.yml
index 39aed9a44c..04554b16eb 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -145,19 +145,75 @@
when: scenario == "ceph"
delegate_to: "primary"
- - shell:
- cmd: |
- set -e
- set -x
- export BASE_DISTRO={{ base_distro }}
- export INSTALL_TYPE={{ install_type }}
- export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs
- export BUILD_IMAGE={{ need_build_image }}
- export KOLLA_SRC_DIR={{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla
- export ACTION={{ scenario }}
-
- tools/setup_gate.sh
+ - name: Run setup_gate.sh script
+ shell:
+ cmd: tools/setup_gate.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ environment:
+ BASE_DISTRO: "{{ base_distro }}"
+ INSTALL_TYPE: "{{ install_type }}"
+ NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs"
+ BUILD_IMAGE: "{{ need_build_image }}"
+ KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla"
+ ACTION: "{{ scenario }}"
+
+ - block:
+ - name: Run deploy.sh script
+ shell:
+ cmd: tests/deploy.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ environment:
+ ACTION: "{{ scenario }}"
+
+ - name: Run test-openstack.sh script
+ shell:
+ cmd: tests/test-openstack.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ environment:
+ ACTION: "{{ scenario }}"
+
+ - name: Run reconfigure.sh script
+ shell:
+ cmd: tests/reconfigure.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ environment:
+ ACTION: "{{ scenario }}"
+
+ - name: Run upgrade.sh script
+ shell:
+ cmd: tests/upgrade.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ environment:
+ ACTION: "{{ scenario }}"
+ when: scenario != "bifrost"
+
+ - block:
+ - name: Run deploy-bifrost.sh script
+ shell:
+ cmd: tests/deploy-bifrost.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+
+ - name: Run test-bifrost.sh script
+ shell:
+ cmd: tests/test-bifrost.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+
+ - name: Run upgrade-bifrost.sh script
+ shell:
+ cmd: tests/upgrade-bifrost.sh
+ executable: /bin/bash
+ chdir: "{{ kolla_ansible_src_dir }}"
+ when: scenario == "bifrost"
+
+ - name: Run check-failure.sh script
+ shell:
+ cmd: tests/check-failure.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
- delegate_to: "primary"
- environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/tests/test-bifrost.sh b/tests/test-bifrost.sh
new file mode 100755
index 0000000000..94c71f482f
--- /dev/null
+++ b/tests/test-bifrost.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function test_bifrost {
+ # TODO(mgoddard): More testing, deploy bare metal nodes.
+ # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
+ # https://bugs.launchpad.net/bifrost/+bug/1754070.
+ attempts=0
+ while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
+ attempts=$((attempts + 1))
+ if [[ $attempts -gt 6 ]]; then
+ echo "Timed out waiting for ironic conductor to become active"
+ exit 1
+ fi
+ sleep 10
+ done
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
+}
+
+
+test_bifrost
diff --git a/tests/test-openstack.sh b/tests/test-openstack.sh
new file mode 100755
index 0000000000..f6cee78841
--- /dev/null
+++ b/tests/test-openstack.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function test_openstack {
+ # Wait for service ready
+ sleep 15
+ . /etc/kolla/admin-openrc.sh
+ # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
+ # Remove this after this bug is fixed
+ # https://bugs.launchpad.net/oslo.cache/+bug/1590779
+ sudo docker restart memcached
+ nova --debug service-list
+ openstack --debug network agent list
+ tools/init-runonce
+ nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
+
+ nova --debug list
+ # If the status is not ACTIVE, print info and exit 1
+ nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
+ if echo $ACTION | grep -q "ceph"; then
+ openstack volume create --size 2 test_volume
+ openstack server add volume kolla_boot_test test_volume --device /dev/vdb
+ fi
+ if echo $ACTION | grep -q "zun"; then
+ openstack --debug appcontainer service list
+ openstack --debug appcontainer host list
+ # TODO(hongbin): Run a Zun container and assert the container becomes
+ # Running
+ fi
+}
+
+test_openstack
diff --git a/tests/upgrade-bifrost.sh b/tests/upgrade-bifrost.sh
new file mode 100755
index 0000000000..d6e9f5d248
--- /dev/null
+++ b/tests/upgrade-bifrost.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function upgrade_bifrost {
+ RAW_INVENTORY=/etc/kolla/inventory
+
+ # TODO(mgoddard): run prechecks.
+ # TODO(mgoddard): add pull action when we have a local registry service in
+ # CI.
+ # TODO(mgoddard): make some configuration file changes and trigger a real
+ # upgrade.
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/upgrade-bifrost
+}
+
+
+upgrade_bifrost
diff --git a/tests/upgrade.sh b/tests/upgrade.sh
new file mode 100755
index 0000000000..d3ebba63d1
--- /dev/null
+++ b/tests/upgrade.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function upgrade {
+ RAW_INVENTORY=/etc/kolla/inventory
+
+ # TODO(jeffrey4l): need run a real upgrade
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/upgrade-prechecks
+ # TODO(mgoddard): add pull action when we have a local registry service in
+ # CI.
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
+}
+
+
+upgrade
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
index a6cb4e391f..9cdbf73e76 100755
--- a/tools/setup_gate.sh
+++ b/tools/setup_gate.sh
@@ -122,118 +122,6 @@ function prepare_images {
popd
}
-function sanity_check {
- # Wait for service ready
- sleep 15
- . /etc/kolla/admin-openrc.sh
- # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
- # Remove this after this bug is fixed
- # https://bugs.launchpad.net/oslo.cache/+bug/1590779
- sudo docker restart memcached
- nova --debug service-list
- openstack --debug network agent list
- tools/init-runonce
- nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
-
- nova --debug list
- # If the status is not ACTIVE, print info and exit 1
- nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
- if echo $ACTION | grep -q "ceph"; then
- openstack volume create --size 2 test_volume
- openstack server add volume kolla_boot_test test_volume --device /dev/vdb
- fi
- if echo $ACTION | grep -q "zun"; then
- openstack --debug appcontainer service list
- openstack --debug appcontainer host list
- # TODO(hongbin): Run a Zun container and assert the container becomes
- # Running
- fi
-}
-
-function test_openstack {
- # Create dummy interface for neutron
- ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
-
- #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
- sudo chmod -R 777 /etc/kolla
- # Actually do the deployment
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
- # TODO(jeffrey4l): add pull action when we have a local registry
- # service in CI
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
-
- # Test OpenStack Environment
- # TODO: use kolla-ansible check when it's ready
-
- sanity_check
-
- # TODO(jeffrey4l): make some configure file change and
- # trigger a real reconfigure
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
- # TODO(jeffrey4l): need run a real upgrade
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
-
- # run prechecks again
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
-}
-
-function sanity_check_bifrost {
- # TODO(mgoddard): More testing, deploy bare metal nodes.
- # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
- # https://bugs.launchpad.net/bifrost/+bug/1754070.
- attempts=0
- while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
- attempts=$((attempts + 1))
- if [[ $attempts -gt 6 ]]; then
- echo "Timed out waiting for ironic conductor to become active"
- exit 1
- fi
- sleep 10
- done
- sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
- sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
- sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
-}
-
-function test_bifrost {
- # TODO(mgoddard): run prechecks.
-
- # Deploy the bifrost container.
- # TODO(mgoddard): add pull action when we have a local registry service in
- # CI.
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
-
- # Test Bifrost Environment
- sanity_check_bifrost
-
- # TODO(mgoddard): make some configuration file changes and trigger a real
- # reconfigure.
- tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost2
-
- # TODO(mgoddard): perform an upgrade.
-}
-
-check_failure() {
- # All docker container's status are created, restarting, running, removing,
- # paused, exited and dead. Containers without running status are treated as
- # failure. removing is added in docker 1.13, just ignore it now.
- failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
- --filter status=created \
- --filter status=restarting \
- --filter status=paused \
- --filter status=exited \
- --filter status=dead)
-
- if [[ -n "$failed_containers" ]]; then
- exit 1;
- fi
-}
-
-
clone_repos
setup_ansible
@@ -242,11 +130,3 @@ setup_node
tools/kolla-ansible -i ${RAW_INVENTORY} -e ansible_user=$USER bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
prepare_images
-
-if [[ $ACTION != bifrost ]]; then
- test_openstack
-else
- test_bifrost
-fi
-
-check_failure