From 45a4f9c07589c9aa2e08fa342a57561eec21d137 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 5 Mar 2018 14:33:04 +0000
Subject: [PATCH] Add a job for testing deployment of bifrost
Deploys a bifrost container using kolla-ansible bifrost-deploy.
IPA and disk images are downloaded rather than built to improve
reliability.
Currently only minimal testing of the deployment is performed, creating
and deleting an ironic node. Ideally we would perform a bare metal node
deployment.
The job is based on CentOS, as Ubuntu bifrost deployment is currently
failing with a python-MySQLdb error.
Change-Id: Ic45094594c21116b5b0d6a606f568fc7954175e3
---
.zuul.yaml | 16 ++++
tests/get_logs.sh | 10 +++
tests/run.yml | 18 ++++
tests/templates/bifrost-dib-overrides.j2 | 7 ++
tests/templates/globals-default.j2 | 4 +
tools/setup_gate.sh | 107 ++++++++++++++++-------
6 files changed, 132 insertions(+), 30 deletions(-)
create mode 100644 tests/templates/bifrost-dib-overrides.j2
diff --git a/.zuul.yaml b/.zuul.yaml
index cb6ef3580d..d52dd7042e 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -8,6 +8,7 @@
- kolla-ansible-ubuntu-source-ceph
- kolla-ansible-centos-source-ceph
- kolla-ansible-oraclelinux-source-ceph
+ - kolla-ansible-bifrost-centos-source
- openstack-tox-lower-constraints
gate:
@@ -81,6 +82,13 @@
roles:
- zuul: openstack-infra/zuul-jobs
+- job:
+ name: kolla-ansible-bifrost-base
+ parent: kolla-ansible-base
+ vars:
+ scenario: bifrost
+ install_type: source
+
- job:
name: kolla-ansible-centos-source
parent: kolla-ansible-base
@@ -164,3 +172,11 @@
base_distro: oraclelinux
install_type: source
scenario: ceph
+
+- job:
+ name: kolla-ansible-bifrost-centos-source
+ parent: kolla-ansible-bifrost-base
+ nodeset: kolla-ansible-centos
+ voting: false
+ vars:
+ base_distro: centos
diff --git a/tests/get_logs.sh b/tests/get_logs.sh
index 486b54a269..5c0a7bc49b 100644
--- a/tests/get_logs.sh
+++ b/tests/get_logs.sh
@@ -46,6 +46,16 @@ copy_logs() {
docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
fi
+ # bifrost related logs
+ if [[ $(docker ps --filter name=bifrost_deploy --format "{{.Names}}") ]]; then
+ for service in dnsmasq ironic-api ironic-conductor ironic-inspector mariadb nginx rabbitmq-server; do
+ mkdir -p ${LOG_DIR}/kolla/$service
+ docker exec bifrost_deploy systemctl status $service > ${LOG_DIR}/kolla/$service/systemd-status-$service.txt
+ done
+ docker exec bifrost_deploy journalctl -u mariadb > ${LOG_DIR}/kolla/mariadb/mariadb.txt
+ docker exec bifrost_deploy journalctl -u rabbitmq-server > ${LOG_DIR}/kolla/rabbitmq-server/rabbitmq.txt
+ fi
+
for container in $(docker ps -a --format "{{.Names}}"); do
docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt
done
diff --git a/tests/run.yml b/tests/run.yml
index cb9d834a91..3300329a11 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -62,6 +62,7 @@
path: "/etc/kolla/config/nova"
state: "directory"
mode: 0777
+ when: scenario != "bifrost"
become: true
delegate_to: "primary"
@@ -69,6 +70,23 @@
template:
src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2"
dest: /etc/kolla/config/nova/nova-compute.conf
+ when: scenario != "bifrost"
+ delegate_to: "primary"
+
+ - name: ensure bifrost conf overrides dir exists
+ file:
+ path: "/etc/kolla/config/bifrost"
+ state: "directory"
+ mode: 0777
+ when: scenario == "bifrost"
+ become: true
+ delegate_to: "primary"
+
+ - name: generate bifrost DIB config overrides
+ template:
+ src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2"
+ dest: /etc/kolla/config/bifrost/dib.yml
+ when: scenario == "bifrost"
delegate_to: "primary"
- name: copy passwords.yml file
diff --git a/tests/templates/bifrost-dib-overrides.j2 b/tests/templates/bifrost-dib-overrides.j2
new file mode 100644
index 0000000000..0ada56a2f9
--- /dev/null
+++ b/tests/templates/bifrost-dib-overrides.j2
@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index 6478e08528..95a7f27c83 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -6,12 +6,14 @@ kolla_install_type: "{{ install_type }}"
# in the CI gate.
keepalived_virtual_router_id: "{{ 250 | random(1) }}"
+{% if scenario != "bifrost" %}
{% if hostvars|length > 2 %}
kolla_internal_vip_address: "{{ api_interface_address }}"
enable_haproxy: "no"
{% else %}
kolla_internal_vip_address: "169.254.169.10"
{% endif %}
+{% endif %}
network_interface: "{{ api_interface_name }}"
docker_restart_policy: "never"
@@ -24,11 +26,13 @@ docker_registry: "{{ api_interface_address }}:4000"
docker_namespace: "kolla"
openstack_release: "{{ zuul.branch | basename }}"
{% endif %}
+{% if scenario != "bifrost" %}
neutron_external_interface: "fake_interface"
enable_horizon: "yes"
enable_heat: "no"
openstack_logging_debug: "True"
openstack_service_workers: "1"
+{% endif %}
# This is experimental feature, disable if gate fail.
glance_enable_rolling_upgrade: "yes"
diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh
index fdfc29ff20..acb4682aba 100755
--- a/tools/setup_gate.sh
+++ b/tools/setup_gate.sh
@@ -34,12 +34,15 @@ EOF
echo "RUN echo $(base64 -w0 ${PIP_CONF}) | base64 -d > /etc/pip.conf" | sudo tee /etc/kolla/header
rm ${PIP_CONF}
-GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon"
+ if [[ $ACTION != "bifrost" ]]; then
+ GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon"
+ else
+ GATE_IMAGES="bifrost"
+ fi
-# TODO(jeffrey4l): this doesn't work with zuulv3
-if echo $ACTION | grep -q "ceph"; then
-GATE_IMAGES+=",ceph,cinder"
-fi
+ if [[ $ACTION == "ceph" ]]; then
+ GATE_IMAGES+=",ceph,cinder"
+ fi
cat <<EOF | sudo tee /etc/kolla/kolla-build.conf
[DEFAULT]
@@ -134,6 +137,70 @@ function sanity_check {
fi
}
+function test_openstack {
+ # Create dummy interface for neutron
+ ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all
+
+ #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+ sudo chmod -R 777 /etc/kolla
+ # Actually do the deployment
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
+ # TODO(jeffrey4l): add pull action when we have a local registry
+ # service in CI
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
+
+ # Test OpenStack Environment
+ # TODO: use kolla-ansible check when it's ready
+
+ sanity_check
+
+ # TODO(jeffrey4l): make some configure file change and
+ # trigger a real reconfigure
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/post-deploy
+ # TODO(jeffrey4l): need run a real upgrade
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
+
+ # run prechecks again
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
+}
+
+function sanity_check_bifrost {
+ # TODO(mgoddard): More testing, deploy bare metal nodes.
+ # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
+ # https://bugs.launchpad.net/bifrost/+bug/1754070.
+ attempts=0
+ while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
+ attempts=$((attempts + 1))
+ if [[ $attempts -gt 6 ]]; then
+ echo "Timed out waiting for ironic conductor to become active"
+ exit 1
+ fi
+ sleep 10
+ done
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
+ sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
+}
+
+function test_bifrost {
+ # TODO(mgoddard): run prechecks.
+
+ # Deploy the bifrost container.
+ # TODO(mgoddard): add pull action when we have a local registry service in
+ # CI.
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
+
+ # Test Bifrost Environment
+ sanity_check_bifrost
+
+ # TODO(mgoddard): make some configuration file changes and trigger a real
+ # reconfigure.
+ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost2
+
+ # TODO(mgoddard): perform an upgrade.
+}
+
check_failure() {
# All docker container's status are created, restarting, running, removing,
# paused, exited and dead. Containers without running status are treated as
@@ -160,30 +227,10 @@ setup_node
tools/kolla-ansible -i ${RAW_INVENTORY} bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
prepare_images
-# Create dummy interface for neutron
-ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all
-
-#TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
-sudo chmod -R 777 /etc/kolla
-# Actually do the deployment
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
-# TODO(jeffrey4l): add pull action when we have a local registry
-# service in CI
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
-
-# Test OpenStack Environment
-# TODO: use kolla-ansible check when it's ready
-
-sanity_check
-
-# TODO(jeffrey4l): make some configure file change and
-# trigger a real reconfigure
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/post-deploy
-# TODO(jeffrey4l): need run a real upgrade
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
-
-# run prechecks again
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
+if [[ $ACTION != bifrost ]]; then
+ test_openstack
+else
+ test_bifrost
+fi
check_failure