From 45a4f9c07589c9aa2e08fa342a57561eec21d137 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 5 Mar 2018 14:33:04 +0000 Subject: [PATCH] Add a job for testing deployment of bifrost Deploys a bifrost container using kolla-ansible bifrost-deploy. IPA and disk images are downloaded rather than built to improve reliability. Currently only minimal testing of the deployment is performed, creating and deleting an ironic node. Ideally we would perform a bare metal node deployment. The job is based on CentOS, as Ubuntu bifrost deployment is currently failing with a python-MySQLdb error. Change-Id: Ic45094594c21116b5b0d6a606f568fc7954175e3 --- .zuul.yaml | 16 ++++ tests/get_logs.sh | 10 +++ tests/run.yml | 18 ++++ tests/templates/bifrost-dib-overrides.j2 | 7 ++ tests/templates/globals-default.j2 | 4 + tools/setup_gate.sh | 107 ++++++++++++++++------- 6 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 tests/templates/bifrost-dib-overrides.j2 diff --git a/.zuul.yaml b/.zuul.yaml index cb6ef3580d..d52dd7042e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -8,6 +8,7 @@ - kolla-ansible-ubuntu-source-ceph - kolla-ansible-centos-source-ceph - kolla-ansible-oraclelinux-source-ceph + - kolla-ansible-bifrost-centos-source - openstack-tox-lower-constraints gate: @@ -81,6 +82,13 @@ roles: - zuul: openstack-infra/zuul-jobs +- job: + name: kolla-ansible-bifrost-base + parent: kolla-ansible-base + vars: + scenario: bifrost + install_type: source + - job: name: kolla-ansible-centos-source parent: kolla-ansible-base @@ -164,3 +172,11 @@ base_distro: oraclelinux install_type: source scenario: ceph + +- job: + name: kolla-ansible-bifrost-centos-source + parent: kolla-ansible-bifrost-base + nodeset: kolla-ansible-centos + voting: false + vars: + base_distro: centos diff --git a/tests/get_logs.sh b/tests/get_logs.sh index 486b54a269..5c0a7bc49b 100644 --- a/tests/get_logs.sh +++ b/tests/get_logs.sh @@ -46,6 +46,16 @@ copy_logs() { docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt fi + # bifrost related logs + if [[ $(docker ps --filter name=bifrost_deploy --format "{{.Names}}") ]]; then + for service in dnsmasq ironic-api ironic-conductor ironic-inspector mariadb nginx rabbitmq-server; do + mkdir -p ${LOG_DIR}/kolla/$service + docker exec bifrost_deploy systemctl status $service > ${LOG_DIR}/kolla/$service/systemd-status-$service.txt + done + docker exec bifrost_deploy journalctl -u mariadb > ${LOG_DIR}/kolla/mariadb/mariadb.txt + docker exec bifrost_deploy journalctl -u rabbitmq-server > ${LOG_DIR}/kolla/rabbitmq-server/rabbitmq.txt + fi + for container in $(docker ps -a --format "{{.Names}}"); do docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt done diff --git a/tests/run.yml b/tests/run.yml index cb9d834a91..3300329a11 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -62,6 +62,7 @@ path: "/etc/kolla/config/nova" state: "directory" mode: 0777 + when: scenario != "bifrost" become: true delegate_to: "primary" @@ -69,6 +70,23 @@ template: src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2" dest: /etc/kolla/config/nova/nova-compute.conf + when: scenario != "bifrost" + delegate_to: "primary" + + - name: ensure bifrost conf overrides dir exists + file: + path: "/etc/kolla/config/bifrost" + state: "directory" + mode: 0777 + when: scenario == "bifrost" + become: true + delegate_to: "primary" + + - name: generate bifrost DIB config overrides + template: + src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2" + dest: /etc/kolla/config/bifrost/dib.yml + when: scenario == "bifrost" delegate_to: "primary" - name: copy passwords.yml file diff --git a/tests/templates/bifrost-dib-overrides.j2 b/tests/templates/bifrost-dib-overrides.j2 new file mode 100644 index 0000000000..0ada56a2f9 --- /dev/null +++ b/tests/templates/bifrost-dib-overrides.j2 @@ -0,0 +1,7 @@ +--- +# Don't build an IPA deployment image, instead download upstream images. +create_ipa_image: false +download_ipa: true + +# Don't build a disk image. It takes time and can be unreliable. +use_cirros: true diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 6478e08528..95a7f27c83 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -6,12 +6,14 @@ kolla_install_type: "{{ install_type }}" # in the CI gate. keepalived_virtual_router_id: "{{ 250 | random(1) }}" +{% if scenario != "bifrost" %} {% if hostvars|length > 2 %} kolla_internal_vip_address: "{{ api_interface_address }}" enable_haproxy: "no" {% else %} kolla_internal_vip_address: "169.254.169.10" {% endif %} +{% endif %} network_interface: "{{ api_interface_name }}" docker_restart_policy: "never" @@ -24,11 +26,13 @@ docker_registry: "{{ api_interface_address }}:4000" docker_namespace: "kolla" openstack_release: "{{ zuul.branch | basename }}" {% endif %} +{% if scenario != "bifrost" %} neutron_external_interface: "fake_interface" enable_horizon: "yes" enable_heat: "no" openstack_logging_debug: "True" openstack_service_workers: "1" +{% endif %} # This is experimental feature, disable if gate fail. glance_enable_rolling_upgrade: "yes" diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index fdfc29ff20..acb4682aba 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -34,12 +34,15 @@ EOF echo "RUN echo $(base64 -w0 ${PIP_CONF}) | base64 -d > /etc/pip.conf" | sudo tee /etc/kolla/header rm ${PIP_CONF} -GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon" + if [[ $ACTION != "bifrost" ]]; then + GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon" + else + GATE_IMAGES="bifrost" + fi -# TODO(jeffrey4l): this doesn't work with zuulv3 -if echo $ACTION | grep -q "ceph"; then -GATE_IMAGES+=",ceph,cinder" -fi + if [[ $ACTION == "ceph" ]]; then + GATE_IMAGES+=",ceph,cinder" + fi cat < /tmp/logs/ansible/prechecks1 + # TODO(jeffrey4l): add pull action when we have a local registry + # service in CI + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy + + # Test OpenStack Environment + # TODO: use kolla-ansible check when it's ready + + sanity_check + + # TODO(jeffrey4l): make some configure file change and + # trigger a real reconfigure + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/post-deploy + # TODO(jeffrey4l): need run a real upgrade + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade + + # run prechecks again + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2 +} + +function sanity_check_bifrost { + # TODO(mgoddard): More testing, deploy bare metal nodes. + # TODO(mgoddard): Use openstackclient when clouds.yaml works. See + # https://bugs.launchpad.net/bifrost/+bug/1754070. + attempts=0 + while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do + attempts=$((attempts + 1)) + if [[ $attempts -gt 6 ]]; then + echo "Timed out waiting for ironic conductor to become active" + exit 1 + fi + sleep 10 + done + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list" + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node" + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node" +} + +function test_bifrost { + # TODO(mgoddard): run prechecks. + + # Deploy the bifrost container. + # TODO(mgoddard): add pull action when we have a local registry service in + # CI. + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost + + # Test Bifrost Environment + sanity_check_bifrost + + # TODO(mgoddard): make some configuration file changes and trigger a real + # reconfigure. + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost2 + + # TODO(mgoddard): perform an upgrade. +} + check_failure() { # All docker container's status are created, restarting, running, removing, # paused, exited and dead. Containers without running status are treated as @@ -160,30 +227,10 @@ setup_node tools/kolla-ansible -i ${RAW_INVENTORY} bootstrap-servers > /tmp/logs/ansible/bootstrap-servers prepare_images -# Create dummy interface for neutron -ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all - -#TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there -sudo chmod -R 777 /etc/kolla -# Actually do the deployment -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1 -# TODO(jeffrey4l): add pull action when we have a local registry -# service in CI -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy - -# Test OpenStack Environment -# TODO: use kolla-ansible check when it's ready - -sanity_check - -# TODO(jeffrey4l): make some configure file change and -# trigger a real reconfigure -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/post-deploy -# TODO(jeffrey4l): need run a real upgrade -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade - -# run prechecks again -tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2 +if [[ $ACTION != bifrost ]]; then + test_openstack +else + test_bifrost +fi check_failure