From fdcbd037847298aa19af6b2bb5be3e36ed9505f2 Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Tue, 7 Jul 2020 00:11:25 -0500 Subject: [PATCH] Allow parallelization in gate runner Change-Id: I393a22cfcaecb00d14d8ac643bd4b7ffbba03b12 Signed-off-by: Andrii Ostapenko --- playbooks/osh-infra-gate-runner.yaml | 8 ++- roles/osh-run-script-set/defaults/main.yaml | 18 +++++ roles/osh-run-script-set/tasks/main.yaml | 55 +++++++++++++++ roles/osh-run-script/tasks/main.yaml | 4 +- yamllint.conf | 3 +- zuul.d/jobs.yaml | 76 ++++++++++----------- 6 files changed, 121 insertions(+), 43 deletions(-) create mode 100644 roles/osh-run-script-set/defaults/main.yaml create mode 100644 roles/osh-run-script-set/tasks/main.yaml diff --git a/playbooks/osh-infra-gate-runner.yaml b/playbooks/osh-infra-gate-runner.yaml index 172c3ef69..409176828 100644 --- a/playbooks/osh-infra-gate-runner.yaml +++ b/playbooks/osh-infra-gate-runner.yaml @@ -27,12 +27,14 @@ file: path: "/tmp/artifacts" state: directory + - name: Run gate scripts include_role: - name: osh-run-script + name: "{{ ([item] | flatten | length == 1) | ternary('osh-run-script', 'osh-run-script-set') }}" vars: - gate_script_path: "{{ item }}" - with_items: "{{ gate_scripts }}" + workload: "{{ [item] | flatten }}" + loop: "{{ gate_scripts }}" + - name: "Downloads artifacts to executor" synchronize: src: "/tmp/artifacts" diff --git a/roles/osh-run-script-set/defaults/main.yaml b/roles/osh-run-script-set/defaults/main.yaml new file mode 100644 index 000000000..8de078a0b --- /dev/null +++ b/roles/osh-run-script-set/defaults/main.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +osh_params: + container_distro_name: ubuntu + container_distro_version: xenial + # feature_gates: +... diff --git a/roles/osh-run-script-set/tasks/main.yaml b/roles/osh-run-script-set/tasks/main.yaml new file mode 100644 index 000000000..b6f8add3c --- /dev/null +++ b/roles/osh-run-script-set/tasks/main.yaml @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- block: + - name: "Run script set {{ workload }}" + shell: | + set -xe; + {{ gate_script_path }} + loop: "{{ workload }}" + loop_control: + loop_var: gate_script_path + pause: 5 + args: + chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}" + environment: + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}" + OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}" + OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}" + CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}" + CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}" + FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}" + # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with + # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637 + async: 3600 + poll: 0 + register: async_results + + - name: Wait for script set to finish + async_status: + jid: '{{ item.ansible_job_id }}' + register: jobs + until: jobs.finished + delay: 5 + retries: 360 + loop: "{{ async_results.results }}" + + always: + - name: Print script set output + shell: | + # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status + echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}' + loop: "{{ jobs.results }}" +... diff --git a/roles/osh-run-script/tasks/main.yaml b/roles/osh-run-script/tasks/main.yaml index b399779cf..6392fc40e 100644 --- a/roles/osh-run-script/tasks/main.yaml +++ b/roles/osh-run-script/tasks/main.yaml @@ -11,10 +11,12 @@ # limitations under the License. --- -- name: "Run script {{ gate_script_path }}" +- name: "Run script {{ workload[0] }}" shell: | set -xe; {{ gate_script_path }} + vars: + gate_script_path: "{{ workload[0] }}" args: chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}" environment: diff --git a/yamllint.conf b/yamllint.conf index 382224b5a..1aa516185 100644 --- a/yamllint.conf +++ b/yamllint.conf @@ -16,7 +16,8 @@ rules: document-start: enable empty-lines: enable empty-values: disable - hyphens: enable + hyphens: + ignore: .yamllint/zuul.d/jobs.yaml indentation: spaces: 2 indent-sequences: whatever diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 6c0ebfe93..3fc11c880 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -139,14 +139,14 @@ gate_scripts: - ./tools/deployment/osh-infra-logging/000-install-packages.sh - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh - - ./tools/deployment/osh-infra-logging/010-ingress.sh - - ./tools/deployment/osh-infra-logging/020-ceph.sh - - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh - - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh - - ./tools/deployment/osh-infra-logging/040-ldap.sh - - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh - - ./tools/deployment/osh-infra-logging/060-fluentd.sh - - ./tools/deployment/osh-infra-logging/070-kibana.sh + - - ./tools/deployment/osh-infra-logging/010-ingress.sh + - ./tools/deployment/osh-infra-logging/020-ceph.sh + - - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh + - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh + - ./tools/deployment/osh-infra-logging/040-ldap.sh + - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh + - - ./tools/deployment/osh-infra-logging/060-fluentd.sh + - ./tools/deployment/osh-infra-logging/070-kibana.sh - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true - job: @@ -183,24 +183,24 @@ gate_scripts: - ./tools/deployment/osh-infra-monitoring/000-install-packages.sh - ./tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh - - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh - - ./tools/deployment/osh-infra-monitoring/020-ingress.sh - - ./tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh - - ./tools/deployment/osh-infra-monitoring/040-ldap.sh - - ./tools/deployment/osh-infra-monitoring/045-mariadb.sh - - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh - - ./tools/deployment/osh-infra-monitoring/060-alertmanager.sh - - ./tools/deployment/osh-infra-monitoring/070-kube-state-metrics.sh - - ./tools/deployment/osh-infra-monitoring/075-node-problem-detector.sh - - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh - - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh - - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh - - ./tools/deployment/osh-infra-monitoring/110-grafana.sh - - ./tools/deployment/osh-infra-monitoring/120-nagios.sh - - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh - - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true - - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true - - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true + - - ./tools/deployment/osh-infra-monitoring/010-deploy-docker-registry.sh + - ./tools/deployment/osh-infra-monitoring/020-ingress.sh + - ./tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh + - ./tools/deployment/osh-infra-monitoring/040-ldap.sh + - ./tools/deployment/osh-infra-monitoring/045-mariadb.sh + - - ./tools/deployment/osh-infra-monitoring/050-prometheus.sh + - ./tools/deployment/osh-infra-monitoring/060-alertmanager.sh + - ./tools/deployment/osh-infra-monitoring/070-kube-state-metrics.sh + - ./tools/deployment/osh-infra-monitoring/075-node-problem-detector.sh + - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh + - - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh + - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh + - ./tools/deployment/osh-infra-monitoring/110-grafana.sh + - ./tools/deployment/osh-infra-monitoring/120-nagios.sh + - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh + - - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true + - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true + - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true - job: name: openstack-helm-infra-federated-monitoring @@ -394,20 +394,20 @@ - ./tools/deployment/openstack-support/000-install-packages.sh - ./tools/deployment/openstack-support/005-deploy-k8s.sh - ./tools/deployment/openstack-support/007-namespace-config.sh - - ./tools/deployment/openstack-support/010-ingress.sh - - ./tools/deployment/openstack-support/020-ceph.sh + - - ./tools/deployment/openstack-support/010-ingress.sh + - ./tools/deployment/openstack-support/020-ceph.sh - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh - - ./tools/deployment/openstack-support/030-rabbitmq.sh - - ./tools/deployment/openstack-support/040-memcached.sh - - ./tools/deployment/openstack-support/050-libvirt.sh - - ./tools/deployment/openstack-support/060-openvswitch.sh - - ./tools/deployment/openstack-support/070-mariadb.sh - - ./tools/deployment/openstack-support/080-setup-client.sh + - - ./tools/deployment/openstack-support/030-rabbitmq.sh + - ./tools/deployment/openstack-support/040-memcached.sh + - ./tools/deployment/openstack-support/050-libvirt.sh + - ./tools/deployment/openstack-support/060-openvswitch.sh + - ./tools/deployment/openstack-support/070-mariadb.sh + - ./tools/deployment/openstack-support/080-setup-client.sh - ./tools/deployment/openstack-support/090-keystone.sh - - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh - - ./tools/deployment/openstack-support/110-openstack-exporter.sh - - ./tools/deployment/openstack-support/120-powerdns.sh - - ./tools/deployment/openstack-support/130-cinder.sh + - - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh + - ./tools/deployment/openstack-support/110-openstack-exporter.sh + - ./tools/deployment/openstack-support/120-powerdns.sh + - ./tools/deployment/openstack-support/130-cinder.sh - job: name: openstack-helm-infra-five-ubuntu