From 906228915107ed2c4c79292a133daed56e7aa69b Mon Sep 17 00:00:00 2001 From: Albin Vass Date: Fri, 1 May 2020 11:46:46 +0200 Subject: [PATCH] Check for loop_control in with_ type loops Change-Id: I191265df7709a6262b44a428d78fe28ffaeb4b75 --- .rules/ZuulJobsNamespaceLoopVar.py | 10 +++++-- roles/add-authorized-keys/tasks/main.yaml | 4 ++- roles/configure-mirrors/tasks/mirror.yaml | 8 ++++-- .../tasks/mirror/CentOS-7.yaml | 6 +++-- .../tasks/mirror/CentOS-8.yaml | 6 +++-- .../tasks/mirror/Debian.yaml | 6 +++-- .../tasks/mirror/Fedora.yaml | 6 +++-- .../configure-mirrors/tasks/mirror/Suse.yaml | 6 +++-- .../tasks/mirror/Ubuntu.yaml | 6 +++-- roles/ensure-docker/tasks/main.yaml | 6 +++-- roles/ensure-openshift/tasks/main.yaml | 8 ++++-- roles/ensure-output-dirs/tasks/main.yaml | 4 ++- roles/ensure-python/tasks/pyenv.yaml | 6 +++-- roles/fetch-output-openshift/tasks/main.yaml | 8 ++++-- roles/fetch-output-openshift/tasks/rsync.yaml | 2 +- roles/fetch-output/tasks/main.yaml | 4 ++- .../tasks/main.yaml | 8 ++++-- .../fetch-python-sdist-output/tasks/main.yaml | 8 ++++-- roles/fetch-sphinx-tarball/tasks/pdf.yaml | 16 +++++++++--- roles/fetch-subunit-output/tasks/process.yaml | 12 ++++++--- roles/fetch-tox-output/tasks/main.yaml | 12 ++++++--- roles/git-prepare-nodecache/tasks/main.yaml | 18 +++++++++---- roles/markdownlint/tasks/main.yaml | 6 +++-- .../tasks/main.yaml | 20 +++++++++----- roles/multi-node-bridge/tasks/common.yaml | 16 ++++++++---- roles/multi-node-firewall/tasks/main.yaml | 8 ++++-- roles/multi-node-hosts-file/tasks/main.yaml | 4 ++- roles/multi-node-known-hosts/tasks/main.yaml | 12 ++++++--- roles/prepare-workspace-git/tasks/main.yaml | 26 ++++++++++++------- roles/sign-artifacts/tasks/main.yaml | 4 ++- roles/sphinx/tasks/main.yaml | 6 +++-- roles/stage-output/tasks/main.yaml | 26 ++++++++++++------- .../tasks/main.yaml | 18 ++++++++----- roles/upload-logs/tasks/main.yaml | 16 ++++++++---- roles/upload-pypi/tasks/main.yaml | 16 +++++++++--- roles/use-buildset-registry/tasks/main.yaml | 4 ++- 36 files changed, 246 insertions(+), 106 deletions(-) diff --git a/.rules/ZuulJobsNamespaceLoopVar.py b/.rules/ZuulJobsNamespaceLoopVar.py index a447aaab0..1d7413c26 100644 --- a/.rules/ZuulJobsNamespaceLoopVar.py +++ b/.rules/ZuulJobsNamespaceLoopVar.py @@ -18,8 +18,14 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\ def matchtask(self, file, task): if file.get('type') != 'tasks': return False - if 'loop' in set(task.keys()): - if 'loop_control' not in set(task.keys()): + + has_loop = 'loop' in task + for key in task.keys(): + if key.startswith('with_'): + has_loop = True + + if has_loop: + if 'loop_control' not in task: return True elif 'loop_var' not in task.get('loop_control'): return True diff --git a/roles/add-authorized-keys/tasks/main.yaml b/roles/add-authorized-keys/tasks/main.yaml index 3abe0fc2b..904870f1a 100644 --- a/roles/add-authorized-keys/tasks/main.yaml +++ b/roles/add-authorized-keys/tasks/main.yaml @@ -2,6 +2,8 @@ authorized_key: user: "{{ ansible_ssh_user }}" state: present - key: "{{ item.public_key }}" + key: "{{ zj_public_key.public_key }}" with_items: - "{{ public_keys }}" + loop_control: + loop_var: zj_public_key diff --git a/roles/configure-mirrors/tasks/mirror.yaml b/roles/configure-mirrors/tasks/mirror.yaml index 42717093c..5b7471ddf 100644 --- a/roles/configure-mirrors/tasks/mirror.yaml +++ b/roles/configure-mirrors/tasks/mirror.yaml @@ -4,13 +4,15 @@ state: absent - name: Include OS-specific variables - include_vars: "{{ item }}" + include_vars: "{{ zj_distro_os }}" with_first_found: - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" - "{{ ansible_distribution }}.yaml" - "{{ ansible_os_family }}.yaml" - "default.yaml" + loop_control: + loop_var: zj_distro_os - name: Install /etc/pip.conf configuration become: yes @@ -22,10 +24,12 @@ src: etc/pip.conf.j2 - name: Setup distribution specific packaging mirrors - include: "{{ item }}" + include: "{{ zj_distro_os }}" static: no with_first_found: - "mirror/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" - "mirror/{{ ansible_distribution }}.yaml" - "mirror/{{ ansible_os_family }}.yaml" - "mirror/default.yaml" + loop_control: + loop_var: zj_distro_os diff --git a/roles/configure-mirrors/tasks/mirror/CentOS-7.yaml b/roles/configure-mirrors/tasks/mirror/CentOS-7.yaml index 799147cf8..bcdce968a 100644 --- a/roles/configure-mirrors/tasks/mirror/CentOS-7.yaml +++ b/roles/configure-mirrors/tasks/mirror/CentOS-7.yaml @@ -1,14 +1,16 @@ - name: Install CentOS repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "centos7/{{ item }}.j2" + src: "centos7/{{ zj_repo }}.j2" with_items: - etc/yum.repos.d/CentOS-Base.repo - etc/yum.repos.d/epel.repo + loop_control: + loop_var: zj_repo notify: - Update yum/dnf cache diff --git a/roles/configure-mirrors/tasks/mirror/CentOS-8.yaml b/roles/configure-mirrors/tasks/mirror/CentOS-8.yaml index 68d9a573b..83a48fffc 100644 --- a/roles/configure-mirrors/tasks/mirror/CentOS-8.yaml +++ b/roles/configure-mirrors/tasks/mirror/CentOS-8.yaml @@ -1,11 +1,11 @@ - name: Install CentOS 8 repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "centos8/{{ item }}.j2" + src: "centos8/{{ zj_repo }}.j2" with_items: - etc/yum.repos.d/CentOS-AppStream.repo - etc/yum.repos.d/CentOS-Base.repo @@ -13,6 +13,8 @@ - etc/yum.repos.d/CentOS-Extras.repo - etc/yum.repos.d/CentOS-PowerTools.repo - etc/yum.repos.d/epel.repo + loop_control: + loop_var: zj_repo notify: - Update yum/dnf cache diff --git a/roles/configure-mirrors/tasks/mirror/Debian.yaml b/roles/configure-mirrors/tasks/mirror/Debian.yaml index 169773f1f..f6f4f0a66 100644 --- a/roles/configure-mirrors/tasks/mirror/Debian.yaml +++ b/roles/configure-mirrors/tasks/mirror/Debian.yaml @@ -1,16 +1,18 @@ - name: Install Debian repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "apt/{{ item }}.j2" + src: "apt/{{ zj_repo }}.j2" with_items: - etc/apt/sources.list.d/default.list - etc/apt/sources.list.d/updates.list - etc/apt/sources.list.d/backports.list - etc/apt/sources.list.d/security.list - etc/apt/apt.conf.d/99unauthenticated + loop_control: + loop_var: zj_repo notify: - Update apt cache diff --git a/roles/configure-mirrors/tasks/mirror/Fedora.yaml b/roles/configure-mirrors/tasks/mirror/Fedora.yaml index 81eb5f460..2411c4e0f 100644 --- a/roles/configure-mirrors/tasks/mirror/Fedora.yaml +++ b/roles/configure-mirrors/tasks/mirror/Fedora.yaml @@ -1,14 +1,16 @@ - name: Install Fedora repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "fedora/{{ item }}.j2" + src: "fedora/{{ zj_repo }}.j2" with_items: - etc/yum.repos.d/fedora.repo - etc/yum.repos.d/fedora-updates.repo + loop_control: + loop_var: zj_repo notify: - Update yum/dnf cache diff --git a/roles/configure-mirrors/tasks/mirror/Suse.yaml b/roles/configure-mirrors/tasks/mirror/Suse.yaml index 549ef445f..19252d52c 100644 --- a/roles/configure-mirrors/tasks/mirror/Suse.yaml +++ b/roles/configure-mirrors/tasks/mirror/Suse.yaml @@ -11,13 +11,15 @@ - name: Install Suse repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "suse/{{ item }}.j2" + src: "suse/{{ zj_repo }}.j2" with_items: - etc/zypp/repos.d/repo-oss.repo + loop_control: + loop_var: zj_repo notify: - Update zypper cache diff --git a/roles/configure-mirrors/tasks/mirror/Ubuntu.yaml b/roles/configure-mirrors/tasks/mirror/Ubuntu.yaml index 8356f53af..cb2ef89b6 100644 --- a/roles/configure-mirrors/tasks/mirror/Ubuntu.yaml +++ b/roles/configure-mirrors/tasks/mirror/Ubuntu.yaml @@ -1,13 +1,15 @@ - name: Install Ubuntu repository files become: yes template: - dest: "/{{ item }}" + dest: "/{{ zj_repo }}" group: root mode: 0644 owner: root - src: "apt/{{ item }}.j2" + src: "apt/{{ zj_repo }}.j2" with_items: - etc/apt/sources.list - etc/apt/apt.conf.d/99unauthenticated + loop_control: + loop_var: zj_repo notify: - Update apt cache diff --git a/roles/ensure-docker/tasks/main.yaml b/roles/ensure-docker/tasks/main.yaml index 01f29b687..deb2bc520 100644 --- a/roles/ensure-docker/tasks/main.yaml +++ b/roles/ensure-docker/tasks/main.yaml @@ -1,7 +1,9 @@ --- - name: Gather variables for each operating system - include_vars: "{{ item }}" + include_vars: "{{ zj_distro_os }}" + loop_control: + loop_var: zj_distro_os with_first_found: - skip: true files: @@ -87,4 +89,4 @@ vars: buildset_registry_docker_user: root when: - - docker_use_buildset_registry | bool \ No newline at end of file + - docker_use_buildset_registry | bool diff --git a/roles/ensure-openshift/tasks/main.yaml b/roles/ensure-openshift/tasks/main.yaml index a36d229ee..04670233a 100644 --- a/roles/ensure-openshift/tasks/main.yaml +++ b/roles/ensure-openshift/tasks/main.yaml @@ -5,10 +5,12 @@ - name: Install requirements yum: - name: "{{ item }}" + name: "{{ zj_package }}" with_items: - origin - docker + loop_control: + loop_var: zj_package become: yes - name: Fix docker start options @@ -46,7 +48,7 @@ become: yes - name: Pull origin images - command: "docker pull docker.io/openshift/{{ item }}:{{ origin_version }}" + command: "docker pull docker.io/openshift/{{ zj_docker_image }}:{{ origin_version }}" with_items: - origin-web-console - origin-docker-registry @@ -54,6 +56,8 @@ - origin-deployer - origin-pod - origin + loop_control: + loop_var: zj_docker_image become: yes - name: Set group ownership of docker socket diff --git a/roles/ensure-output-dirs/tasks/main.yaml b/roles/ensure-output-dirs/tasks/main.yaml index 2c2a82193..40238d529 100644 --- a/roles/ensure-output-dirs/tasks/main.yaml +++ b/roles/ensure-output-dirs/tasks/main.yaml @@ -1,8 +1,10 @@ - name: Ensure Zuul Output directories exist file: - path: "{{ zuul_output_dir }}/{{ item }}" + path: "{{ zuul_output_dir }}/{{ zj_output_dir }}" state: directory with_items: - logs - artifacts - docs + loop_control: + loop_var: zj_output_dir diff --git a/roles/ensure-python/tasks/pyenv.yaml b/roles/ensure-python/tasks/pyenv.yaml index 775271a21..d282a06e7 100644 --- a/roles/ensure-python/tasks/pyenv.yaml +++ b/roles/ensure-python/tasks/pyenv.yaml @@ -1,5 +1,7 @@ - name: Include OS-specific variables - include_vars: "{{ item }}" + include_vars: "{{ zj_distro_os }}" + loop_control: + loop_var: zj_distro_os with_first_found: - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" @@ -35,4 +37,4 @@ become: true command: "{{ ansible_user_dir }}/.pyenv/plugins/python-build/bin/python-build {{ _python_version.stdout }} /usr/local" environment: - CFLAGS: -O2 \ No newline at end of file + CFLAGS: -O2 diff --git a/roles/fetch-output-openshift/tasks/main.yaml b/roles/fetch-output-openshift/tasks/main.yaml index 28ba3b317..54ef176c3 100644 --- a/roles/fetch-output-openshift/tasks/main.yaml +++ b/roles/fetch-output-openshift/tasks/main.yaml @@ -11,17 +11,21 @@ - name: Ensure local output dirs delegate_to: localhost file: - path: "{{ item }}" + path: "{{ zj_output_dir }}" state: directory with_items: - "{{ log_path }}" - "{{ log_path }}/npm" - "{{ zuul.executor.work_root }}/artifacts" - "{{ zuul.executor.work_root }}/docs" + loop_control: + loop_var: zj_output_dir - include_tasks: rsync.yaml - when: item.1.pod is defined + when: zj_pod.1.pod is defined loop: "{{ openshift_pods.items()|list }}" + loop_control: + loop_var: zj_pod run_once: true - name: Remove empty directory diff --git a/roles/fetch-output-openshift/tasks/rsync.yaml b/roles/fetch-output-openshift/tasks/rsync.yaml index 8eb1aa83d..8f64db038 100644 --- a/roles/fetch-output-openshift/tasks/rsync.yaml +++ b/roles/fetch-output-openshift/tasks/rsync.yaml @@ -4,7 +4,7 @@ oc --context "{{ item.1.context }}" --namespace "{{ item.1.namespace }}" rsync -q --progress=false - {{ item.1.pod }}:{{ zj_output.src }}/ + {{ zj_pod.1.pod }}:{{ zj_output.src }}/ {{ zj_output.dst }}/ no_log: "{{ not zuul_log_verbose }}" delegate_to: localhost diff --git a/roles/fetch-output/tasks/main.yaml b/roles/fetch-output/tasks/main.yaml index b827a656d..55f21824e 100644 --- a/roles/fetch-output/tasks/main.yaml +++ b/roles/fetch-output/tasks/main.yaml @@ -11,12 +11,14 @@ - name: Ensure local output dirs delegate_to: localhost file: - path: "{{ item }}" + path: "{{ zj_output_dir }}" state: directory with_items: - "{{ log_path }}" - "{{ zuul.executor.work_root }}/artifacts" - "{{ zuul.executor.work_root }}/docs" + loop_control: + loop_var: zj_output_dir - name: Collect log output synchronize: diff --git a/roles/fetch-puppet-module-output/tasks/main.yaml b/roles/fetch-puppet-module-output/tasks/main.yaml index 089843d31..cb02ef8d7 100644 --- a/roles/fetch-puppet-module-output/tasks/main.yaml +++ b/roles/fetch-puppet-module-output/tasks/main.yaml @@ -7,8 +7,10 @@ - name: Display stat for tarballs stat: - path: "{{ item.path }}" + path: "{{ zj_tarball.path }}" with_items: "{{ result.files }}" + loop_control: + loop_var: zj_tarball - name: Ensure artifacts directory exists file: @@ -20,6 +22,8 @@ synchronize: dest: "{{ zuul.executor.work_root }}/artifacts/" mode: pull - src: "{{ item.path }}" + src: "{{ zj_artifact.path }}" verify_host: true with_items: "{{ result.files }}" + loop_control: + loop_var: zj_artifact diff --git a/roles/fetch-python-sdist-output/tasks/main.yaml b/roles/fetch-python-sdist-output/tasks/main.yaml index 6eb37a283..59a896c98 100644 --- a/roles/fetch-python-sdist-output/tasks/main.yaml +++ b/roles/fetch-python-sdist-output/tasks/main.yaml @@ -7,8 +7,10 @@ - name: Display stat for tarballs and wheels stat: - path: "{{ item.path }}" + path: "{{ zj_tarball_wheel.path }}" with_items: "{{ result.files }}" + loop_control: + loop_var: zj_tarball_wheel - name: Ensure artifacts directory exists file: @@ -20,6 +22,8 @@ synchronize: dest: "{{ zuul.executor.work_root }}/artifacts/" mode: pull - src: "{{ item.path }}" + src: "{{ zj_artifact.path }}" verify_host: true with_items: "{{ result.files }}" + loop_control: + loop_var: zj_artifact diff --git a/roles/fetch-sphinx-tarball/tasks/pdf.yaml b/roles/fetch-sphinx-tarball/tasks/pdf.yaml index 59edd137e..c054b77d4 100644 --- a/roles/fetch-sphinx-tarball/tasks/pdf.yaml +++ b/roles/fetch-sphinx-tarball/tasks/pdf.yaml @@ -4,11 +4,13 @@ - name: Check for PDF file names stat: - path: "{{ zuul_work_dir }}/{{ sphinx_build_dir }}/pdf/{{ item }}" + path: "{{ zuul_work_dir }}/{{ sphinx_build_dir }}/pdf/{{ zj_sphinx_pdf }}" get_checksum: false get_mime: false get_md5: false with_items: "{{ sphinx_pdf_files }}" + loop_control: + loop_var: zj_sphinx_pdf register: pdf_file_stat - name: Set pdf_files_found to default @@ -18,8 +20,10 @@ - name: Check if any file found set_fact: pdf_files_found: true - when: item.stat.exists + when: zj_pdf.stat.exists with_items: "{{ pdf_file_stat.results }}" + loop_control: + loop_var: zj_pdf # Now loop... @@ -40,6 +44,8 @@ src: "{{ item.stat.path }}" verify_host: true with_items: "{{ pdf_file_stat.results }}" + loop_control: + loop_var: zj_pdf when: item.stat.exists @@ -48,9 +54,11 @@ data: zuul: artifacts: - - name: "Docs PDF: {{ item.item }}" - url: "pdf/{{ item.item }}" + - name: "Docs PDF: {{ zj_pdf.item }}" + url: "pdf/{{ zj_pdf.item }}" metadata: type: docs_pdf with_items: "{{ pdf_file_stat.results }}" + loop_control: + loop_var: zj_pdf when: item.stat.exists diff --git a/roles/fetch-subunit-output/tasks/process.yaml b/roles/fetch-subunit-output/tasks/process.yaml index e9becfccc..5858f3776 100644 --- a/roles/fetch-subunit-output/tasks/process.yaml +++ b/roles/fetch-subunit-output/tasks/process.yaml @@ -20,17 +20,21 @@ synchronize: dest: "{{ zuul.executor.log_root }}" mode: pull - src: "{{ item.path }}" + src: "{{ zj_subunit_file.path }}" verify_host: true with_items: "{{ subunit_files.files }}" + loop_control: + loop_var: zj_subunit_file when: not zuul_use_fetch_output - name: Copy test-results copy: dest: "{{ zuul_output_dir }}/logs/" - src: "{{ item.path }}" + src: "{{ zj_subunit_file.path }}" remote_src: true with_items: "{{ subunit_files.files }}" + loop_control: + loop_var: zj_subunit_file when: zuul_use_fetch_output - name: Return artifact to Zuul @@ -42,5 +46,7 @@ url: "testr_results.html" metadata: type: unit_test_report - when: "'testr_results.html' in item.path" + when: "'testr_results.html' in zj_subunit_file.path" with_items: "{{ subunit_files.files }}" + loop_control: + loop_var: zj_subunit_file diff --git a/roles/fetch-tox-output/tasks/main.yaml b/roles/fetch-tox-output/tasks/main.yaml index 9d58a7c51..22dae3e12 100644 --- a/roles/fetch-tox-output/tasks/main.yaml +++ b/roles/fetch-tox-output/tasks/main.yaml @@ -52,9 +52,11 @@ - name: Copy tox logs copy: dest: "{{ zuul_output_dir }}/logs/tox/" - src: "{{ zuul_work_dir }}/.tox/{{ item }}/log/" + src: "{{ zuul_work_dir }}/.tox/{{ zj_testenv }}/log/" remote_src: true - with_items: "{{ envlist }}" + loop: "{{ envlist }}" + loop_control: + loop_var: zj_testenv # some tox runs may not create a virtualenv and thus have # no ./tox/env directory failed_when: false @@ -64,9 +66,11 @@ synchronize: dest: "{{ log_path }}" mode: pull - src: "{{ zuul_work_dir }}/.tox/{{ item }}/log/" + src: "{{ zuul_work_dir }}/.tox/{{ zj_testenv }}/log/" verify_host: true - with_items: "{{ envlist }}" + loop: "{{ envlist }}" + loop_control: + loop_var: zj_testenv # some tox runs may not create a virtualenv and thus have # no ./tox/env directory failed_when: false diff --git a/roles/git-prepare-nodecache/tasks/main.yaml b/roles/git-prepare-nodecache/tasks/main.yaml index e3066f299..0f69804c2 100644 --- a/roles/git-prepare-nodecache/tasks/main.yaml +++ b/roles/git-prepare-nodecache/tasks/main.yaml @@ -1,19 +1,25 @@ - name: Create git parent dirs file: - path: "{{ git_cache_root }}/{{ item.canonical_name | dirname }}" + path: "{{ git_cache_root }}/{{ zj_project.canonical_name | dirname }}" state: directory with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project - name: Copy git repos to {{ git_cache_root }} - command: cp -r "{{ ansible_user_dir }}/{{ item.src_dir }}/.git" - "{{ git_cache_root }}/{{ item.canonical_name }}" + command: cp -r "{{ ansible_user_dir }}/{{ zj_project.src_dir }}/.git" + "{{ git_cache_root }}/{{ zj_project.canonical_name }}" with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project - name: Mark repos as bare command: git config --bool core.bare true args: - chdir: "{{ git_cache_root }}/{{ item.canonical_name }}" + chdir: "{{ git_cache_root }}/{{ zj_project.canonical_name }}" with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project # We don't want git module foo, just set a config value. tags: - skip_ansible_lint @@ -21,8 +27,10 @@ - name: Run git garbage collection command: git gc --prune=now --aggressive args: - chdir: "{{ git_cache_root }}/{{ item.canonical_name }}" + chdir: "{{ git_cache_root }}/{{ zj_project.canonical_name }}" with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project # The ansible git module doesn't support garbage collection. tags: - skip_ansible_lint diff --git a/roles/markdownlint/tasks/main.yaml b/roles/markdownlint/tasks/main.yaml index 9f58e6bdb..612ec0154 100644 --- a/roles/markdownlint/tasks/main.yaml +++ b/roles/markdownlint/tasks/main.yaml @@ -8,9 +8,11 @@ shell: | set -o pipefail set -e - ~/.markdownlint/node_modules/.bin/markdownlint {{ item|relpath(zuul_work_dir) }} 2>&1 | tee -a markdownlint.txt + ~/.markdownlint/node_modules/.bin/markdownlint {{ zj_markdown|relpath(zuul_work_dir) }} 2>&1 | tee -a markdownlint.txt args: chdir: "{{ zuul_work_dir }}" executable: /bin/bash - with_items: "{{ markdown_find.files|map(attribute='path')|list }}" + loop: "{{ markdown_find.files|map(attribute='path')|list }}" + loop_control: + loop_var: zj_markdown changed_when: false diff --git a/roles/mirror-workspace-git-repos/tasks/main.yaml b/roles/mirror-workspace-git-repos/tasks/main.yaml index 39d6202c9..3235d93eb 100644 --- a/roles/mirror-workspace-git-repos/tasks/main.yaml +++ b/roles/mirror-workspace-git-repos/tasks/main.yaml @@ -3,21 +3,25 @@ name: receive.denyCurrentBranch ignore value: ignore scope: local - repo: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + repo: "{{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project - name: Synchronize src repos to workspace directory command: |- {% if ansible_connection == "kubectl" %} - git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ ansible_user_dir }}/{{ item.value.src_dir }}" + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" {% else %} - git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ ansible_user_dir }}/{{ item.value.src_dir }} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ ansible_user_dir }}/{{ zj_project.value.src_dir }} {% endif %} args: - chdir: "{{ zuul.executor.work_root }}/{{ item.value.src_dir }}" + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" environment: GIT_ALLOW_PROTOCOL: ext:ssh with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project delegate_to: localhost # We occasionally see git pushes in the middle of this loop fail then # subsequent pushes for other repos succeed. The entire loop ends up @@ -41,13 +45,15 @@ # Undo the config setting we did above git config --local --unset receive.denyCurrentBranch # checkout the branch matching the branch set up by the executor - git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ item.value.checkout }} + git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ zj_project.value.checkout }} # put out a status line with the current HEAD - echo "{{ item.value.canonical_name }} checked out to:" + echo "{{ zj_project.value.canonical_name }} checked out to:" git log --pretty=oneline -1 args: - chdir: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + chdir: "{{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, # but we prefer the shell above tags: diff --git a/roles/multi-node-bridge/tasks/common.yaml b/roles/multi-node-bridge/tasks/common.yaml index 8fda29f2e..9ca363f7b 100644 --- a/roles/multi-node-bridge/tasks/common.yaml +++ b/roles/multi-node-bridge/tasks/common.yaml @@ -1,9 +1,11 @@ - name: Include OS-specific variables - include_vars: "{{ item }}" + include_vars: "{{ zj_distro_os }}" with_first_found: - "{{ ansible_distribution }}.yaml" - "{{ ansible_os_family }}.yaml" - "default.yaml" + loop_control: + loop_var: zj_distro_os # openvswitch for CentOS is available from the RDO repositories. # We're setting it up manually to prevent centos-release-openstack or rdo-release @@ -27,12 +29,14 @@ become: yes lineinfile: path: /etc/portage/package.use/ovs - line: "{{ item.line }}" + line: "{{ zj_item.line }}" create: yes - with_items: + loop: - { line: 'dev-python/twisted conch # for openvswitch' } - { line: 'sys-apps/util-linux caps # for openvswitch' } - { line: 'net-misc/openvswitch -modules # ovs/gre are staticly built' } + loop_control: + loop_var: zj_item when: - ansible_distribution == 'Gentoo' @@ -63,11 +67,13 @@ - name: Remove RDO repository files become: yes file: - path: "{{ item }}" + path: "{{ zj_rdo }}" state: absent - with_items: + loop: - /tmp/RPM-GPG-KEY-CentOS-SIG-Cloud - /etc/yum.repos.d/zuul-multi-node-bridge-ovs.repo + loop_control: + loop_var: zj_rdo when: - ansible_os_family == "RedHat" - ansible_distribution != "Fedora" diff --git a/roles/multi-node-firewall/tasks/main.yaml b/roles/multi-node-firewall/tasks/main.yaml index 3a43b0a09..578fb2cd0 100644 --- a/roles/multi-node-firewall/tasks/main.yaml +++ b/roles/multi-node-firewall/tasks/main.yaml @@ -32,9 +32,11 @@ action: insert chain: INPUT ip_version: ipv4 - source: "{{ item }}" + source: "{{ zj_ipv4 }}" jump: ACCEPT with_items: "{{ ipv4_addresses }}" + loop_control: + loop_var: zj_ipv4 - name: Set up ipv6 iptables rules become: yes @@ -43,9 +45,11 @@ action: insert chain: INPUT ip_version: ipv6 - source: "{{ item }}" + source: "{{ zj_ipv6 }}" jump: ACCEPT with_items: "{{ ipv6_addresses }}" + loop_control: + loop_var: zj_ipv6 - name: Persist iptables rules include_role: diff --git a/roles/multi-node-hosts-file/tasks/main.yaml b/roles/multi-node-hosts-file/tasks/main.yaml index a54bc5f21..1a630091c 100644 --- a/roles/multi-node-hosts-file/tasks/main.yaml +++ b/roles/multi-node-hosts-file/tasks/main.yaml @@ -14,5 +14,7 @@ dest: /etc/hosts state: present insertafter: EOF - line: "{{ item.value }} {{ item.key }}" + line: "{{ zj_host_address.value }} {{ zj_host_address.key }}" with_dict: "{{ host_addresses }}" + loop_control: + loop_var: zj_host_address diff --git a/roles/multi-node-known-hosts/tasks/main.yaml b/roles/multi-node-known-hosts/tasks/main.yaml index 75aedbc27..077431b40 100644 --- a/roles/multi-node-known-hosts/tasks/main.yaml +++ b/roles/multi-node-known-hosts/tasks/main.yaml @@ -4,14 +4,18 @@ - name: add known_host record for every public key of every other ip, hostname for ansible_user known_hosts: - name: "{{ item.name }}" - key: "{{ item.key }}" + name: "{{ zj_known_host.name }}" + key: "{{ zj_known_host.key }}" with_items: "{{ ansible_facts.all_known_hosts }}" + loop_control: + loop_var: zj_known_host - name: add default known_host record for every user become: true known_hosts: - name: "{{ item.name }}" - key: "{{ item.key }}" + name: "{{ zj_known_host.name }}" + key: "{{ zj_known_host.key }}" path: /etc/ssh/ssh_known_hosts with_items: "{{ ansible_facts.all_known_hosts }}" + loop_control: + loop_var: zj_known_host diff --git a/roles/prepare-workspace-git/tasks/main.yaml b/roles/prepare-workspace-git/tasks/main.yaml index 9c5de2146..9b0715d51 100644 --- a/roles/prepare-workspace-git/tasks/main.yaml +++ b/roles/prepare-workspace-git/tasks/main.yaml @@ -1,7 +1,9 @@ - name: Find locally cached git repos stat: - path: "{{ cached_repos_root }}/{{ item.canonical_name }}" + path: "{{ cached_repos_root }}/{{ zj_project.canonical_name }}" with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project register: cached_repos # We do a bare clone here first so that we skip creating a working copy that @@ -9,28 +11,32 @@ - name: Clone cached repo to workspace shell: | set -e - git clone --bare {{ cached_repos_root }}/{{ item.0.canonical_name }} {{ ansible_user_dir }}/{{ item.0.src_dir }}/.git - cd {{ ansible_user_dir }}/{{ item.0.src_dir }} + git clone --bare {{ cached_repos_root }}/{{ zj_project.0.canonical_name }} {{ ansible_user_dir }}/{{ zj_project.0.src_dir }}/.git + cd {{ ansible_user_dir }}/{{ zj_project.0.src_dir }} git config --local --bool core.bare false args: - creates: "{{ ansible_user_dir }}/{{ item.0.src_dir }}" - when: item.1.stat.exists + creates: "{{ ansible_user_dir }}/{{ zj_project.0.src_dir }}" + when: zj_project.1.stat.exists with_together: - "{{ zuul.projects.values() | list }}" - "{{ cached_repos.results }}" + loop_control: + loop_var: zj_project # ANSIBLE0006: If we use the git module, we get warning # ANSIBLE0004 since we do not give an explicit version tags: - skip_ansible_lint - name: Initialize non-cached repos - command: "git init {{ ansible_user_dir }}/{{ item.0.src_dir }}" + command: "git init {{ ansible_user_dir }}/{{ zj_project.0.src_dir }}" args: - creates: "{{ ansible_user_dir }}/{{ item.0.src_dir }}" - when: not item.1.stat.exists + creates: "{{ ansible_user_dir }}/{{ zj_project.0.src_dir }}" + when: not zj_project.1.stat.exists with_together: - "{{ zuul.projects.values() | list }}" - "{{ cached_repos.results }}" + loop_control: + loop_var: zj_project # ANSIBLE0006: If we use the git module, we get warning # ANSIBLE0004 since we do not give an explicit version tags: @@ -43,8 +49,10 @@ git remote -v | grep origin && git remote rm origin || true git remote add origin file:///dev/null args: - chdir: "{{ ansible_user_dir }}/{{ item.src_dir }}" + chdir: "{{ ansible_user_dir }}/{{ zj_project.src_dir }}" with_items: "{{ zuul.projects.values() | list }}" + loop_control: + loop_var: zj_project # ANSIBLE0006: git remote is not supported by ansible module tags: - skip_ansible_lint diff --git a/roles/sign-artifacts/tasks/main.yaml b/roles/sign-artifacts/tasks/main.yaml index 32c6c083c..42d52cf6f 100644 --- a/roles/sign-artifacts/tasks/main.yaml +++ b/roles/sign-artifacts/tasks/main.yaml @@ -28,8 +28,10 @@ register: artifacts - name: Sign artifacts - command: "gpg --homedir {{ gnupg_tmpdir.path }} --armor --detach-sign {{ item.path }}" + command: "gpg --homedir {{ gnupg_tmpdir.path }} --armor --detach-sign {{ zj_artifact.path }}" with_items: "{{ artifacts.files }}" + loop_control: + loop_var: zj_artifact when: artifacts.matched > 0 - name: Delete keyring directory diff --git a/roles/sphinx/tasks/main.yaml b/roles/sphinx/tasks/main.yaml index a08d428bb..25d94d0bf 100644 --- a/roles/sphinx/tasks/main.yaml +++ b/roles/sphinx/tasks/main.yaml @@ -39,11 +39,13 @@ # Source the activate file so that sphinx subcommands have the correct # paths set. source {{ zuul_work_virtualenv }}/bin/activate - sphinx-build -b {{ item }} \ + sphinx-build -b {{ zj_sphinx_builder }} \ {% if sphinx_warning_is_error %} -W {% endif %} \ - {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }} + {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ zj_sphinx_builder }} chdir: "{{ zuul_work_dir }}" with_items: "{{ sphinx_builders }}" + loop_control: + loop_var: zj_sphinx_builder when: - not use_doc_dir or not sphinx_pbr_autodoc diff --git a/roles/stage-output/tasks/main.yaml b/roles/stage-output/tasks/main.yaml index 29d0bc526..0100e72da 100644 --- a/roles/stage-output/tasks/main.yaml +++ b/roles/stage-output/tasks/main.yaml @@ -1,7 +1,9 @@ - name: Register sources stat: - path: "{{ item.key }}" + path: "{{ zj_source.key }}" with_dict: "{{ zuul_copy_output }}" + loop_control: + loop_var: zj_source register: sources no_log: true @@ -22,7 +24,7 @@ set_fact: extensions_regex: "^(.*)\\.({{ extension_list | join('|') }})$" -# TODO(andreaf) We might want to enforce that item.value is a valid value +# TODO(andreaf) We might want to enforce that zj_source.value is a valid value # in docs, artifacts, logs. Null case already handled. # NOTE(andreaf) Files or folders that start with a '.' are renamed to starting # with an '_' else they would not be visible in the logs folder once uploaded. @@ -31,13 +33,15 @@ # which is handled here). - name: Set source and destination for files and folders set_fact: - source: "{{ item.stat.path }}" - dest: "{{ item.item.value.split('_')[0] }}/{{ item.stat.path|basename|regex_replace('^(\\..*)$', '_\\1') }}{% if item.item.value.endswith('_txt') %}.txt{% endif %}" - type: "{{ item.item.value.split('_')[0] }}" + source: "{{ zj_source.stat.path }}" + dest: "{{ zj_source.item.value.split('_')[0] }}/{{ zj_source.stat.path|basename|regex_replace('^(\\..*)$', '_\\1') }}{% if zj_source.item.value.endswith('_txt') %}.txt{% endif %}" + type: "{{ zj_source.item.value.split('_')[0] }}" with_items: "{{ sources.results }}" + loop_control: + loop_var: zj_source when: - - item.stat.exists - - item.item.value + - zj_source.stat.exists + - zj_source.item.value register: results no_log: true @@ -48,20 +52,24 @@ - name: Ensure target folders exist become: true file: - path: "{{ stage_dir }}/{{ item }}" + path: "{{ stage_dir }}/{{ zj_output_dirs }}" state: directory owner: "{{ ansible_user }}" with_items: - docs - artifacts - logs + loop_control: + loop_var: zj_output_dirs - name: Copy files and folders to staging folder # remote_src copy does not work recursively, synchronise is restricted by # zuul, using command - command: cp -pRL {{ item.source }} {{ stage_dir }}/{{ item.dest }} + command: cp -pRL {{ zj_source.source }} {{ stage_dir }}/{{ zj_source.dest }} become: true with_items: "{{ all_sources }}" + loop_control: + loop_var: zj_source - name: Make all log files readable file: diff --git a/roles/test-mirror-workspace-git-repos/tasks/main.yaml b/roles/test-mirror-workspace-git-repos/tasks/main.yaml index 65e9fec23..37e105b37 100644 --- a/roles/test-mirror-workspace-git-repos/tasks/main.yaml +++ b/roles/test-mirror-workspace-git-repos/tasks/main.yaml @@ -3,13 +3,15 @@ name: receive.denyCurrentBranch ignore value: ignore scope: local - repo: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + repo: "{{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project - name: Synchronize src repos to workspace directory - command: "git push --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ ansible_user_dir }}/{{ item.value.src_dir }}" + command: "git push --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" args: - chdir: "{{ zuul.executor.work_root }}/{{ item.value.src_dir }}" + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" with_dict: "{{ zuul.projects }}" delegate_to: localhost # We occasionally see git pushes in the middle of this loop fail then @@ -23,6 +25,8 @@ # but push is not supported by ansible git module. tags: - skip_ansible_lint + loop_control: + loop_var: zj_project # Do this as a multi-line shell so that we can do the loop once - name: Update remote repository state correctly @@ -32,13 +36,15 @@ # Undo the config setting we did above git config --local --unset receive.denyCurrentBranch # checkout the branch matching the branch set up by the executor - git checkout {{ item.value.checkout }} + git checkout {{ zj_project.value.checkout }} # put out a status line with the current HEAD - echo "{{ item.value.canonical_name }} checked out to:" + echo "{{ zj_project.value.canonical_name }} checked out to:" git log --pretty=oneline -1 args: - chdir: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + chdir: "{{ ansible_user_dir }}/{{ zj_project.value.src_dir }}" with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, # but we prefer the shell above tags: diff --git a/roles/upload-logs/tasks/main.yaml b/roles/upload-logs/tasks/main.yaml index e692f995a..26810af62 100644 --- a/roles/upload-logs/tasks/main.yaml +++ b/roles/upload-logs/tasks/main.yaml @@ -36,30 +36,36 @@ - name: gzip console log and json output delegate_to: localhost archive: - path: "{{ zuul.executor.log_root }}/{{ item }}" + path: "{{ zuul.executor.log_root }}/{{ zj_log }}" with_items: - job-output.txt - job-output.json + loop_control: + loop_var: zj_log when: zuul_log_compress | bool - name: Upload compressed console log and json output synchronize: - src: "{{ zuul.executor.log_root }}/{{ item }}.gz" - dest: "{{ zuul_logserver_root }}/{{ zuul_log_path }}/{{ item }}.gz" + src: "{{ zuul.executor.log_root }}/{{ zj_log }}.gz" + dest: "{{ zuul_logserver_root }}/{{ zuul_log_path }}/{{ zj_log }}.gz" verify_host: true with_items: - job-output.txt - job-output.json + loop_control: + loop_var: zj_log when: zuul_log_compress | bool - name: Upload console log and json output synchronize: - src: "{{ zuul.executor.log_root }}/{{ item }}" - dest: "{{ zuul_logserver_root }}/{{ zuul_log_path }}/{{ item }}" + src: "{{ zuul.executor.log_root }}/{{ zj_log }}" + dest: "{{ zuul_logserver_root }}/{{ zuul_log_path }}/{{ zj_log }}" verify_host: true with_items: - job-output.txt - job-output.json + loop_control: + loop_var: zj_log when: not zuul_log_compress | bool - name: Return log URL to Zuul diff --git a/roles/upload-pypi/tasks/main.yaml b/roles/upload-pypi/tasks/main.yaml index 671daeac7..7712e56b3 100644 --- a/roles/upload-pypi/tasks/main.yaml +++ b/roles/upload-pypi/tasks/main.yaml @@ -22,13 +22,17 @@ when: found_wheels.files == [] - name: Register packages on the PyPI server (via wheels) - command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ item.path }}" + command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ zj_wheel.path }}" with_items: "{{ found_wheels.files }}" + loop_control: + loop_var: zj_wheel when: pypi_register_first - name: Upload wheel with twine before tarballs - command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}" + command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ zj_wheel.path }}" with_items: "{{ found_wheels.files }}" + loop_control: + loop_var: zj_wheel - name: Find tarballs to upload find: @@ -42,15 +46,19 @@ when: found_tarballs.files == [] - name: Register packages on the PyPI server (via tarballs) - command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ item.path }}" + command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ zj_tarball.path }}" with_items: "{{ found_tarballs.files }}" + loop_control: + loop_var: zj_tarball when: - pypi_register_first - found_wheels.files == [] - name: Upload tarballs with twine - command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}" + command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ zj_tarball.path }}" with_items: "{{ found_tarballs.files }}" + loop_control: + loop_var: zj_tarball - name: Delete .pypirc configuration file file: diff --git a/roles/use-buildset-registry/tasks/main.yaml b/roles/use-buildset-registry/tasks/main.yaml index 9b578643a..9db6cd60f 100644 --- a/roles/use-buildset-registry/tasks/main.yaml +++ b/roles/use-buildset-registry/tasks/main.yaml @@ -1,11 +1,13 @@ - name: Include OS-specific variables - include_vars: "{{ item }}" + include_vars: "{{ zj_distro_os }}" with_first_found: - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml" - "{{ ansible_distribution }}.{{ ansible_architecture }}.yaml" - "{{ ansible_distribution }}.yaml" - "{{ ansible_os_family }}.yaml" - "default.yaml" + loop_control: + loop_var: zj_distro_os # Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag # so we set up /etc/hosts with a registry alias name to support ipv6 and 4.