zuul/zuul-jobs
Code Issues Proposed changes

ansible-lint: use matchplay instead of matchtask

Browse Source 
For some reason matchtask doesn't match includes, matchplay does so use
that instead.

Change-Id: I040f7f3394503e92d06c05e8ff671a43b14baebc
This commit is contained in:
Albin Vass 2020-05-01 20:51:18 +02:00
parent a7b4e5e906
commit bee0c6ae2f
34 changed files with 224 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.rules/ZuulJobsNamespaceLoopVar.py
View File

@ -15,9 +15,10 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\
tags = {'zuul-jobs-namespace-loop-var'}
def matchtask(self, file, task):
if file.get('type') != 'tasks':
return False
def matchplay(self, file, task):
results = []
if file.get('type') not in ('tasks', 'handlers'):
return results
has_loop = 'loop' in task
for key in task.keys():
@ -26,10 +27,10 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\
if has_loop:
if 'loop_control' not in task:
return True
results.append(("", self.shortdesc))
elif 'loop_var' not in task.get('loop_control'):
return True
results.append(("", self.shortdesc))
elif not task.get('loop_control')\
.get('loop_var').startswith('zj_'):
return True
return False
results.append(("", self.shortdesc))
return results

34
roles/build-container-image/tasks/build.yaml
View File

@ -1,6 +1,6 @@
- name: Check sibling directory
stat:
path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
register: _dot_zuul_siblings
# This should have been cleaned up; multiple builds may specify
@ -11,42 +11,42 @@
- name: Create sibling source directory
file:
path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
state: directory
mode: 0755
when: item.siblings is defined
when: zj_image.siblings is defined
- name: Copy sibling source directories
command:
cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
chdir: '~/src'
loop: '{{ item.siblings }}'
loop: '{{ zj_image.siblings }}'
loop_control:
loop_var: zj_sibling
when: item.siblings is defined
when: zj_image.siblings is defined
- name: Set container filename arg
set_fact:
containerfile: "{{ item.container_filename|default(container_filename)|default('') }}"
containerfile: "{{ zj_image.container_filename|default(container_filename)|default('') }}"
- name: Build a container image
command: >-
{{ container_command }} build {{ item.path | default('.') }} {% if containerfile %}-f {{ containerfile }}{% endif %}
{% if item.target | default(false) -%}
--target {{ item.target }}
{{ container_command }} build {{ zj_image.path | default('.') }} {% if containerfile %}-f {{ containerfile }}{% endif %}
{% if zj_image.target | default(false) -%}
--target {{ zj_image.target }}
{% endif -%}
{% for build_arg in item.build_args | default([]) -%}
{% for build_arg in zj_image.build_args | default([]) -%}
--build-arg {{ build_arg }}
{% endfor -%}
{% if items.siblings | default(false) -%}
--build-arg "ZUUL_SIBLINGS={{ item.siblings | join(' ') }}"
{% if zj_images.siblings | default(false) -%}
--build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
{% endif -%}
{% for tag in item.tags | default(['latest']) -%}
--tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }}
--tag {{ item.repository }}:{{ tag }}
{% for tag in zj_image.tags | default(['latest']) -%}
--tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
--tag {{ zj_image.repository }}:{{ tag }}
{% endfor -%}
args:
chdir: "{{ zuul_work_dir }}/{{ item.context }}"
chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
- name: Cleanup sibling source directory
file:

7
roles/build-container-image/tasks/main.yaml
View File

@ -17,6 +17,8 @@
- name: Build container images
include_tasks: build.yaml
loop: "{{ container_images }}"
loop_control:
loop_var: zj_image
# Docker, and therefore skopeo and podman, don't understand docker
# push [1234:5678::]:5000/image/path:tag so we set up /etc/hosts with
@ -30,18 +32,21 @@
line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
insertafter: EOF
when: buildset_registry is defined and buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using ip
set_fact:
buildset_registry_alias: zuul-jobs.buildset-registry
when: buildset_registry is defined and buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using name
set_fact:
buildset_registry_alias: "{{ buildset_registry.host }}"
when: buildset_registry is defined and not ( buildset_registry.host | ipaddr )
# Push each image.
- name: Push image to buildset registry
when: buildset_registry is defined
include_tasks: push.yaml
loop: "{{ container_images }}"
loop_control:
loop_var: image
loop_var: zj_image

9
roles/build-container-image/tasks/push.yaml
View File

@ -1,12 +1,13 @@
- name: Tag image for buildset registry
command: >-
{{ container_command }} tag {{ image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
loop: "{{ image.tags | default(['latest']) }}"
{{ container_command }} tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
loop: "{{ zj_image.tags | default(['latest']) }}"
loop_control:
loop_var: zj_image_tag
- name: Push tag to buildset registry
command: >-
{{ container_command }} push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
loop: "{{ image.tags | default(['latest']) }}"
{{ container_command }} push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
loop: "{{ zj_image.tags | default(['latest']) }}"
loop_control:
loop_var: zj_image_tag

2
roles/build-docker-image/tasks/main.yaml
View File

@ -31,10 +31,12 @@
line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
insertafter: EOF
when: buildset_registry is defined and buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using ip
set_fact:
buildset_registry_alias: zuul-jobs.buildset-registry
when: buildset_registry is defined and buildset_registry.host | ipaddr
- name: Set buildset_registry alias variable when using name
set_fact:
buildset_registry_alias: "{{ buildset_registry.host }}"

8
roles/configure-mirrors/handlers/main.yaml
View File

@ -9,20 +9,24 @@
- name: Update yum/dnf cache
become: yes
command: "{{ item }}"
command: "{{ zj_command }}"
args:
warn: false
with_items:
- "{{ ansible_pkg_mgr }} clean all"
- "{{ ansible_pkg_mgr }} makecache -v"
loop_control:
loop_var: zj_command
# verbose is needed in order to make it possible to debug potential failures
- name: Update zypper cache
become: yes
command: "{{ item }}"
command: "{{ zj_command }}"
with_items:
- zypper clean
- zypper refresh
loop_control:
loop_var: zj_command
- name: Update Gentoo cache
become: yes

13
roles/ensure-bazel/tasks/install-bazel.yaml
View File

@ -9,7 +9,8 @@
return_content: true
register: bazel_installer_checksum
- debug: msg="Checksum is {{ bazel_installer_checksum.content.split(' ')[0] }}"
- debug:
msg: "Checksum is {{ bazel_installer_checksum.content.split(' ')[0] }}"
- name: Download bazel installer
get_url:
@ -18,12 +19,16 @@
mode: 0755
checksum: "sha256:{{ bazel_installer_checksum.content.split(' ')[0] }}"
- debug: msg="Distribution is {{ ansible_distribution }}"
- debug: msg="OS family is {{ ansible_os_family }}"
- debug:
msg: "Distribution is {{ ansible_distribution }}"
- debug:
msg: "OS family is {{ ansible_os_family }}"
- name: Install bazel and platform-specific dependencies
include: "{{ item }}"
include: "{{ zj_distro_os }}"
with_first_found:
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- "default.yaml"
loop_control:
loop_var: zj_distro_os

4
roles/ensure-pip/tasks/main.yaml
View File

@ -27,7 +27,7 @@
failed_when: false
- name: Install pip from packages
include: "{{ item }}"
include: "{{ zj_distro_os }}"
with_first_found:
- "{{ ansible_distribution_release }}.yaml"
- "{{ ansible_distribution }}.yaml"
@ -36,6 +36,8 @@
when:
- ensure_pip_from_packages
- pip_preinstalled.rc != 0
loop_control:
loop_var: zj_distro_os
- name: Install pip from source
include: source.yaml

4
roles/ensure-podman/tasks/main.yaml
View File

@ -1,11 +1,13 @@
- name: Find distribution installation
include: "{{ item }}"
include: "{{ zj_distro_os }}"
static: no
with_first_found:
- "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- "default.yaml"
loop_control:
loop_var: zj_distro_os
- name: Validate podman engine
when: ensure_podman_validate

4
roles/ensure-virtualenv/tasks/main.yaml
View File

@ -7,12 +7,14 @@
failed_when: false
- name: Install virtualenv package
include: "{{ item }}"
include: "{{ zj_distro_os }}"
with_first_found:
- "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
- "{{ ansible_distribution_release }}.yaml"
- "{{ ansible_distribution }}.yaml"
- "{{ ansible_os_family }}.yaml"
- "default.yaml"
loop_control:
loop_var: zj_distro_os
when:
- virtualenv_preinstalled.rc != 0

4
roles/persistent-firewall/tasks/main.yaml
View File

@ -19,10 +19,12 @@
- skip_ansible_lint
- name: Configure persistent iptables rules
include: "{{ item }}"
include: "{{ zj_distro_os }}"
static: no
with_first_found:
- "persist/{{ ansible_distribution }}_{{ ansible_distribution_release }}.yaml"
- "persist/{{ ansible_distribution }}.yaml"
- "persist/{{ ansible_os_family }}.yaml"
- "persist/default.yaml"
loop_control:
loop_var: zj_distro_os

4
roles/phoronix-combine-results/tasks/fetch-result.yaml
View File

@ -1,10 +1,10 @@
---
- name: Create result dir
file:
path: "{{ ansible_env.HOME }}/test-results/{{ item.url.rstrip('/')|basename }}"
path: "{{ ansible_env.HOME }}/test-results/{{ zj_artifact.url.rstrip('/')|basename }}"
state: directory
- name: Fetch artifact
get_url:
url: "{{ item.url }}/composite.xml"
url: "{{ zj_artifact.url }}/composite.xml"
dest: "{{ ansible_env.HOME }}/test-results/{{ item.url.rstrip('/')|basename }}/composite.xml"

8
roles/phoronix-combine-results/tasks/main.yaml
View File

@ -2,10 +2,12 @@
- name: Fetch previous result
include_tasks: fetch-result.yaml
when:
- item.metadata is defined
- item.metadata.type is defined
- item.metadata.type == "phoronix_result"
- zj_artifact.metadata is defined
- zj_artifact.metadata.type is defined
- zj_artifact.metadata.type == "phoronix_result"
loop: "{{ zuul.artifacts }}"
loop_control:
loop_var: zj_artifact
- name: List previous result name
command: "ls {{ ansible_env.HOME }}/test-results/"

4
roles/prepare-workspace-openshift/tasks/main.yaml
View File

@ -1,4 +1,6 @@
---
- include_tasks: rsync.yaml
when: item.1.pod is defined
when: zj_pod.1.pod is defined
loop: "{{ openshift_pods.items()|list }}"
loop_control:
loop_var: zj_pod

12
roles/prepare-workspace-openshift/tasks/rsync.yaml
View File

@ -1,17 +1,17 @@
---
- name: Create src directory
command: >
oc --context "{{ item.1.context }}"
--namespace "{{ item.1.namespace }}"
exec {{ item.1.pod }} mkdir src
oc --context "{{ zj_pod.1.context }}"
--namespace "{{ zj_pod.1.namespace }}"
exec {{ zj_pod.1.pod }} mkdir src
delegate_to: localhost
- name: Copy src repos to the pod
command: >
oc --context "{{ item.1.context }}"
--namespace "{{ item.1.namespace }}"
oc --context "{{ zj_pod.1.context }}"
--namespace "{{ zj_pod.1.namespace }}"
rsync -q --progress=false
{{ zuul.executor.src_root }}/
{{ item.1.pod }}:src/
{{ zj_pod.1.pod }}:src/
no_log: true
delegate_to: localhost

6
roles/promote-docker-image/tasks/promote-retag-inner.yaml
View File

@ -1,7 +1,7 @@
- name: Get manifest
no_log: true
uri:
url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/change_{{ zuul.change }}_{{ image_tag }}"
url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/change_{{ zuul.change }}_{{ zj_image_tag }}"
status_code: 200
headers:
Accept: "application/vnd.docker.distribution.manifest.v2+json"
@ -11,7 +11,7 @@
- name: Put manifest
no_log: true
uri:
url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/{{ image_tag }}"
url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/{{ zj_image_tag }}"
method: PUT
status_code: 201
body: "{{ manifest.content | string }}"
@ -21,7 +21,7 @@
- name: Delete the current change tag
no_log: true
uri:
url: "https://hub.docker.com/v2/repositories/{{ zj_image.repository }}/tags/change_{{ zuul.change }}_{{ image_tag }}/"
url: "https://hub.docker.com/v2/repositories/{{ zj_image.repository }}/tags/change_{{ zuul.change }}_{{ zj_image_tag }}/"
method: DELETE
status_code: [200, 204]
headers:

3
roles/promote-docker-image/tasks/promote-retag.yaml
View File

@ -6,8 +6,9 @@
password: "{{ docker_credentials.password | trim }}"
force_basic_auth: true
register: token
- name: Retag image
loop: "{{ zj_image.tags | default(['latest']) }}"
loop_control:
loop_var: image_tag
loop_var: zj_image_tag
include_tasks: promote-retag-inner.yaml

4
roles/stage-output/tasks/main.yaml
View File

@ -89,8 +89,10 @@
register: log_files_to_rename
- name: Rename log files that match extension_list
shell: "mv {{ item.path }} {{ item.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}"
shell: "mv {{ zj_log_file.path }} {{ zj_log_file.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}"
with_items: "{{ log_files_to_rename.files }}"
loop_control:
loop_var: zj_log_file
args:
chdir: "{{ stage_dir }}/logs"
tags:

4
roles/upload-docker-image/tasks/main.yaml
View File

@ -7,11 +7,13 @@
loop_var: zj_image
fail:
msg: "{{ zj_image.repository }} not permitted by {{ docker_credentials.repository }}"
- name: Log in to dockerhub
command: "docker login -u {{ docker_credentials.username }} -p {{ docker_credentials.password }}"
no_log: true
- name: Upload image to dockerhub
loop: "{{ docker_images }}"
loop_control:
loop_var: image
loop_var: zj_image
include_tasks: push.yaml

4
roles/upload-docker-image/tasks/push.yaml
View File

@ -1,6 +1,6 @@
- name: Upload tag to dockerhub
command: "docker push {{ image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}"
loop: "{{ image.tags | default(['latest']) }}"
command: "docker push {{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}"
loop: "{{ zj_image.tags | default(['latest']) }}"
loop_control:
loop_var: zj_image_tag
register: result

8
test-playbooks/ansible-lint-rules/roles/handlers-loopvar-loop/handlers/main.yaml Normal file
View File

@ -0,0 +1,8 @@
- debug:
msg: "I should pass: {{ zj_item }}"
loop:
- 1
- 2
loop_control:
loop_var: zj_item

8
test-playbooks/ansible-lint-rules/roles/handlers-loopvar-with/handlers/main.yaml Normal file
View File

@ -0,0 +1,8 @@
- debug:
msg: "I should pass: {{ zj_item }}"
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

5
test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-loop/handlers/main.yaml Normal file
View File

@ -0,0 +1,5 @@
- debug:
msg: "I should fail: {{ item }}"
loop:
- 1
- 2

5
test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-with/handlers/main.yaml Normal file
View File

@ -0,0 +1,5 @@
- debug:
msg: "I should fail: {{ item }}"
with_list:
- 1
- 2

7
test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-loop/tasks/main.yaml Normal file
View File

@ -0,0 +1,7 @@
- include: "{{ zj_item }}.yaml"
loop:
- 1
- 2
loop_control:
loop_var: zj_item

6
test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-with/tasks/main.yaml Normal file
View File

@ -0,0 +1,6 @@
- include: "{{ zj_item }}.yaml"
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

4
test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
- include: "{{ item }}.yaml"
loop:
- 1
- 2

4
test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-with/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
- include: "{{ item }}.yaml"
with_list:
- 1
- 2

8
test-playbooks/ansible-lint-rules/roles/tasks-loopvar-loop/tasks/main.yaml Normal file
View File

@ -0,0 +1,8 @@
- debug:
msg: "I should pass: {{ zj_item }} "
loop:
- 1
- 2
loop_control:
loop_var: zj_item

7
test-playbooks/ansible-lint-rules/roles/tasks-loopvar-with/tasks/main.yaml Normal file
View File

@ -0,0 +1,7 @@
- debug:
msg: "I should pass: {{ zj_item }} "
with_list:
- 1
- 2
loop_control:
loop_var: zj_item

5
test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-loop/tasks/main.yaml Normal file
View File

@ -0,0 +1,5 @@
- debug:
msg: "I should error: {{ item }} "
loop:
- 1
- 2

5
test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-with/tasks/main.yaml Normal file
View File

@ -0,0 +1,5 @@
- debug:
msg: "I should error: {{ item }} "
with_list:
- 1
- 2

42
test-playbooks/ansible-lint-rules/run.yaml Normal file
View File

@ -0,0 +1,42 @@
- hosts: all
roles:
- ensure-pip
tasks:
- name: Create tempdir for ansible-lint venv
tempfile:
state: directory
register: ansible_lint_tempdir
- name: Install test-requirements.txt
pip:
requirements: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}/test-requirements.txt"
virtualenv: "{{ ansible_lint_tempdir.path }}"
- name: Make sure ansible-lint is installed
command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint --version"
- name: Make sure missing loopvars fail linting
command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
args:
chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
register: ansible_lint
failed_when: ansible_lint.rc == 0
loop:
- tasks-missing-loopvar-with
- tasks-missing-loopvar-loop
- tasks-include-missing-loopvar-with
- tasks-include-missing-loopvar-loop
- handlers-missing-loopvar-with
- handlers-missing-loopvar-loop
- name: Make sure valid configuration passes linting
command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
args:
chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
loop:
- tasks-loopvar-with
- tasks-loopvar-loop
- tasks-include-loopvar-with
- tasks-include-loopvar-loop
- handlers-loopvar-with
- handlers-loopvar-loop

16
zuul-tests.d/ansible-list-rules.yaml Normal file
View File

@ -0,0 +1,16 @@
- job:
name: zuul-jobs-test-ansible-lint-rules
description: |
Test custom ansible-lint rules in zuul-jobs
run: test-playbooks/ansible-lint-rules/run.yaml
files:
- ^/.rules/.*
- ^test-playbooks/ansible-lint-rules/.*
- ^\.ansible-lint
- project:
check:
jobs: &id001
- zuul-jobs-test-ansible-lint-rules
gate:
jobs: *id001