From 664b016fa561afea24ef97d534c83ff45ea02899 Mon Sep 17 00:00:00 2001 From: Albin Vass Date: Tue, 5 May 2020 22:14:39 +0200 Subject: [PATCH] Add upload-artifactory role Adds a role to let users define a manifest of artifacts located in zuul-output/artifacts that should be uploaded to pre defined artifactory instances. Change-Id: I00dc0302e85ce59b3808f6e62e2bcdadf2e41fde --- doc/source/general-roles.rst | 1 + roles/upload-artifactory/README.rst | 88 ++++++++++++++++++++ roles/upload-artifactory/defaults/main.yaml | 1 + roles/upload-artifactory/tasks/main.yaml | 13 +++ roles/upload-artifactory/tasks/upload.yaml | 69 ++++++++++++++++ test-playbooks/artifactory/cleanup.yaml | 4 + test-playbooks/artifactory/run.yaml | 90 +++++++++++++++++++++ zuul-tests.d/artifactory-roles.yaml | 15 ++++ 8 files changed, 281 insertions(+) create mode 100644 roles/upload-artifactory/README.rst create mode 100644 roles/upload-artifactory/defaults/main.yaml create mode 100644 roles/upload-artifactory/tasks/main.yaml create mode 100644 roles/upload-artifactory/tasks/upload.yaml create mode 100644 test-playbooks/artifactory/cleanup.yaml create mode 100644 test-playbooks/artifactory/run.yaml create mode 100644 zuul-tests.d/artifactory-roles.yaml diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst index 9e8ceacc8..e8153efbc 100644 --- a/doc/source/general-roles.rst +++ b/doc/source/general-roles.rst @@ -42,6 +42,7 @@ General Purpose Roles .. zuul:autorole:: start-zuul-console .. zuul:autorole:: test-setup .. zuul:autorole:: trigger-readthedocs +.. zuul:autorole:: upload-artifactory .. zuul:autorole:: upload-git-mirror .. zuul:autorole:: validate-dco-license .. zuul:autorole:: validate-host diff --git a/roles/upload-artifactory/README.rst b/roles/upload-artifactory/README.rst new file mode 100644 index 000000000..94a2f2658 --- /dev/null +++ b/roles/upload-artifactory/README.rst @@ -0,0 +1,88 @@ +Upload artifacts specified from the executor to artifactory. + +.. note:: + This role uses the ``src`` function of the ``uri`` module + introduced in Ansible 2.7 therefore any ansible version + lower than that is not supported. + +**Role Variables** + +.. zuul:rolevar:: upload_artifactory_instances + + Complex argument that contains the information about credentials, + fqdn and name. This argument is expected to come from a secret. + + .. zuul:rolevar:: upload_artifactory_instances..user + + User for authenticating. + + .. zuul:rolevar:: upload_artifactory_instances..password + + Password for authenticating. + Has a lower precedense than ``api_key``. + + .. zuul:rolevar:: upload_artifactory_instances..api_key + + API key for authenticating. + Has a higher precedense than ``password``. + + .. zuul:rolevar:: upload_artifactory_instances..fqdn + + Fully qualified domain name to the instance. + + .. zuul:rolevar:: upload_artifactory_instances..transport + :default: https + + Set to ``http`` if the instance does not support https. + + .. zuul:rolevar:: force_basic_auth + :default: false + + Set to ``true`` if the instance requires basic auth to be used. + +.. zuul:rolevar:: artifacts + + Variable that contains a manifest of the artifacts that should be + uploaded to a specific instance of artifactory. This is expected to + be set during the build as a cached fact. + + .. code-block:: yaml + + artifacts: + - name: tarball + src: artifact.tar.gz + dest: /destination/to/put/artifact/artifact.tar.gz + instance: artifact-server1 + headers: + Content-Type: application/gzip + + The attributes available on an artifact are the following. + + .. zuul:rolevar:: name + + Name of the artifact. + This will be displayed in the build page. + + .. zuul:rolevar:: src + + Path relative to ``{{ zuul.executor.work_root }}/artifacts/``. + + .. zuul:rolevar:: dest + + Destination where the artifact should be put in. + + .. zuul:rolevar:: instance + + Artifactory instance to place the artiface in, this is to + choose which entry in :attr:`upload_artifactory_instances` to upload + the artifact to. + + .. zuul:rolevar:: headers + + Any headers that should be passed to ansibles uri module + when uploading. + + .. zuul:rolevar:: metadata + + Any metadata that should be returned to Zuul together with the + artifact link. diff --git a/roles/upload-artifactory/defaults/main.yaml b/roles/upload-artifactory/defaults/main.yaml new file mode 100644 index 000000000..19ae6d198 --- /dev/null +++ b/roles/upload-artifactory/defaults/main.yaml @@ -0,0 +1 @@ +upload_artifactory_workdir: "{{ zuul.executor.work_root }}" diff --git a/roles/upload-artifactory/tasks/main.yaml b/roles/upload-artifactory/tasks/main.yaml new file mode 100644 index 000000000..d031d93c6 --- /dev/null +++ b/roles/upload-artifactory/tasks/main.yaml @@ -0,0 +1,13 @@ +- name: Fail if run by an unsupported ansible version + fail: + msg: This role can only be used by ansible version 2.7 and greater. + when: ansible_version.full < "2.7.0" + +- name: Upload artifacts + include_tasks: upload.yaml + loop: "{{ upload_artifactory_manifest.artifacts }}" + loop_control: + loop_var: zj_artifact + when: + - upload_artifactory_manifest is defined + - "'artifacts' in upload_artifactory_manifest" diff --git a/roles/upload-artifactory/tasks/upload.yaml b/roles/upload-artifactory/tasks/upload.yaml new file mode 100644 index 000000000..4fe7f39a9 --- /dev/null +++ b/roles/upload-artifactory/tasks/upload.yaml @@ -0,0 +1,69 @@ +- name: Make sure artifact exists on the executor + stat: + path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\ + /artifacts/{{ zj_artifact.src }}" + register: artifact + delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + failed_when: not artifact.stat.exists + +- name: Get sha256 checksum + stat: + path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\ + /artifacts/{{ zj_artifact.src }}" + checksum_algorithm: sha256 + delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + register: artifact_sha256_checksum + +- name: Set request header fact + set_fact: + request_header: + X-Checksum-Sha1: "{{ artifact.stat.checksum }}" + X-Checksum-Sha256: "{{ artifact_sha256_checksum.stat.checksum }}" + +- name: Add artifact headers + set_fact: + request_header: "{{ request_header | combine(zj_artifact.headers|default({})) }}" + +- name: Add api key to header + when: "'api_key' in upload_artifactory_instance[zj_artifact.instance]" + no_log: true + set_fact: + request_header: "{{ request_header | + combine({'X-JFrog-Art-Api': upload_artifactory_instance[zj_artifact.instance].api_key}) }}" + +- name: Set artifactory password + no_log: true + set_fact: + _artifactory_password: "{{ upload_artifactory_instance[zj_artifact.instance].password }}" + when: + - "'api_key' not in upload_artifactory_instance[zj_artifact.instance]" + - "'password' in upload_artifactory_instance[zj_artifact.instance]" + +- name: Set artifactory url fact + set_fact: + _artifactory_url: "{{ upload_artifactory_instance[zj_artifact.instance].transport | default('https') }}://\ + {{ upload_artifactory_instance[zj_artifact.instance].fqdn }}\ + /artifactory/{{ zj_artifact.dest }}" + +- name: Upload artifact + uri: + user: "{{ upload_artifactory_instance[zj_artifact.instance].user }}" + password: "{{ _artifactory_password | default(omit) }}" + url: "{{ _artifactory_url }}" + src: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}/artifacts/{{ zj_artifact.src }}" + headers: "{{ request_header }}" + status_code: 201 + method: PUT + force_basic_auth: "{{ upload_artifactory_instance[zj_artifact.instance].force_basic_auth | + default(false) | bool }}" + remote_src: yes # To not unecessarily copy artifact to a tempfile + delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + +- name: Add artifact link to build page + zuul_return: + data: + zuul: + artifacts: + - name: "{{ zj_artifact.name }}" + url: "{{ _artifactory_url }}" + metadata: "{{ zj_artifact.metadata | default(omit) }}" diff --git a/test-playbooks/artifactory/cleanup.yaml b/test-playbooks/artifactory/cleanup.yaml new file mode 100644 index 000000000..086b2b6c8 --- /dev/null +++ b/test-playbooks/artifactory/cleanup.yaml @@ -0,0 +1,4 @@ +- hosts: all + tasks: + - name: Remove artifactory container + command: docker rm -f "{{ zuul.build }}" diff --git a/test-playbooks/artifactory/run.yaml b/test-playbooks/artifactory/run.yaml new file mode 100644 index 000000000..3fb910be6 --- /dev/null +++ b/test-playbooks/artifactory/run.yaml @@ -0,0 +1,90 @@ +- hosts: all + tasks: + - name: Ensure docker is installed + include_role: + name: ensure-docker + + - name: Start artifactory in a container + command: >- + docker run -d + -p 8081:8081 -p 8082:8082 + --name {{ zuul.build }} + docker.bintray.io/jfrog/artifactory-oss:latest + + - name: Wait for artifactory to start + uri: + url: http://localhost:8082/artifactory/api/system/ping + method: GET + register: artifactory_status + until: artifactory_status.status == 200 + retries: 12 + delay: 10 + + - name: Create a generic repository in artifactory + uri: + url: http://localhost:8082/artifactory/api/system/configuration + user: admin + password: password + force_basic_auth: true + method: PATCH + body: | + localRepositories: + generic-repository: + type: generic + headers: + Content-Type: application/yaml + + - name: Create an api key for the admin user + uri: + url: http://localhost:8082/artifactory/api/security/apiKey + user: admin + password: password + status_code: 201 + return_content: true + method: POST + register: artifactory_api_key + + - name: Set artifactory instances fact + set_fact: + cacheable: true + upload_artifactory_instance: + localhost_password: + fqdn: localhost:8081 + transport: http + user: admin + password: password + force_basic_auth: true + localhost_api_key: + fqdn: localhost:8081 + transport: http + user: admin + api_key: "{{ (artifactory_api_key.content | from_json)['apiKey'] }}" + +- hosts: all + vars: + # Since we're testing a role that normally requires a + # trusted context flip the delegate_to so we execute on the + # remote instead. Also set the working directory to something + # that is known to exist on the remote. + _undocumented_test_worker_node_: "{{ inventory_hostname }}" + _undocumented_test_work_dir_: "{{ ansible_user_dir }}/zuul-output" + pre_tasks: + - name: Write a file with some content to artifacts directory + copy: + content: | + First file + dest: "{{ ansible_user_dir }}/zuul-output/artifacts/test-file.txt" + - name: Set upload_artifactory_manifest fact + set_fact: + upload_artifactory_manifest: + artifacts: + - name: test-file.txt + src: test-file.txt + dest: generic-repository/path/to/dest/test-file-password.txt + instance: localhost_password + - name: test-file.txt + src: test-file.txt + dest: generic-repository/path/to/dest/test-file-api-key.txt + instance: localhost_api_key + roles: + - upload-artifactory diff --git a/zuul-tests.d/artifactory-roles.yaml b/zuul-tests.d/artifactory-roles.yaml new file mode 100644 index 000000000..6ae2910c4 --- /dev/null +++ b/zuul-tests.d/artifactory-roles.yaml @@ -0,0 +1,15 @@ +- job: + name: zuul-jobs-test-upload-artifactory + description: Test the upload-artifactory role + files: + - roles/upload-artifactory/.* + - test-playbooks/artifactory/.* + run: test-playbooks/artifactory/run.yaml + cleanup-run: test-playbooks/artifactory/cleanup.yaml + +- project: + check: + jobs: &id001 + - zuul-jobs-test-upload-artifactory + gate: + jobs: *id001