Add upload-artifactory role

Adds a role to let users define a manifest of artifacts located
in zuul-output/artifacts that should be uploaded to pre defined
artifactory instances.

Change-Id: I00dc0302e85ce59b3808f6e62e2bcdadf2e41fde

doc/source/general-roles.rst

@@ -42,6 +42,7 @@ General Purpose Roles
 .. zuul:autorole:: start-zuul-console
 .. zuul:autorole:: test-setup
 .. zuul:autorole:: trigger-readthedocs
+.. zuul:autorole:: upload-artifactory
 .. zuul:autorole:: upload-git-mirror
 .. zuul:autorole:: validate-dco-license
 .. zuul:autorole:: validate-host

roles/upload-artifactory/README.rst

@@ -0,0 +1,88 @@
+Upload artifacts specified from the executor to artifactory.
+
+.. note::
+   This role uses the ``src`` function of the ``uri`` module
+   introduced in Ansible 2.7 therefore any ansible version
+   lower than that is not supported.
+
+**Role Variables**
+
+.. zuul:rolevar:: upload_artifactory_instances
+
+   Complex argument that contains the information about credentials,
+   fqdn and name. This argument is expected to come from a secret.
+
+   .. zuul:rolevar:: upload_artifactory_instances.<name>.user
+
+      User for authenticating.
+
+   .. zuul:rolevar:: upload_artifactory_instances.<name>.password
+
+      Password for authenticating.
+      Has a lower precedense than ``api_key``.
+
+   .. zuul:rolevar:: upload_artifactory_instances.<name>.api_key
+
+      API key for authenticating.
+      Has a higher precedense than ``password``.
+
+   .. zuul:rolevar:: upload_artifactory_instances.<name>.fqdn
+
+      Fully qualified domain name to the instance.
+
+   .. zuul:rolevar:: upload_artifactory_instances.<name>.transport
+      :default: https
+
+      Set to ``http`` if the instance does not support https.
+
+   .. zuul:rolevar:: force_basic_auth
+      :default: false
+
+      Set to ``true`` if the instance requires basic auth to be used.
+
+.. zuul:rolevar:: artifacts
+
+   Variable that contains a manifest of the artifacts that should be
+   uploaded to a specific instance of artifactory. This is expected to
+   be set during the build as a cached fact.
+
+    .. code-block:: yaml
+
+       artifacts:
+         - name: tarball
+           src: artifact.tar.gz
+           dest: /destination/to/put/artifact/artifact.tar.gz
+           instance: artifact-server1
+           headers:
+             Content-Type: application/gzip
+
+   The attributes available on an artifact are the following.
+
+   .. zuul:rolevar:: name
+
+      Name of the artifact.
+      This will be displayed in the build page.
+
+   .. zuul:rolevar:: src
+
+      Path relative to ``{{ zuul.executor.work_root }}/artifacts/``.
+
+   .. zuul:rolevar:: dest
+
+      Destination where the artifact should be put in.
+
+   .. zuul:rolevar:: instance
+
+      Artifactory instance to place the artiface in, this is to
+      choose which entry in :attr:`upload_artifactory_instances` to upload
+      the artifact to.
+
+   .. zuul:rolevar:: headers
+
+      Any headers that should be passed to ansibles uri module
+      when uploading.
+
+   .. zuul:rolevar:: metadata
+
+      Any metadata that should be returned to Zuul together with the
+      artifact link.

roles/upload-artifactory/defaults/main.yaml

@@ -0,0 +1 @@
+upload_artifactory_workdir: "{{ zuul.executor.work_root }}"

roles/upload-artifactory/tasks/main.yaml

@@ -0,0 +1,13 @@
+- name: Fail if run by an unsupported ansible version
+  fail:
+    msg: This role can only be used by ansible version 2.7 and greater.
+  when: ansible_version.full < "2.7.0"
+
+- name: Upload artifacts
+  include_tasks: upload.yaml
+  loop: "{{ upload_artifactory_manifest.artifacts }}"
+  loop_control:
+    loop_var: zj_artifact
+  when:
+    - upload_artifactory_manifest is defined
+    - "'artifacts' in upload_artifactory_manifest"

roles/upload-artifactory/tasks/upload.yaml

@@ -0,0 +1,69 @@
+- name: Make sure artifact exists on the executor
+  stat:
+    path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\
+           /artifacts/{{ zj_artifact.src }}"
+  register: artifact
+  delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}"
+  failed_when: not artifact.stat.exists
+
+- name: Get sha256 checksum
+  stat:
+    path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\
+           /artifacts/{{ zj_artifact.src }}"
+    checksum_algorithm: sha256
+  delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}"
+  register: artifact_sha256_checksum
+
+- name: Set request header fact
+  set_fact:
+    request_header:
+      X-Checksum-Sha1: "{{ artifact.stat.checksum }}"
+      X-Checksum-Sha256: "{{ artifact_sha256_checksum.stat.checksum }}"
+
+- name: Add artifact headers
+  set_fact:
+    request_header: "{{ request_header | combine(zj_artifact.headers|default({})) }}"
+
+- name: Add api key to header
+  when: "'api_key' in upload_artifactory_instance[zj_artifact.instance]"
+  no_log: true
+  set_fact:
+    request_header: "{{ request_header |
+                        combine({'X-JFrog-Art-Api': upload_artifactory_instance[zj_artifact.instance].api_key}) }}"
+
+- name: Set artifactory password
+  no_log: true
+  set_fact:
+    _artifactory_password: "{{ upload_artifactory_instance[zj_artifact.instance].password }}"
+  when:
+    - "'api_key' not in upload_artifactory_instance[zj_artifact.instance]"
+    - "'password' in upload_artifactory_instance[zj_artifact.instance]"
+
+- name: Set artifactory url fact
+  set_fact:
+    _artifactory_url: "{{ upload_artifactory_instance[zj_artifact.instance].transport | default('https') }}://\
+                       {{ upload_artifactory_instance[zj_artifact.instance].fqdn }}\
+                       /artifactory/{{ zj_artifact.dest }}"
+
+- name: Upload artifact
+  uri:
+    user: "{{ upload_artifactory_instance[zj_artifact.instance].user }}"
+    password: "{{ _artifactory_password | default(omit) }}"
+    url: "{{ _artifactory_url }}"
+    src: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}/artifacts/{{ zj_artifact.src }}"
+    headers: "{{ request_header }}"
+    status_code: 201
+    method: PUT
+    force_basic_auth: "{{ upload_artifactory_instance[zj_artifact.instance].force_basic_auth |
+                          default(false) | bool }}"
+    remote_src: yes  # To not unecessarily copy artifact to a tempfile
+  delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}"
+
+- name: Add artifact link to build page
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: "{{ zj_artifact.name }}"
+            url: "{{ _artifactory_url }}"
+            metadata: "{{ zj_artifact.metadata | default(omit) }}"

test-playbooks/artifactory/cleanup.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  tasks:
+    - name: Remove artifactory container
+      command: docker rm -f "{{ zuul.build }}"

test-playbooks/artifactory/run.yaml

@@ -0,0 +1,90 @@
+- hosts: all
+  tasks:
+    - name: Ensure docker is installed
+      include_role:
+        name: ensure-docker
+
+    - name: Start artifactory in a container
+      command: >-
+        docker run -d
+        -p 8081:8081 -p 8082:8082
+        --name {{ zuul.build }}
+        docker.bintray.io/jfrog/artifactory-oss:latest
+
+    - name: Wait for artifactory to start
+      uri:
+        url: http://localhost:8082/artifactory/api/system/ping
+        method: GET
+      register: artifactory_status
+      until: artifactory_status.status == 200
+      retries: 12
+      delay: 10
+
+    - name: Create a generic repository in artifactory
+      uri:
+        url: http://localhost:8082/artifactory/api/system/configuration
+        user: admin
+        password: password
+        force_basic_auth: true
+        method: PATCH
+        body: |
+          localRepositories:
+            generic-repository:
+              type: generic
+        headers:
+          Content-Type: application/yaml
+
+    - name: Create an api key for the admin user
+      uri:
+        url: http://localhost:8082/artifactory/api/security/apiKey
+        user: admin
+        password: password
+        status_code: 201
+        return_content: true
+        method: POST
+      register: artifactory_api_key
+
+    - name: Set artifactory instances fact
+      set_fact:
+        cacheable: true
+        upload_artifactory_instance:
+          localhost_password:
+            fqdn: localhost:8081
+            transport: http
+            user: admin
+            password: password
+            force_basic_auth: true
+          localhost_api_key:
+            fqdn: localhost:8081
+            transport: http
+            user: admin
+            api_key: "{{ (artifactory_api_key.content | from_json)['apiKey'] }}"
+
+- hosts: all
+  vars:
+    # Since we're testing a role that normally requires a
+    # trusted context flip the delegate_to so we execute on the
+    # remote instead. Also set the working directory to something
+    # that is known to exist on the remote.
+    _undocumented_test_worker_node_: "{{ inventory_hostname }}"
+    _undocumented_test_work_dir_: "{{ ansible_user_dir }}/zuul-output"
+  pre_tasks:
+    - name: Write a file with some content to artifacts directory
+      copy:
+        content: |
+          First file
+        dest: "{{ ansible_user_dir }}/zuul-output/artifacts/test-file.txt"
+    - name: Set upload_artifactory_manifest fact
+      set_fact:
+        upload_artifactory_manifest:
+          artifacts:
+            - name: test-file.txt
+              src: test-file.txt
+              dest: generic-repository/path/to/dest/test-file-password.txt
+              instance: localhost_password
+            - name: test-file.txt
+              src: test-file.txt
+              dest: generic-repository/path/to/dest/test-file-api-key.txt
+              instance: localhost_api_key
+  roles:
+    - upload-artifactory

zuul-tests.d/artifactory-roles.yaml

@@ -0,0 +1,15 @@
+- job:
+    name: zuul-jobs-test-upload-artifactory
+    description: Test the upload-artifactory role
+    files:
+      - roles/upload-artifactory/.*
+      - test-playbooks/artifactory/.*
+    run: test-playbooks/artifactory/run.yaml
+    cleanup-run: test-playbooks/artifactory/cleanup.yaml
+
+- project:
+    check:
+      jobs: &id001
+        - zuul-jobs-test-upload-artifactory
+    gate:
+      jobs: *id001