From 8640466183efd59e1b773f1ccf17538624b73cd9 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Tue, 15 Jan 2019 14:03:00 -0800
Subject: [PATCH] Add docker image promotion roles

This adds three roles which can be used to build a docker image
promotion system.

Change-Id: Iefd9278cdb90bbbaab93a4d23c055e9289fde5ba
---
 roles/build-docker-image/README.rst           |  3 +
 roles/build-docker-image/common.rst           | 98 +++++++++++++++++++
 roles/build-docker-image/defaults/main.yaml   |  1 +
 roles/build-docker-image/tasks/main.yaml      | 13 +++
 roles/promote-docker-image/README.rst         |  3 +
 roles/promote-docker-image/defaults/main.yaml |  1 +
 roles/promote-docker-image/tasks/main.yaml    | 20 ++++
 .../tasks/promote-cleanup.yaml                | 20 ++++
 .../tasks/promote-retag.yaml                  | 39 ++++++++
 roles/upload-docker-image/README.rst          |  3 +
 roles/upload-docker-image/defaults/main.yaml  |  1 +
 roles/upload-docker-image/tasks/main.yaml     |  6 ++
 12 files changed, 208 insertions(+)
 create mode 100644 roles/build-docker-image/README.rst
 create mode 100644 roles/build-docker-image/common.rst
 create mode 100644 roles/build-docker-image/defaults/main.yaml
 create mode 100644 roles/build-docker-image/tasks/main.yaml
 create mode 100644 roles/promote-docker-image/README.rst
 create mode 100644 roles/promote-docker-image/defaults/main.yaml
 create mode 100644 roles/promote-docker-image/tasks/main.yaml
 create mode 100644 roles/promote-docker-image/tasks/promote-cleanup.yaml
 create mode 100644 roles/promote-docker-image/tasks/promote-retag.yaml
 create mode 100644 roles/upload-docker-image/README.rst
 create mode 100644 roles/upload-docker-image/defaults/main.yaml
 create mode 100644 roles/upload-docker-image/tasks/main.yaml

diff --git a/roles/build-docker-image/README.rst b/roles/build-docker-image/README.rst
new file mode 100644
index 000000000..f533afa1a
--- /dev/null
+++ b/roles/build-docker-image/README.rst
@@ -0,0 +1,3 @@
+Build one or more docker images.
+
+.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/build-docker-image/common.rst b/roles/build-docker-image/common.rst
new file mode 100644
index 000000000..ccaf68d1c
--- /dev/null
+++ b/roles/build-docker-image/common.rst
@@ -0,0 +1,98 @@
+This is one of a collection of roles which are designed to work
+together to build, upload, and promote docker images in a gating
+context:
+
+* :zuul:role:`build-docker-image`: Build the images.
+* :zuul:role:`upload-docker-image`: Stage the images on dockerhub.
+* :zuul:role:`promote-docker-image`: Promote previously uploaded images.
+
+The :zuul:role:`build-docker-image` role is designed to be used in
+`check` and `gate` pipelines and simply builds the images.  It can be
+used to verify that the build functions, or it can be followed by the
+use of subsequent roles to upload the images to Docker Hub.
+
+The :zuul:role:`upload-docker-image` role uploads the images to Docker
+Hub, but only with a single tag corresponding to the change ID.  This
+role is designed to be used in a job in a `gate` pipeline so that the
+build produced by the gate is staged and can later be promoted to
+production if the change is successful.
+
+The :zuul:role:`promote-docker-image` role is designed to be used in a
+`promote` pipeline.  It requires no nodes and runs very quickly on the
+Zuul executor.  It simply re-tags a previously uploaded image for a
+change with whatever tags are supplied by the
+:zuul:rolevar:`build-docker-image.docker_images.context`.  It also
+removes the change ID tag from the repository in Docker Hub, and
+removes any similar change ID tags more than 24 hours old.  This keeps
+the repository tidy in the case that gated changes fail to merge after
+uploading their staged images.
+
+They all accept the same input data, principally a list of
+dictionaries representing the images to build.  YAML anchors_ can be
+used to supply the same data to all three jobs.
+
+Use the :zuul:role:`install-docker` role to install Docker before
+using this role.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   The project directory.  Serves as the base for
+   :zuul:rolevar:`build-docker-image.docker_images.context`.
+
+.. zuul:rolevar:: credentials
+   :type: dict
+
+   This is only required for the upload and promote roles.  This is
+   expected to be a Zuul Secret with two keys:
+
+   .. zuul:rolevar:: username
+
+      The Docker Hub username.
+
+   .. zuul:rolevar:: username
+
+      The Docker Hub password
+
+.. zuul:rolevar:: docker_images
+   :type: list
+
+   A list of images to build.  Each item in the list should have:
+
+   .. zuul:rolevar:: context
+
+      The docker build context; this should be a directory underneath
+      :zuul:rolevar:`build-docker-image.zuul_work_dir`.
+
+   .. zuul:rolevar:: repository
+
+      The name of the target repository in dockerhub for the
+      image.  Supply this even if the image is not going to be
+      uploaded (it will be tagged with this in the local
+      registry).
+
+   .. zuul:rolevar:: path
+
+      Optional: the directory that should be passed to docker build.
+      Useful for building images with a Dockerfile in the context
+      directory but a source repository elsewhere.
+
+   .. zuul:jobvar:: build_args
+      :type: list
+
+      Optional: a list of values to pass to the docker ``--build-arg``
+      parameter.
+
+   .. zuul:rolevar:: target
+
+      Optional: the target for a multi-stage build.
+
+   .. zuul:jobvar:: tags
+      :type: list
+      :default: ['latest']
+
+      A list of tags to be added to the image when promoted.
+
+.. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor//
diff --git a/roles/build-docker-image/defaults/main.yaml b/roles/build-docker-image/defaults/main.yaml
new file mode 100644
index 000000000..9739eb171
--- /dev/null
+++ b/roles/build-docker-image/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml
new file mode 100644
index 000000000..5db905099
--- /dev/null
+++ b/roles/build-docker-image/tasks/main.yaml
@@ -0,0 +1,13 @@
+- name: Build a docker image
+  command: >-
+    docker build {{ item.path | default('.') }} -f Dockerfile
+    {% if target | default(false) -%}
+      --target {{ target }}
+    {% endif -%}
+    {% for build_arg in item.build_args | default([]) -%}
+      --build-arg {{ build_arg }}
+    {% endfor -%}
+    --tag {{ item.repository }}:change_{{ zuul.change }}
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ item.context }}"
+  loop: "{{ images }}"
diff --git a/roles/promote-docker-image/README.rst b/roles/promote-docker-image/README.rst
new file mode 100644
index 000000000..abce78fc3
--- /dev/null
+++ b/roles/promote-docker-image/README.rst
@@ -0,0 +1,3 @@
+Promote one or more previously uploaded docker images.
+
+.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/promote-docker-image/defaults/main.yaml b/roles/promote-docker-image/defaults/main.yaml
new file mode 100644
index 000000000..9739eb171
--- /dev/null
+++ b/roles/promote-docker-image/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/promote-docker-image/tasks/main.yaml b/roles/promote-docker-image/tasks/main.yaml
new file mode 100644
index 000000000..025303a89
--- /dev/null
+++ b/roles/promote-docker-image/tasks/main.yaml
@@ -0,0 +1,20 @@
+# This is used by the delete tasks
+- name: Get dockerhub JWT token
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/users/login/"
+    body_format: json
+    body:
+      username: "{{ credentials.username }}"
+      password: "{{ credentials.password }}"
+  register: jwt_token
+- name: Promote image
+  loop: "{{ images }}"
+  loop_control:
+    loop_var: image
+  include_tasks: promote-retag.yaml
+- name: Delete obsolete tags
+  loop: "{{ images }}"
+  loop_control:
+    loop_var: image
+  include_tasks: promote-cleanup.yaml
diff --git a/roles/promote-docker-image/tasks/promote-cleanup.yaml b/roles/promote-docker-image/tasks/promote-cleanup.yaml
new file mode 100644
index 000000000..d8435b439
--- /dev/null
+++ b/roles/promote-docker-image/tasks/promote-cleanup.yaml
@@ -0,0 +1,20 @@
+- name: List tags
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags?page_size=1000"
+    status_code: 200
+  register: tags
+- name: Set cutoff timestamp to 24 hours ago
+  command: "python3 -c \"import datetime; print((datetime.datetime.utcnow()-datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%fZ'))\""
+  register: cutoff
+- name: Delete all change tags older than the cutoff
+  no_log: true
+  loop: "{{ tags.json.results }}"
+  loop_control:
+    loop_var: docker_tag
+  when: docker_tag.last_updated < cutoff.stdout and docker_tag.name.startswith('change_')
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags/{{ docker_tag.name }}/"
+    method: DELETE
+    status_code: 204
+    headers:
+      Authorization: "JWT {{ jwt_token.json.token }}"
diff --git a/roles/promote-docker-image/tasks/promote-retag.yaml b/roles/promote-docker-image/tasks/promote-retag.yaml
new file mode 100644
index 000000000..77b611ac8
--- /dev/null
+++ b/roles/promote-docker-image/tasks/promote-retag.yaml
@@ -0,0 +1,39 @@
+- name: Get dockerhub token
+  no_log: true
+  uri:
+    url: "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{{ image.repository }}:pull,push"
+    user: "{{ credentials.username }}"
+    password: "{{ credentials.password }}"
+    force_basic_auth: true
+  register: token
+- name: Get manifest
+  no_log: true
+  uri:
+    url: "https://registry.hub.docker.com/v2/{{ image.repository }}/manifests/change_{{ zuul.change }}"
+    status_code: 200
+    headers:
+      Accept: "application/vnd.docker.distribution.manifestv2+json"
+      Authorization: "Bearer {{ token.json.token }}"
+    return_content: true
+  register: manifest
+- name: "Put manifest"
+  no_log: true
+  loop: "{{ image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: new_tag
+  uri:
+    url: "https://registry.hub.docker.com/v2/{{ image.repository }}/manifests/{{ new_tag }}"
+    method: PUT
+    status_code: 201
+    body: "{{ manifest.content | string }}"
+    headers:
+      Content-Type: "application/vnd.docker.distribution.manifestv2+json"
+      Authorization: "Bearer {{ token.json.token }}"
+- name: Delete the current change tag
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags/change_{{ zuul.change }}/"
+    method: DELETE
+    status_code: 204
+    headers:
+      Authorization: "JWT {{ jwt_token.json.token }}"
diff --git a/roles/upload-docker-image/README.rst b/roles/upload-docker-image/README.rst
new file mode 100644
index 000000000..2b04c2e42
--- /dev/null
+++ b/roles/upload-docker-image/README.rst
@@ -0,0 +1,3 @@
+Upload one or more docker images.
+
+.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/upload-docker-image/defaults/main.yaml b/roles/upload-docker-image/defaults/main.yaml
new file mode 100644
index 000000000..9739eb171
--- /dev/null
+++ b/roles/upload-docker-image/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/upload-docker-image/tasks/main.yaml b/roles/upload-docker-image/tasks/main.yaml
new file mode 100644
index 000000000..ff49915a6
--- /dev/null
+++ b/roles/upload-docker-image/tasks/main.yaml
@@ -0,0 +1,6 @@
+- name: Log in to dockerhub
+  command: "docker login -u {{ credentials.username }} -p {{ credentials.password }}"
+  no_log: true
+- name: Upload to dockerhub
+  command: "docker push {{ item.repository }}:change_{{ zuul.change }}"
+  loop: "{{ images }}"