Fix buildset registry

The approach of having the proxy serve the local data as well as
the remote wasn't working -- it seems that the proxy would always
check upstream and prefer that data even if it had been pushed
locally.

To correct this, separate the data stores of the two registries,
and add both of them to the registry_mirror setting for the
docker daemon.  Now we will pull from our buildset registry first,
and fall back on the proxy to talk to upstream if an image is not
found locally.

The proxy is still required in order to mask out the username and
password which dockerd will otherwise use when talking to upstream.

Change-Id: Iab11954a4b5431d3b1a4d4753f519b6b71f64094

roles/build-docker-image/tasks/push.yaml

@@ -1,12 +1,12 @@
 - name: Tag image for buildset registry
   command: >-
-    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
+    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
   loop: "{{ image.tags | default(['latest']) }}"
   loop_control:
     loop_var: image_tag
 - name: Push tag to buildset registry
   command: >-
-    docker push {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
+    docker push {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
   loop: "{{ image.tags | default(['latest']) }}"
   loop_control:
     loop_var: image_tag

roles/pull-from-intermediate-registry/tasks/main.yaml

@@ -5,19 +5,19 @@
     buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
 - name: Ensure registry cert directory exists
   file:
-    path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
+    path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
     state: directory
 - name: Write registry TLS certificate
   copy:
     content: "{{ buildset_registry.cert }}"
-    dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
+    dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
 - name: Pull artifact from intermediate registry
   command: >-
     skopeo --insecure-policy copy
     --src-creds={{ intermediate_registry.username }}:{{ intermediate_registry.password }}
     --dest-creds={{ buildset_registry.username }}:{{ buildset_registry.password }}
     {{ item.url }}
-    docker://{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
+    docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
   when: "item.metadata.type | default('') == 'container_image'"
   loop: "{{ zuul.artifacts | default([]) }}"
   # no_log: true  TODO(corvus): replace

roles/run-buildset-registry/README.rst

@@ -3,9 +3,7 @@ Runs a docker registry for the use of this buildset.
 This may be used for a single job running on a single node, or it may
 be used at the root of a job graph so that multiple jobs running for a
 single change can share the registry.  Two registry endpoints are
-provided -- one is a read-only endpoint which acts as a pull-through
+provided -- one is a local registry, the second is an upstream proxy.
-proxy and serves upstream images as well as those which are pushed to
-the registry.  The second is intended only for pushing images.
 
 **Role Variables**
 
@@ -28,13 +26,9 @@ the registry.  The second is intended only for pushing images.
 
       The port on which the registry is listening.
 
-   .. zuul:rolevar:: push_host
+   .. zuul:rolevar:: proxy_port
 
-      The host (IP address) to use when pushing images to the registry.
+      The port on which the proxy is listening.
-
-   .. zuul:rolevar:: push_port
-
-      The port to use when pushing images to the registry.
 
    .. zuul:rolevar:: username
 

roles/run-buildset-registry/tasks/main.yaml

@@ -59,9 +59,9 @@
 - name: Decode TLS certificate
   set_fact:
     certificate: "{{ certificate.content | b64decode }}"
-- name: Start a docker proxy
+- name: Start a docker registry
   docker_container:
-    name: buildset_proxy
+    name: buildset_registry
     image: registry:2
     state: started
     restart_policy: always
@@ -73,16 +73,12 @@
       REGISTRY_AUTH: htpasswd
       REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
       REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
-      REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
-      REGISTRY_PROXY_USERNAME: ''
-      REGISTRY_PROXY_PASSWORD: ''
     volumes:
-      - "{{ buildset_registry_root}}/data:/var/lib/registry"
       - "{{ buildset_registry_root}}/certs:/certs"
       - "{{ buildset_registry_root}}/auth:/auth"
-- name: Start a docker registry
+- name: Start a docker proxy
   docker_container:
-    name: buildset_registry
+    name: buildset_proxy
     image: registry:2
     state: started
     restart_policy: always
@@ -94,8 +90,10 @@
       REGISTRY_AUTH: htpasswd
       REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
       REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
+      REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
+      REGISTRY_PROXY_USERNAME: ''
+      REGISTRY_PROXY_PASSWORD: ''
     volumes:
-      - "{{ buildset_registry_root}}/data:/var/lib/registry"
       - "{{ buildset_registry_root}}/certs:/certs"
       - "{{ buildset_registry_root}}/auth:/auth"
 - name: Set registry information fact
@@ -103,8 +101,7 @@
     buildset_registry:
       host: "{{ ansible_host }}"
       port: 5000
-      push_host: "{{ ansible_host }}"
+      proxy_port: 5001
-      push_port: 5001
       username: zuul
       password: "{{ registry_password }}"
       cert: "{{ certificate }}"

roles/use-buildset-registry/README.rst

@@ -17,13 +17,9 @@ Use this role on any host which should use the buildset registry.
 
       The port on which the registry is listening.
 
-   .. zuul:rolevar:: push_host
+   .. zuul:rolevar:: proxy_port
 
-      The host (IP address) to use when pushing images to the registry.
+      The port on which the registry proxy is listening.
-
-   .. zuul:rolevar:: push_port
-
-      The port to use when pushing images to the registry.
 
    .. zuul:rolevar:: username
 

roles/use-buildset-registry/tasks/main.yaml

@@ -3,26 +3,26 @@
   file:
     state: directory
     path: /etc/docker
-- name: Ensure registry cert directory exists
+- name: Ensure buildset registry cert directory exists
   become: true
   file:
     path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
     state: directory
-- name: Ensure push registry cert directory exists
+- name: Ensure proxy registry cert directory exists
   become: true
   file:
-    path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
+    path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/"
     state: directory
-- name: Write registry TLS certificate
+- name: Write buildset registry TLS certificate
   become: true
   copy:
     content: "{{ buildset_registry.cert }}"
     dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
-- name: Write push registry TLS certificate
+- name: Write proxy registry TLS certificate
   become: true
   copy:
     content: "{{ buildset_registry.cert }}"
-    dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
+    dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/ca.crt"
 
 # Update daemon config
 - name: Check if docker daemon configuration exists
@@ -46,7 +46,7 @@
 - name: Add registry to docker daemon configuration
   vars:
     new_config:
-      registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/']"
+      registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host }}:{{ buildset_registry.proxy_port}}/']"
   set_fact:
     docker_config: "{{ docker_config | combine(new_config) }}"
 - name: Save docker daemon configuration

roles/use-buildset-registry/tasks/user-config.yaml

@@ -31,7 +31,7 @@
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
           "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
-          "{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}":
+          "{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
         }
   set_fact: