zuul/zuul-jobs
Code Issues Proposed changes

Fix buildset registry

Browse Source 
The approach of having the proxy serve the local data as well as
the remote wasn't working -- it seems that the proxy would always
check upstream and prefer that data even if it had been pushed
locally.

To correct this, separate the data stores of the two registries,
and add both of them to the registry_mirror setting for the
docker daemon.  Now we will pull from our buildset registry first,
and fall back on the proxy to talk to upstream if an image is not
found locally.

The proxy is still required in order to mask out the username and
password which dockerd will otherwise use when talking to upstream.

Change-Id: Iab11954a4b5431d3b1a4d4753f519b6b71f64094
This commit is contained in:
James E. Blair 2019-03-01 15:52:01 -08:00
parent 2da8976da0
commit 9c0d25f349
7 changed files with 26 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
roles/build-docker-image/tasks/push.yaml
View File

@ -1,12 +1,12 @@
- name: Tag image for buildset registry
command: >-
docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
loop: "{{ image.tags | default(['latest']) }}"
loop_control:
loop_var: image_tag
- name: Push tag to buildset registry
command: >-
docker push {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }}
docker push {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
loop: "{{ image.tags | default(['latest']) }}"
loop_control:
loop_var: image_tag

6
roles/pull-from-intermediate-registry/tasks/main.yaml
View File

@ -5,19 +5,19 @@
buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
- name: Ensure registry cert directory exists
file:
path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
state: directory
- name: Write registry TLS certificate
copy:
content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
- name: Pull artifact from intermediate registry
command: >-
skopeo --insecure-policy copy
--src-creds={{ intermediate_registry.username }}:{{ intermediate_registry.password }}
--dest-creds={{ buildset_registry.username }}:{{ buildset_registry.password }}
{{ item.url }}
docker://{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
when: "item.metadata.type | default('') == 'container_image'"
loop: "{{ zuul.artifacts | default([]) }}"
# no_log: true TODO(corvus): replace

12
roles/run-buildset-registry/README.rst
View File

@ -3,9 +3,7 @@ Runs a docker registry for the use of this buildset.
This may be used for a single job running on a single node, or it may
be used at the root of a job graph so that multiple jobs running for a
single change can share the registry. Two registry endpoints are
provided -- one is a read-only endpoint which acts as a pull-through
proxy and serves upstream images as well as those which are pushed to
the registry. The second is intended only for pushing images.
provided -- one is a local registry, the second is an upstream proxy.
**Role Variables**
@ -28,13 +26,9 @@ the registry. The second is intended only for pushing images.
The port on which the registry is listening.
.. zuul:rolevar:: push_host
.. zuul:rolevar:: proxy_port
The host (IP address) to use when pushing images to the registry.
.. zuul:rolevar:: push_port
The port to use when pushing images to the registry.
The port on which the proxy is listening.
.. zuul:rolevar:: username

19
roles/run-buildset-registry/tasks/main.yaml
View File

@ -59,9 +59,9 @@
- name: Decode TLS certificate
set_fact:
certificate: "{{ certificate.content | b64decode }}"
- name: Start a docker proxy
- name: Start a docker registry
docker_container:
name: buildset_proxy
name: buildset_registry
image: registry:2
state: started
restart_policy: always
@ -73,16 +73,12 @@
REGISTRY_AUTH: htpasswd
REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
REGISTRY_PROXY_USERNAME: ''
REGISTRY_PROXY_PASSWORD: ''
volumes:
- "{{ buildset_registry_root}}/data:/var/lib/registry"
- "{{ buildset_registry_root}}/certs:/certs"
- "{{ buildset_registry_root}}/auth:/auth"
- name: Start a docker registry
- name: Start a docker proxy
docker_container:
name: buildset_registry
name: buildset_proxy
image: registry:2
state: started
restart_policy: always
@ -94,8 +90,10 @@
REGISTRY_AUTH: htpasswd
REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
REGISTRY_PROXY_USERNAME: ''
REGISTRY_PROXY_PASSWORD: ''
volumes:
- "{{ buildset_registry_root}}/data:/var/lib/registry"
- "{{ buildset_registry_root}}/certs:/certs"
- "{{ buildset_registry_root}}/auth:/auth"
- name: Set registry information fact
@ -103,8 +101,7 @@
buildset_registry:
host: "{{ ansible_host }}"
port: 5000
push_host: "{{ ansible_host }}"
push_port: 5001
proxy_port: 5001
username: zuul
password: "{{ registry_password }}"
cert: "{{ certificate }}"

8
roles/use-buildset-registry/README.rst
View File

@ -17,13 +17,9 @@ Use this role on any host which should use the buildset registry.
The port on which the registry is listening.
.. zuul:rolevar:: push_host
.. zuul:rolevar:: proxy_port
The host (IP address) to use when pushing images to the registry.
.. zuul:rolevar:: push_port
The port to use when pushing images to the registry.
The port on which the registry proxy is listening.
.. zuul:rolevar:: username

14
roles/use-buildset-registry/tasks/main.yaml
View File

@ -3,26 +3,26 @@
file:
state: directory
path: /etc/docker
- name: Ensure registry cert directory exists
- name: Ensure buildset registry cert directory exists
become: true
file:
path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
state: directory
- name: Ensure push registry cert directory exists
- name: Ensure proxy registry cert directory exists
become: true
file:
path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/"
path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/"
state: directory
- name: Write registry TLS certificate
- name: Write buildset registry TLS certificate
become: true
copy:
content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
- name: Write push registry TLS certificate
- name: Write proxy registry TLS certificate
become: true
copy:
content: "{{ buildset_registry.cert }}"
dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt"
dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/ca.crt"
# Update daemon config
- name: Check if docker daemon configuration exists
@ -46,7 +46,7 @@
- name: Add registry to docker daemon configuration
vars:
new_config:
registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/']"
registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host }}:{{ buildset_registry.proxy_port}}/']"
set_fact:
docker_config: "{{ docker_config | combine(new_config) }}"
- name: Save docker daemon configuration

2
roles/use-buildset-registry/tasks/user-config.yaml
View File

@ -31,7 +31,7 @@
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
"{{ buildset_registry.host }}:{{ buildset_registry.port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
"{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}":
"{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}":
{"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
}
set_fact: