Add a registry test job

This exercises the intermediate and buildset registry roles.

Change-Id: Ic0da2d6b48a7b9f9b616033c70db471ba98181b4
This commit is contained in:
James E. Blair 2019-05-24 08:03:54 -07:00
parent ad16143c23
commit 2f2d6ce3f7
13 changed files with 405 additions and 5 deletions

View File

@ -8,6 +8,7 @@
jobs jobs
roles roles
docker-image docker-image
test-jobs
Indices and tables Indices and tables
================== ==================

9
doc/source/test-jobs.rst Normal file
View File

@ -0,0 +1,9 @@
Zuul-Jobs Tests
===============
These are jobs which are used for testing changes to roles in this
repository. They are not meant for general consumption and are only
expected to be run by the OpenDev Zuul. Generally you can ignore
these unless you are making changes to the roles in this repository.
.. zuul:autojob:: zuul-jobs-test-registry

View File

@ -9,9 +9,16 @@
# process will terminate when the playbook ends. # process will terminate when the playbook ends.
- name: Start socat to work around https://github.com/moby/moby/issues/39033 - name: Start socat to work around https://github.com/moby/moby/issues/39033
shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &" shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
- name: Find socat port
# Use slurp instead of file lookup to make this testable on a fake
# executor node.
- name: Read socat port
slurp:
src: "{{ zuul.executor.work_root }}/socat_port"
register: read_socat_port
- name: Set socat port
set_fact: set_fact:
socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}" socat_port: "{{ read_socat_port['content'] | b64decode | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') | regex_replace('\n', '') }}"
# Set up cert files for the buildset registry # Set up cert files for the buildset registry
- name: Ensure registry cert directory exists - name: Ensure registry cert directory exists

View File

@ -9,9 +9,16 @@
# process will terminate when the playbook ends. # process will terminate when the playbook ends.
- name: Start socat to work around https://github.com/moby/moby/issues/39033 - name: Start socat to work around https://github.com/moby/moby/issues/39033
shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &" shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
- name: Find socat port
# Use slurp instead of file lookup to make this testable on a fake
# executor node.
- name: Read socat port
slurp:
src: "{{ zuul.executor.work_root }}/socat_port"
register: read_socat_port
- name: Set socat port
set_fact: set_fact:
socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}" socat_port: "{{ read_socat_port['content'] | b64decode | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') | regex_replace('\n', '') }}"
# Set up cert files for the buildset registry # Set up cert files for the buildset registry
- name: Ensure registry cert directory exists - name: Ensure registry cert directory exists

View File

@ -0,0 +1,2 @@
FROM debian:testing
RUN touch /tmp/foo

View File

@ -0,0 +1 @@
ansible_python_interpreter: python3

View File

@ -0,0 +1,10 @@
- name: Ensure registry cert directory exists
become: true
file:
path: "/etc/docker/certs.d/{{ registry_host }}:{{ registry_port }}/"
state: directory
- name: Write registry TLS certificate
become: true
copy:
content: "{{ registry_cert }}"
dest: "/etc/docker/certs.d/{{ registry_host }}:{{ registry_port }}/ca.crt"

View File

@ -0,0 +1,39 @@
# Update user config
- name: Ensure docker user directory exists
file:
state: directory
path: "~/.docker"
mode: 0700
- name: Check if docker user configuration exists
stat:
path: "~/.docker/config.json"
register: docker_config_stat
- name: Load docker user configuration
when: docker_config_stat.stat.exists
slurp:
path: "~/.docker/config.json"
register: docker_config
- name: Parse docker user configuration
when: docker_config_stat.stat.exists
set_fact:
docker_config: "{{ docker_config.content | b64decode | from_json }}"
- name: Set default docker user configuration
when: not docker_config_stat.stat.exists
set_fact:
docker_config:
auths: {}
- name: Add registry to docker user configuration
vars:
new_config:
auths: |
{
"localhost:5000":
{"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
}
set_fact:
docker_config: "{{ docker_config | combine(new_config, recursive=True) }}"
- name: Save docker user configuration
copy:
content: "{{ docker_config | to_nice_json }}"
dest: "~/.docker/config.json"
mode: 0600

View File

@ -0,0 +1,46 @@
- name: Ensure registry volume directories exists
file:
state: directory
path: "/var/registry/{{ item }}"
loop:
- certs
- auth
- name: Install python packages
package:
name:
- python3-docker
- python3-passlib
- python3-bcrypt
state: present
- name: Write htpassword file
htpasswd:
create: true
crypt_scheme: bcrypt
path: /var/registry/auth/htpasswd
name: "{{ intermediate_registry.username }}"
password: "{{ intermediate_registry.password }}"
- name: Write TLS private key
copy:
content: "{{ intermediate_registry_tls_key }}"
dest: /var/registry/certs/domain.key
- name: Write TLS certificate
copy:
content: "{{ intermediate_registry_tls_cert }}{{ intermediate_registry_tls_chain | default('') }}"
dest: /var/registry/certs/domain.crt
- name: Start intermediate docker registry
docker_container:
name: intermediate_registry
image: registry:2
state: started
restart_policy: always
ports:
- "5000:5000"
env:
REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
REGISTRY_HTTP_TLS_KEY: /certs/domain.key
REGISTRY_AUTH: htpasswd
REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
volumes:
- "/var/registry/certs:/certs"
- "/var/registry/auth:/auth"

View File

@ -0,0 +1,177 @@
- hosts: intermediate-registry:builder
name: Install docker on registry and builder hosts
roles:
- install-docker
# Run the intermediate registry on this host, and also build an image
# and place it in the registry to simulate an artifact from a previous
# build which has been passed to this one (so that we can test pulling
# from the intermediate registry in the correct order).
- hosts: intermediate-registry
name: Set up the intermediate registry and add a build
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Include previous build vars
include_vars: vars/previous-build.yaml
- name: Run the intermediate registry
include_role:
name: run-test-intermediate-registry
apply:
become: true
- name: Install the intermediate registry cert
include_role:
name: install-registry-cert
vars:
registry_host: localhost
registry_port: 5000
registry_cert: "{{ intermediate_registry_tls_cert }}"
- name: Set up user credentials for the intermediate registry
include_role:
name: intermediate-registry-user-config
- name: Build a docker image for the previous build
include_role:
name: build-docker-image
vars:
docker_images:
- context: test-playbooks/registry/docker
repository: "{{ previous_build_repository }}"
- name: Tag the previous build
command: "docker tag {{ previous_build_repository }}:latest localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
- name: Push the previous build to the intermediate registry
command: "docker push localhost:5000/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
# This is also essentially pre-configuration for the real test of the
# roles. This sets up a fake executor (since we can't run the
# necessary commands untrusted on the real one).
- hosts: executor
name: Set up a simulated executor
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Create simulated zuul work directory
become: true
file:
state: directory
path: "{{ zuul.executor.work_root }}"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
- name: Add project atomic PPA
become: true
apt_repository:
repo: ppa:projectatomic/ppa
- name: Install packages
become: true
package:
name:
- socat
- skopeo
state: present
- name: Install the intermediate registry cert
include_role:
name: install-registry-cert
vars:
registry_host: "{{ intermediate_registry.host }}"
registry_port: "{{ intermediate_registry.port }}"
registry_cert: "{{ intermediate_registry_tls_cert }}"
- name: Make /etc/docker directory zuul-owned
become: true
file:
state: directory
path: "/etc/docker"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
recurse: true
- name: Configure /etc/hosts for intermediate registry
become: true
lineinfile:
path: /etc/hosts
state: present
regex: "^{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}$"
line: "{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}"
insertafter: EOF
# This begins the simulation of what we would expect to happen in a
# normal job.
- hosts: builder
name: Test the buildset registry roles
roles:
- run-buildset-registry
- use-buildset-registry
- hosts: executor
name: Test pulling from the intermediate registry
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Include previous build vars
include_vars: vars/previous-build.yaml
- name: Prepare a replacement zuul variable
set_fact:
test_zuul: "{{ previous_build_zuul }}"
- name: Run pull-from-intermediate-registry role
include_role:
name: pull-from-intermediate-registry
vars:
zuul: "{{ test_zuul }}"
# This simulates a build actually using the previous build.
- hosts: builder
name: Test that the previous build is available
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Include previous build vars
include_vars: vars/previous-build.yaml
- name: Pull the previous build from buildset registry to the builder host
command: "docker pull {{ previous_build_repository }}:latest"
- name: Show local docker images for debugging
command: "docker image ls"
- name: Verify previously built image is in buildset registry
command: "docker image inspect {{ previous_build_repository }}:latest"
# Back to straightforward use of the roles under test.
- hosts: builder
name: Test building a docker image
roles:
- role: build-docker-image
vars:
docker_images:
- context: test-playbooks/registry/docker
repository: downstream/image
- hosts: executor
name: Test pushing to the intermediate registry
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Run push-to-intermediate-registry role
include_role:
name: push-to-intermediate-registry
vars:
docker_images:
- context: playbooks/registry/docker
repository: downstream/image
# And finally an external verification step.
- hosts: executor
name: Test that the newly built image was pushed to the intermediate registry
tasks:
- name: Include intermediate registry vars
include_vars: vars/intermediate-registry-auth.yaml
- name: Fetch intermediate registry catalog
uri:
url: "https://{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/v2/_catalog"
validate_certs: false
user: "{{ intermediate_registry.username }}"
password: "{{ intermediate_registry.password }}"
register: catalog
- name: Verify newly built image is in intermediate registry catalog
assert:
that: "'downstream/image' in catalog.json.repositories"

View File

@ -0,0 +1,58 @@
intermediate_registry:
host: zuul-jobs.intermediate-registry
port: 5000
username: "zuul"
password: dQI83awO8Akuw0WU
intermediate_registry_tls_key: |
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDYkpjfIz7bziCa
mFrWqQ84ldeAs2jvSKs2JG0RhYNNLokr2AU/5TUvqtAisyyd5AX5dBHQ7u/7Vgmj
towt7loFfAG/2/rpdSGi2Njx11roBUoDsjwdE9w3aNnrDvOCyJcepx5TWYS86+vZ
IqodvdnuoWTk9VuolWfHsCgPRQV4uwMbIC5kbv2o4FORsOEzbuRfCEX9UTcAMEGg
K/m/kM/valkrYeBbLILsOcivg4Jh0m+PFC7NTcQFo+uwpZzZvlNtVbmQ3LqkHDAE
KDK94uBcQtdYjvvl6UZ+pNo+puD9iakYtcpQFuU8rpavMLE87+SuPVgi2Rk6QtTz
OAP2mDMJAgMBAAECggEBANM9MfS7WQ1mIXEI19l2roz/wmIbHGgAllbJ8sRbWLWI
hW0JWB15gIYM8tRVtVgP2C/3IYWL+PFKez5+yH3odU/SI5ayhyr8/6DqJ7jD2Dxl
JEs0puOpwmsdTyixvZy78IKKeM7NiuYGq1VwNUOrMQ1LyLB2DUAC8mXYkUpLhUm6
O4wVaGie7XwMOJazRs66ceU9k7Nuv3b57yc3PN2bzTqYUVjmJ1XeuAiBJaAeHts6
NfG1+vO9xLXIRTRWvDGKByNsYJJLLPOXZkQZZFYYe8TTduxyCmZgShY6sZmmnWua
cAdBL6b/5B3PZ2SkhdLHklaZmH8PTeAoqI2RDz/8eIECgYEA8gofU8LrK1Xjgrig
ItQxYxqZCrggm9lMMcaADc7u3nff68NyImZ5bSXhvZCu74cAIMx12HbU1UvSCsQ4
/cncHrlBOzG529878+iWgiUrJ29GsQiHGj+qHA4qGBSP0Qan7ISunskj4GezTeHd
/A3oTn5rLuld9V++647O35lXArkCgYEA5RBwV5nle49UT38hNqL/K+TUX5oZJXB8
Xl9FT1L799toHUPEWEkSpf7Suf1hDwv6+tsIPO6tN7YirxK390JRxPaT948J8n1d
TkurGDs1uwLQdUWgXIwvQ8ms+8rYvTU7vg2hI7/BZhH09LmGCiYSwnem0QYXjGnc
kk56VeExytECgYBmBDw2Ctcied4eEAF3DKcQVXqiGP+tkMZbyIXazBjEbhRUhBmM
RFLz3V6rjtsdHHLCYEtfhJ6qlH2gihpXZgjAbmb/MzNaaFoVsTgW/OGWioFqRuTi
/GiP0KyPX8NKYBrRRw9u3+qeQDdEIWp2Pcpno0M8D6LJtKR9FsE9X51cCQKBgQCs
8u5/ldjoo91acHhZUlQrhgi7bhQSao3ciz4/mD5ac7R2dBYpOnL0FiRw/VhtDfSf
twTPTL5IVCJ34UA5Vj964VnzDnLKPdFXLlauYvY8jvFpufpMJiQBoKIVMqDWqvzC
kHPcFAon0OMMa49C1mBPqBuxslHRWJSLeulvMipwIQKBgDFzDTH49cmKP8YQmCuT
vC5PJJ+hutbf/dOVJuOZ5KlKwnRkbMwoamYKrkjgmWMBgtzyz12/a46lZ58ul4xW
1fKw/nx8uQcbnKnigyjsAUzI9FgBR4d10cYdxPlfYVmj4TAUA3os5Gu6VKySy6SV
xuHEIA6nFsXLXGBu25vI5tEv
-----END PRIVATE KEY-----
intermediate_registry_tls_cert: |
-----BEGIN CERTIFICATE-----
MIIDtDCCApygAwIBAgIJANpxowfzYw4vMA0GCSqGSIb3DQEBCwUAMG8xCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQxKDAmBgNVBAMMH3p1dWwtam9icy5pbnRlcm1lZGlhdGUt
cmVnaXN0cnkwHhcNMTkwNTMwMjAwOTQxWhcNMzkwNTI1MjAwOTQxWjBvMQswCQYD
VQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQg
V2lkZ2l0cyBQdHkgTHRkMSgwJgYDVQQDDB96dXVsLWpvYnMuaW50ZXJtZWRpYXRl
LXJlZ2lzdHJ5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2JKY3yM+
284gmpha1qkPOJXXgLNo70irNiRtEYWDTS6JK9gFP+U1L6rQIrMsneQF+XQR0O7v
+1YJo7aMLe5aBXwBv9v66XUhotjY8dda6AVKA7I8HRPcN2jZ6w7zgsiXHqceU1mE
vOvr2SKqHb3Z7qFk5PVbqJVnx7AoD0UFeLsDGyAuZG79qOBTkbDhM27kXwhF/VE3
ADBBoCv5v5DP72pZK2HgWyyC7DnIr4OCYdJvjxQuzU3EBaPrsKWc2b5TbVW5kNy6
pBwwBCgyveLgXELXWI775elGfqTaPqbg/YmpGLXKUBblPK6WrzCxPO/krj1YItkZ
OkLU8zgD9pgzCQIDAQABo1MwUTAdBgNVHQ4EFgQU00qH9bMUPRacZwgvBgczgR8Z
424wHwYDVR0jBBgwFoAU00qH9bMUPRacZwgvBgczgR8Z424wDwYDVR0TAQH/BAUw
AwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAHEX2Tw19w5okaJ+6gHMFjA338ffwU9n5
2piBMypbYr50yyPyUaTmz4SIBsTLkIWu00a0pdo9pqZDnv1KwxtJtP4o4qQXhMd4
Ve3FFF+6AMaOy5y5+hRkE8iHOOik/rNPFqkVDatNGuOMSNYO/jUFXc+C6Ol7gM/J
edyWaafjQbvdKapKPbdP4Y69R8OlRTNK1lJMIGJrsCdaeaK4EpLpbJPHnagIMdmQ
HDsTf978weRrjJ4JEODTabsKVHKyx0GBwe8CmR0NzpfO2ORCyNUO1rLK2rzh5YTQ
qKGyfY0DAyiSHxKaUeGiskc4/WMxaYv2FzD63Xvzmot9atSwCMjN1A==
-----END CERTIFICATE-----
#intermediate_registry_tls_chain

View File

@ -0,0 +1,14 @@
# This simulates a build from a previous change which appears in this
# buildset via provides/requires. This build should be copied from
# the intermediate registry to the buildset registry.
previous_build_repository: upstream/image
previous_build_uuid: 48a84fe22a744cb5b0310f396358d912
previous_build_zuul:
artifacts:
- url: "docker://{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/{{ previous_build_repository }}:{{ previous_build_uuid }}_latest"
metadata:
repository: "{{ previous_build_repository }}"
tag: latest
type: container_image
executor: "{{ zuul.executor }}"

View File

@ -595,4 +595,33 @@
Override the default searching above with explicit Override the default searching above with explicit
domain/path references (see validate-zone-db role) domain/path references (see validate-zone-db role)
run: playbooks/validate-zone-db/run.yaml run: playbooks/validate-zone-db/run.yaml
- job:
name: zuul-jobs-test-registry
description: |
Test the intermediate registry roles.
This job tests changes to the intermediate registry roles. It
is not meant to be used directly but rather run on changes to
roles in the zuul-jobs repo.
allowed-projects:
- zuul/zuul-jobs
files:
- roles/pull-from-intermediate-registry/.*
- roles/push-to-intermediate-registry/.*
- roles/install-docker/.*
- roles/build-docker-image/.*
- roles/run-buildset-registry/.*
- roles/use-buildset-registry/.*
- test-playbooks/registry/.*
run:
test-playbooks/registry/test-registry.yaml
nodeset:
nodes:
- name: intermediate-registry
label: ubuntu-bionic
- name: executor
label: ubuntu-bionic
- name: builder
label: ubuntu-bionic