Only alias docker registries on test nodes

Skopeo has problems with ipv6 address literals just like docker as they
use the same underlying checks for url validity. However, we think we
can fix that by using a port forward from the executor to the buildset
registry so that skopeo can connect via ipv4

Go back to aliases the registries on test nodes via /etc/hosts.

Change-Id: I5f9316ffe84de06cb2fb2b65a7e1c31d9f8b0e35
Co-Authored-By: James E. Blair <jeblair@redhat.com>

roles/build-docker-image/tasks/push.yaml

@@ -1,12 +1,32 @@
+# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
+# so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  become: yes
+  lineinfile:
+    path: /etc/hosts
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry"
+    insertafter: EOF
+  when: buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using ip
+  set_fact:
+    buildset_registry_alias: zuul-jobs.buildset_registry
+  when: buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using name
+  set_fact:
+    buildset_registry_alias: "{{ buildset_registry.host }}"
+  when: not ( buildset_registry.host | ipaddr )
+
 - name: Tag image for buildset registry
   command: >-
-    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
+    docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
   loop: "{{ image.tags | default(['latest']) }}"
   loop_control:
     loop_var: image_tag
 - name: Push tag to buildset registry
   command: >-
-    docker push {{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
+    docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
   loop: "{{ image.tags | default(['latest']) }}"
   loop_control:
     loop_var: image_tag

roles/pull-from-intermediate-registry/README.rst

@@ -10,8 +10,8 @@ root of the job dependency graph).
 
 This requires the :zuul:role:`run-buildset-registry` role already
 applied.  It also requires an externally managed "intermediate"
-registry operating for the use of Zuul, and it requires "skopeo" to be
+registry operating for the use of Zuul, and it requires "skopeo" and
-installed on the Zuul executors.
+"socat" to be installed on the Zuul executors.
 
 **Role Variables**
 

roles/pull-from-intermediate-registry/tasks/main.yaml

@@ -12,6 +12,14 @@
     content: "{{ buildset_registry.cert }}"
     dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt"
 
+# Start a socat tunnel to the buildset registry to work around the
+# fact that docker does not correctly parse ipv6 addresses.  The socat
+# process will terminate when the playbook ends.
+- name: Start socat to work around https://github.com/moby/moby/issues/39033
+  shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
+- name: Find socat port
+  set_fact:
+    socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
 
 # Update user config for intermediate and buildset registries
 - name: Ensure docker user directory exists
@@ -44,7 +52,7 @@
         {
           "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}":
             {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
-          "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}":
+          "127.0.0.1:{{ socat_port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
         }
   set_fact:
@@ -62,7 +70,7 @@
       command: >-
         skopeo --insecure-policy copy
         {{ item.url }}
-        docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
+        docker://127.0.0.1:{{ socat_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
       when: "'metadata' in item and item.metadata.type | default('') == 'container_image'"
       loop: "{{ zuul.artifacts | default([]) }}"
   always:

roles/push-to-intermediate-registry/README.rst

@@ -8,8 +8,8 @@ image build.
 
 This requires the :zuul:role:`run-buildset-registry` role already
 applied.  It also requires an externally managed "intermediate"
-registry operating for the use of Zuul, and it requires "skopeo" to be
+registry operating for the use of Zuul, and it requires "skopeo" and
-installed on the Zuul executors.
+"socat" to be installed on the Zuul executors.
 
 **Role Variables**
 

roles/push-to-intermediate-registry/tasks/push-image.yaml

@@ -1,7 +1,7 @@
 - name: Push tag to intermediate registry
   command: >-
     skopeo --insecure-policy copy
-    docker://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
+    docker://127.0.0.1:{{ socat_port }}/{{ image.repository }}:{{ image_tag }}
     docker://{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }}
   loop: "{{ image.tags | default(['latest']) }}"
   loop_control:

roles/push-to-intermediate-registry/tasks/push.yaml

@@ -12,6 +12,15 @@
     content: "{{ buildset_registry.cert }}"
     dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt"
 
+# Start a socat tunnel to the buildset registry to work around the
+# fact that docker does not correctly parse ipv6 addresses.  The socat
+# process will terminate when the playbook ends.
+- name: Start socat to work around https://github.com/moby/moby/issues/39033
+  shell: "socat -d -d TCP-LISTEN:0,fork TCP:{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }} 2> {{ zuul.executor.work_root }}/socat_port &"
+- name: Find socat port
+  set_fact:
+    socat_port: "{{ lookup('file', zuul.executor.work_root + '/socat_port') | regex_replace('.*?0\\.0\\.0\\.0:(\\d+)', '\\1') }}"
+
 # Update user config for intermediate and buildset registries
 - name: Ensure docker user directory exists
   file:
@@ -43,7 +52,7 @@
         {
           "{{ intermediate_registry.host | ipwrap }}:{{ intermediate_registry.port }}":
             {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
-          "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}":
+          "localhost:{{ socat_port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
         }
   set_fact:

roles/run-buildset-registry/tasks/main.yaml

@@ -44,7 +44,7 @@
     path: "{{ buildset_registry_root}}/certs/domain.csr"
     privatekey_path: "{{ buildset_registry_root}}/certs/domain.key"
     common_name: "{{ ansible_host }}"
-    subject_alt_name: "DNS:{{ ansible_host }},IP:{{ ansible_host }}"
+    subject_alt_name: "DNS:zuul-jobs.buildset-registry,DNS:{{ ansible_host }},IP:{{ ansible_host }},IP:127.0.0.1"
 - name: Generate a TLS cert for the Docker registry
   openssl_certificate:
     path: "{{ buildset_registry_root}}/certs/domain.crt"

roles/use-buildset-registry/tasks/main.yaml

@@ -1,3 +1,23 @@
+# Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
+# so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  become: yes
+  lineinfile:
+    path: /etc/hosts
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset_registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset_registry"
+    insertafter: EOF
+  when: buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using ip
+  set_fact:
+    buildset_registry_alias: zuul-jobs.buildset_registry
+  when: buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using name
+  set_fact:
+    buildset_registry_alias: "{{ buildset_registry.host }}"
+  when: not ( buildset_registry.host | ipaddr )
+
 - name: Ensure docker directory exists
   become: yes
   file:
@@ -6,23 +26,23 @@
 - name: Ensure buildset registry cert directory exists
   become: true
   file:
-    path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/"
+    path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/"
     state: directory
 - name: Ensure proxy registry cert directory exists
   become: true
   file:
-    path: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/"
+    path: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/"
     state: directory
 - name: Write buildset registry TLS certificate
   become: true
   copy:
     content: "{{ buildset_registry.cert }}"
-    dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}/ca.crt"
+    dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.port }}/ca.crt"
 - name: Write proxy registry TLS certificate
   become: true
   copy:
     content: "{{ buildset_registry.cert }}"
-    dest: "/etc/docker/certs.d/{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}/ca.crt"
+    dest: "/etc/docker/certs.d/{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}/ca.crt"
 
 # Update daemon config
 - name: Check if docker daemon configuration exists
@@ -46,7 +66,7 @@
 - name: Add registry to docker daemon configuration
   vars:
     new_config:
-      registry-mirrors: "['https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port}}/']"
+      registry-mirrors: "['https://{{ buildset_registry_alias }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port}}/']"
   set_fact:
     docker_config: "{{ docker_config | combine(new_config) }}"
 - name: Save docker daemon configuration

roles/use-buildset-registry/tasks/user-config.yaml

@@ -29,9 +29,9 @@
         {
           "https://index.docker.io/v1/":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
-          "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.port }}":
+          "{{ buildset_registry_alias }}:{{ buildset_registry.port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
-          "{{ buildset_registry.host | ipwrap }}:{{ buildset_registry.proxy_port }}":
+          "{{ buildset_registry_alias }}:{{ buildset_registry.proxy_port }}":
             {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
         }
   set_fact: