use-buildset-registry: support microk8s

This enables microk8s/containerd to pull through the intermediate zuul
registry.  This is tested with the new
zuul-jobs-test-registry-buildset-registry-k8s-microk8s job.

Change-Id: I5a6c0d63a6ba0acf94ab9f0ef94777fab58fec6e

roles/use-buildset-registry/tasks/main.yaml

@@ -144,9 +144,30 @@
     state: restarted
   become: true
 
+# microk8s (containerd) setup
+- name: Check for microk8s
+  stat:
+    path: '/var/snap/microk8s'
+  register: _microk8s
+- name: Setup microk8s mirrors
+  when: _microk8s.stat.exists
+  become: yes
+  block:
+    - name: Setup mirrors
+      include_tasks: microk8s-mirror.yaml
+      loop: '{{ buildset_registry_namespaces }}'
+      loop_control:
+        loop_var: zj_uk8s_mirror
+
+    # NOTE(ianw) 2022-12-13 : I don't think this is strictly necessary
+    # when updating mirror configs.  It also shouldn't hurt, so leave
+    # it for now.
+    - name: Restart microk8s
+      command: snap restart microk8s
+
 - name: Wait for kubernetes connection to come back
   command: timeout 10s kubectl get pods
-  when: kubelet_config.stat.exists or crio_path.stat.exists
+  when: kubelet_config.stat.exists or crio_path.stat.exists or _microk8s.stat.exists
   register: _api_ready
   until: _api_ready.rc == 0
   retries: 6

roles/use-buildset-registry/tasks/microk8s-mirror.yaml

@@ -0,0 +1,28 @@
+- name: Make config dir
+  file:
+    path: /var/snap/microk8s/current/args/certs.d/{{ zj_uk8s_mirror[0] }}
+    state: directory
+    owner: 'root'
+    group: 'root'
+    mode: 0755
+
+# NOTE(ianw) : I'm not 100% if microk8s can globally reference this
+# because of it's snap bundling.  Copy it and reference directly
+- name: Copy buildset registry CA
+  copy:
+    src: '{{ ca_dir }}/{{ buildset_registry_alias }}.crt'
+    dest: '/var/snap/microk8s/current/args/certs.d/{{ zj_uk8s_mirror[0] }}/ca.crt'
+    remote_src: true
+
+- name: Write content
+  copy:
+    dest: '/var/snap/microk8s/current/args/certs.d/{{ zj_uk8s_mirror[0] }}/hosts.toml'
+    content: |
+      server = "{{ zj_uk8s_mirror[1] }}"
+      [host."{{ buildset_registry_alias }}:{{ buildset_registry.port }}/v2/{{ zj_uk8s_mirror [0]}}"]
+      capabilities = ["pull", "resolve"]
+      ca = "/var/snap/microk8s/current/args/certs.d/{{ zj_uk8s_mirror[0] }}/ca.crt"
+      override_path = true
+    owner: root
+    group: root
+    mode: 0644

test-playbooks/registry/buildset-registry-k8s-microk8s.yaml

@@ -0,0 +1,51 @@
+- hosts: all
+  roles:
+    - role: clear-firewall
+    - role: ensure-kubernetes
+      vars:
+        ensure_kubernetes_type: microk8s
+    - role: use-buildset-registry
+      buildset_registry_docker_user: root
+  tasks:
+    - name: Wait for cluster to come up
+      command: kubectl cluster-info
+      register: result
+      until: result.rc == 0
+      retries: 5
+      delay: 30
+
+    - name: Run a local test pod
+      command: kubectl run --image=quay.io/zuul/quay-testimage quaytest
+
+    - name: Pause
+      pause:
+        seconds: 60
+
+    - name: Describe pod
+      shell: kubectl describe pods quaytest
+
+    - name: Wait for the pod to be ready
+      command: kubectl wait --for=condition=Ready pod/quaytest --timeout=60s
+
+    - name: Check the output of the pod
+      shell: "kubectl logs pod/quaytest | grep 'Zuul container test'"
+      args:
+        warn: false
+
+    - name: Run a remote test pod
+      command: kubectl run --image=docker.io/debian:testing upstream-dockertest --command -- /bin/bash -c 'echo Upstream; sleep infinity'
+
+    - name: Pause
+      pause:
+        seconds: 60
+
+    - name: Describe pod
+      shell: kubectl describe pods upstream-dockertest
+
+    - name: Wait for the pod to be ready
+      command: kubectl wait --for=condition=Ready pod/upstream-dockertest --timeout=60s
+
+    - name: Check the output of the pod
+      shell: "kubectl logs pod/upstream-dockertest | grep 'Upstream'"
+      args:
+        warn: false

zuul-tests.d/container-roles-jobs.yaml

@@ -286,6 +286,7 @@
       - roles/run-buildset-registry/.*
       - roles/use-buildset-registry/.*
       - test-playbooks/registry/.*
+      - zuul-tests.d/container-roles-jobs.yaml
     pre-run: test-playbooks/registry/buildset-registry-pre.yaml
     run: test-playbooks/registry/buildset-registry.yaml
     post-run: test-playbooks/registry/test-registry-post.yaml
@@ -345,6 +346,27 @@
         - name: controller
           label: centos-7
 
+- job:
+    name: zuul-jobs-test-registry-buildset-registry-k8s-microk8s
+    dependencies: zuul-jobs-test-registry-buildset-registry
+    description: |
+      Test a buildset registry with microk8s
+
+      It is not meant to be used directly but rather run on changes
+      to roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/ensure-docker/.*
+      - roles/ensure-package-repositories/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    run: test-playbooks/registry/buildset-registry-k8s-microk8s.yaml
+    post-run:
+      - test-playbooks/registry/test-registry-post.yaml
+
 - job:
     name: zuul-jobs-test-ensure-kubernetes-crio
     description: |
@@ -557,6 +579,7 @@
         - zuul-jobs-test-registry-buildset-registry
         - zuul-jobs-test-registry-buildset-registry-k8s-crio
         - zuul-jobs-test-registry-buildset-registry-openshift-docker
+        - zuul-jobs-test-registry-buildset-registry-k8s-microk8s
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
@@ -589,6 +612,7 @@
         - zuul-jobs-test-registry-buildset-registry
         - zuul-jobs-test-registry-buildset-registry-k8s-crio
         - zuul-jobs-test-registry-buildset-registry-openshift-docker
+        - zuul-jobs-test-registry-buildset-registry-k8s-microk8s
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy