From 99678c46e04e703119fb082d804b9626e629ee1d Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 8 Dec 2022 14:53:52 +1100 Subject: [PATCH] ensure-kubernetes: add microk8s support Add microk8s support to the ensure-kubernetes role. This installs via a snap, and is currently only implemented for Ubuntu Jammy. Mostly this is a straight-forward installation. I did notice though it needs a little more time to be stable in the test, so the timeout is bumped slightly. microk8s is the Ubuntu "blessed" way of doing things. This should be a better choice for Ubuntu platforms, because minikube is tightly tied to cri-o, which is only packaged through kubic, which is currently in some sort of deprecated but best-effort supported mode [1]. This was inspired by an outage where the kubic gpg expired. This appears fixed now. [1] https://kubic.opensuse.org/blog/2022-06-10-kubic-retired/ Change-Id: Id3e31c70a35dde218e35e7c50964f8a3c0348150 --- roles/ensure-kubernetes/README.rst | 23 +++++++- roles/ensure-kubernetes/defaults/main.yaml | 9 +++ roles/ensure-kubernetes/tasks/main.yaml | 45 ++++++++------ roles/ensure-kubernetes/tasks/microk8s.yaml | 58 +++++++++++++++++++ .../ensure-kubernetes/microk8s.yaml | 6 ++ test-playbooks/ensure-kubernetes/post.yaml | 11 +++- zuul-tests.d/container-roles-jobs.yaml | 30 ++++++++++ 7 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 roles/ensure-kubernetes/tasks/microk8s.yaml create mode 100644 test-playbooks/ensure-kubernetes/microk8s.yaml diff --git a/roles/ensure-kubernetes/README.rst b/roles/ensure-kubernetes/README.rst index 22348ac92..37f51a0b0 100644 --- a/roles/ensure-kubernetes/README.rst +++ b/roles/ensure-kubernetes/README.rst @@ -2,6 +2,25 @@ An ansible role to install kubernetes. **Role Variables** +.. zuul:rolevar:: ensure_kubernetes_type + :default: minikube + + The kubernetes distribution to use. Currently ```minikube`` or + ```microk8s```. Note that ```microk8s``` is only implemented for + Ubuntu Jammy distributions currently. + +.. zuul:rolevar:: ensure_kubernetes_microk8s_channel + :default: latest/stable + + The ``snap`` channel to use for ```microk8s```. See + ``__. + +.. zuul:rolevar:: ensure_kubernetes_microk8s_addons + :default: ['dns', 'storage'] + + The addons for ``microk8s```. See + ``__ + .. zuul:rolevar:: install_kubernetes_with_cluster :default: True @@ -21,10 +40,10 @@ An ansible role to install kubernetes. .. zuul:rolevar:: kubernetes_runtime :default: docker - Which kubernetes runtime to use; values are ``docker`` or + Which kubernetes runtime to use for minikube; values are ``docker`` or ``cri-o``. .. zuul:rolevar:: ensure_kubernetes_minikube_addons :default: [] - List of addons to configure in k8s. Use this to enable the addons. \ No newline at end of file + List of addons to configure in k8s. Use this to enable the addons. diff --git a/roles/ensure-kubernetes/defaults/main.yaml b/roles/ensure-kubernetes/defaults/main.yaml index 1bc8767f9..63283d5a2 100644 --- a/roles/ensure-kubernetes/defaults/main.yaml +++ b/roles/ensure-kubernetes/defaults/main.yaml @@ -1,3 +1,12 @@ +ensure_kubernetes_type: minikube +ensure_kubernetes_microk8s_channel: 'latest/stable' +# NOTE(ianw) : 2022-12-13 +# - "storage" is deprecated and has become "hostpath-storage" in +# >1.24, but we still need to support 1.23. If it really goes away +# we might need to switch on the channel name +ensure_kubernetes_microk8s_addons: + - dns + - storage install_kubernetes_with_cluster: True minikube_version: latest minikube_dns_resolvers: [] diff --git a/roles/ensure-kubernetes/tasks/main.yaml b/roles/ensure-kubernetes/tasks/main.yaml index 25535ab8c..f5bf559c3 100644 --- a/roles/ensure-kubernetes/tasks/main.yaml +++ b/roles/ensure-kubernetes/tasks/main.yaml @@ -1,21 +1,28 @@ -- name: Add all repositories - include_role: - name: ensure-package-repositories - vars: - repositories_keys: - - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - repositories_list: - - repo: deb http://apt.kubernetes.io/ kubernetes-xenial main +- name: Install microk8s + when: ensure_kubernetes_type == 'microk8s' + include_tasks: microk8s.yaml -- name: Install kubernetes packages and dependencies - become: yes - apt: - name: - - conntrack - - kubectl - - liblz4-tool - update_cache: yes +- name: Install minikube + when: ensure_kubernetes_type == 'minikube' + block: + - name: Add all repositories + include_role: + name: ensure-package-repositories + vars: + repositories_keys: + - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + repositories_list: + - repo: deb http://apt.kubernetes.io/ kubernetes-xenial main -- name: Run cluster tasks - include_tasks: minikube.yaml - when: install_kubernetes_with_cluster + - name: Install kubernetes packages and dependencies + become: yes + apt: + name: + - conntrack + - kubectl + - liblz4-tool + update_cache: yes + + - name: Run cluster tasks + include_tasks: minikube.yaml + when: install_kubernetes_with_cluster diff --git a/roles/ensure-kubernetes/tasks/microk8s.yaml b/roles/ensure-kubernetes/tasks/microk8s.yaml new file mode 100644 index 000000000..cc9eed699 --- /dev/null +++ b/roles/ensure-kubernetes/tasks/microk8s.yaml @@ -0,0 +1,58 @@ +- name: Check distro + assert: + that: ansible_distribution_release == 'jammy' + msg: 'This role only supported on Jammy' + +- name: Install snapd + become: yes + package: + name: snapd + state: present + +- name: Install microk8s snap + become: yes + command: 'snap install microk8s --classic --channel={{ ensure_kubernetes_microk8s_channel }}' + +- name: Install add-ons + become: yes + command: '/snap/bin/microk8s enable {{ ensure_kubernetes_microk8s_addons | join(" ") }}' + +- name: Install kubectl wrapper + become: yes + copy: + content: | + #!/bin/bash + /snap/bin/microk8s.kubectl "$@" + dest: /usr/local/bin/kubectl + owner: root + group: root + mode: 0755 + +- name: Allow zuul user kubectl access + become: yes + user: + name: 'zuul' + groups: 'microk8s' + append: yes + +- name: Create .kube directory + file: + path: "{{ ansible_user_dir }}/.kube" + state: directory + mode: 0755 + +- name: Populate kube config + become: yes + shell: /snap/bin/microk8s config > {{ ansible_user_dir }}/.kube/config + +- name: Ensure we can read config + become: yes + file: + path: "{{ ansible_user_dir }}/.kube/config" + owner: zuul + group: zuul + state: touch + mode: 0644 + +- name: Reset connection to pickup group + meta: reset_connection diff --git a/test-playbooks/ensure-kubernetes/microk8s.yaml b/test-playbooks/ensure-kubernetes/microk8s.yaml new file mode 100644 index 000000000..79630d392 --- /dev/null +++ b/test-playbooks/ensure-kubernetes/microk8s.yaml @@ -0,0 +1,6 @@ +- hosts: all + name: Install kubernetes with minikube + roles: + - role: ensure-kubernetes + vars: + ensure_kubernetes_type: microk8s diff --git a/test-playbooks/ensure-kubernetes/post.yaml b/test-playbooks/ensure-kubernetes/post.yaml index 441f27bcf..76c5d0bda 100644 --- a/test-playbooks/ensure-kubernetes/post.yaml +++ b/test-playbooks/ensure-kubernetes/post.yaml @@ -29,12 +29,19 @@ - name: Start pod command: kubectl apply -f test-pod.yaml + - name: Wait a bit + pause: + seconds: 30 + + - name: Describe pod + shell: sleep 5; kubectl describe pods test + - name: Ensure pod is running shell: sleep 5; kubectl get pods register: _get_pods_output until: "'Running' in _get_pods_output.stdout" retries: 3 - delay: 5 + delay: 10 always: - name: Collect container logs @@ -47,6 +54,6 @@ - name: Get minikube logs become: true - shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt" + shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt || true" environment: MINIKUBE_HOME: "{{ ansible_user_dir }}" diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index dc150f7c0..3032ba842 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -392,6 +392,34 @@ - name: ubuntu-jammy label: ubuntu-jammy +- job: + name: zuul-jobs-test-ensure-kubernetes-microk8s + description: | + Test the ensure-kubernetes role with microk8s + + This job tests changes to the ensure-kubernetes roles. It + is not meant to be used directly but rather run on changes to + roles in the zuul-jobs repo. + abstract: true + files: + - roles/ensure-docker/.* + - roles/ensure-kubernetes/.* + - roles/ensure-package-repositories/.* + - test-playbooks/ensure-kubernetes/.* + run: test-playbooks/ensure-kubernetes/microk8s.yaml + post-run: test-playbooks/ensure-kubernetes/post.yaml + +# NOTE(ianw) 2022-12-08 : only supported on jammy + +- job: + name: zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy + description: Test the ensure-kubernetes role with docker on ubuntu-jammy + parent: zuul-jobs-test-ensure-kubernetes-microk8s + nodeset: + nodes: + - name: ubuntu-jammy + label: ubuntu-jammy + - job: name: zuul-jobs-test-ensure-podman description: | @@ -532,6 +560,7 @@ - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy + - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy - zuul-jobs-test-ensure-skopeo-ubuntu-bionic - zuul-jobs-test-ensure-skopeo-ubuntu-focal - zuul-jobs-test-ensure-skopeo-ubuntu-jammy @@ -563,6 +592,7 @@ - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy + - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy - zuul-jobs-test-ensure-skopeo-ubuntu-bionic - zuul-jobs-test-ensure-skopeo-ubuntu-focal - zuul-jobs-test-ensure-skopeo-ubuntu-jammy