Bump the default ensure-kubernetes microk8s version to 1.31/stable

Previously we pinned to 1.28/stable due to a bug that prevented
1.29/stable from working. Now we've hit a new issue with 1.28/stable on
bookworm. The fix for that appears to simply be to upgrade to
1.31/stable so we do so here. More details can be found in this GitHub
issue:

  https://github.com/canonical/microk8s/issues/4361

The new version appears to return from the snap installation before the
k8s installation is fully ready to deal with add-on installation. This
occasionally produces errors like:

  subprocess.CalledProcessError:
  Command '('/snap/microk8s/7178/microk8s-kubectl.wrapper', 'get',
            'all,ingress', '--all-namespaces')'
  returned non-zero exit status 1.

Work around that with `microk8s status --wait-ready` to ensure that k8s
is up before adding addons.

While we are at it we also update the collect-kubernetes-logs role to
collect microk8s inspect output as that would've enabled us to debug the
above issue without holding nodes. We also update test jobs to trigger
when the collect-kubernetes-logs and collect-container-logs roles are
updated to ensure we get coverage from those jobs when updating these
roles.

Change-Id: I60022ec6468c2cadd723a71bbc583f20096b27dc

roles/collect-kubernetes-logs/tasks/main.yaml

@@ -42,3 +42,19 @@
     dest: "{{ ansible_user_dir }}/zuul-output/logs/kubelet"
     mode: u=rwX,g=rX,o=rX
     recurse: yes
+
+- name: Collect microk8s inspect tarball
+  shell: |
+    set -x -o pipefail
+    # The microk8s snap isnt in path by default on debian (it is on ubuntu)
+    export PATH=/snap/bin:$PATH
+    if type -p microk8s ; then
+      mkdir {{ ansible_user_dir }}/zuul-output/logs/microk8s
+      INSPECT_TARBALL="$(microk8s inspect | tee {{ ansible_user_dir }}/zuul-output/logs/microk8s/inspect.log | grep '  Report tarball is at' | cut -d' ' -f7)"
+      if [[ -n "$INSPECT_TARBALL" ]] ; then
+        cp $INSPECT_TARBALL {{ ansible_user_dir }}/zuul-output/logs/microk8s/
+      fi
+    fi
+  args:
+    executable: /bin/bash
+  failed_when: false

roles/ensure-kubernetes/defaults/main.yaml

@@ -1,7 +1,5 @@
 ensure_kubernetes_type: minikube
-# Pin to 1.28 until https://github.com/canonical/microk8s/issues/4361
-# is fixed.
-ensure_kubernetes_microk8s_channel: '1.28/stable'
+ensure_kubernetes_microk8s_channel: '1.31/stable'
 # NOTE(ianw) : 2022-12-13
 #  - "storage" is deprecated and has become "hostpath-storage" in
 #     >1.24, but we still need to support 1.23.  If it really goes away

roles/ensure-kubernetes/tasks/microk8s.yaml

@@ -13,6 +13,10 @@
   become: yes
   command: 'snap install microk8s --classic --channel={{ ensure_kubernetes_microk8s_channel }}'
 
+- name: Wait for kubernetes to be ready before adding add-ons
+  become: yes
+  command: '/snap/bin/microk8s status --wait-ready'
+
 - name: Install add-ons
   become: yes
   command: '/snap/bin/microk8s enable {{ ensure_kubernetes_microk8s_addons | join(" ") }}'

zuul-tests.d/container-roles-jobs.yaml

@@ -385,6 +385,8 @@
     # Does not work under ubuntu-bionic
     abstract: true
     files:
+      - roles/collect-container-logs/.*
+      - roles/collect-kubernetes-logs/.*
       - roles/ensure-docker/.*
       - roles/ensure-kubernetes/.*
       - roles/ensure-package-repositories/.*
@@ -412,6 +414,8 @@
     # Requires at least ubuntu-noble
     abstract: true
     files:
+      - roles/collect-container-logs/.*
+      - roles/collect-kubernetes-logs/.*
       - roles/ensure-docker/.*
       - roles/ensure-kubernetes/.*
       - roles/ensure-package-repositories/.*
@@ -438,6 +442,8 @@
       roles in the zuul-jobs repo.
     abstract: true
     files:
+      - roles/collect-container-logs/.*
+      - roles/collect-kubernetes-logs/.*
       - roles/ensure-docker/.*
       - roles/ensure-kubernetes/.*
       - roles/ensure-package-repositories/.*