From e89e5dcb1515372afe4c85c0e60388a9cd091bb0 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 20 Nov 2024 09:11:38 -0800
Subject: [PATCH] Update system packages and reboot when building centos
 openafs

We often hit issues with our CentOS arm images being slightly stale.
When this happens we get openafs package build errors because available
headers don't match the running kernel. Address this by updating the
entire running system and rebooting onto any new kernel before we build
the openafs package. This should ensure our local install matches
upstream packages when it comes to kernel packages.

We also manually select the new kernel as the default kernel and remove
the old kernel with grubby as dnf doesn't reliably update the kernel to
boot automatically on arm64 centos 9 images. This is likely due to this
issue:

  https://bugzilla.redhat.com/show_bug.cgi?id=2032680

Also switch package module installs to dnf module installs since all of
our installations are occuring on CentOS 9 or newer.

Change-Id: Idb42386071d6652312bb343923473a62c55e2828
---
 playbooks/openafs-rpm-package-build/pre.yaml  | 47 +++++++++++++++++++
 .../openafs-rpm-package-build/tasks/main.yaml |  6 +--
 zuul.d/jobs.yaml                              |  1 +
 3 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 playbooks/openafs-rpm-package-build/pre.yaml

diff --git a/playbooks/openafs-rpm-package-build/pre.yaml b/playbooks/openafs-rpm-package-build/pre.yaml
new file mode 100644
index 00000000..96ceb536
--- /dev/null
+++ b/playbooks/openafs-rpm-package-build/pre.yaml
@@ -0,0 +1,47 @@
+# Update system packages and reboot to ensure we're running a kernel that
+# matches what available headers in package mirrors.
+- name: Update and reboot nodes before installing openafs
+  hosts: all
+  tasks:
+    - name: DNF Update
+      dnf:
+        name: "*"
+        state: latest  # noqa: package-latest
+      become: yes
+
+    - name: Hacky script to force default kernel to new version
+      shell: |
+        set -x
+        # Get the newest kernel version in /boot
+        NEWEST=$(ls /boot | grep vmlinuz | sort -V -r | head -1)
+        OLDEST=$(ls /boot | grep vmlinuz | sort -V | head -1)
+        grubby --set-default=/boot/$NEWEST
+        if [[ "$OLDEST" != "$NEWEST" ]] ; then
+          grubby --remove-kernel=/boot/$OLDEST
+        fi
+      args:
+        executable: /usr/bin/bash
+      become: yes
+
+    - name: Tell grub about the new kernel setup
+      command: grub2-mkconfig --update-bls-cmdline -o /boot/grub2/grub.cfg
+      become: yes
+
+    - name: Record running kernel version
+      command: uname -a
+
+    - name: Reboot
+      reboot:
+        reboot_timeout: 900
+      become: yes
+
+    - name: Restart zuul console log daemon
+      include_role:
+        name: start-zuul-console
+
+    - name: Record running kernel version
+      command: uname -a
+
+    - name: Pause for a bit to ensure system is up post reboot
+      pause:
+        seconds: 60
diff --git a/roles/openafs-rpm-package-build/tasks/main.yaml b/roles/openafs-rpm-package-build/tasks/main.yaml
index 2aa9d0e7..f2fc82ca 100644
--- a/roles/openafs-rpm-package-build/tasks/main.yaml
+++ b/roles/openafs-rpm-package-build/tasks/main.yaml
@@ -1,5 +1,5 @@
 - name: Install prereqs
-  package:
+  dnf:
     name:
       - rpm-build
       - kernel-devel
@@ -7,14 +7,14 @@
   become: yes
 
 - name: Install centos 8/9 common packages
-  package:
+  dnf:
     name:
       - elfutils-devel
   become: yes
   when: ansible_distribution_major_version is version('8', '>=')
 
 - name: Install centos9 packages
-  package:
+  dnf:
     name:
       - perl-File-Copy
   become: yes
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 47678550..6429716c 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -1356,6 +1356,7 @@
       There are no official builds for AFS on Centos, hence we build
       our own and publish them to tarballs.openstack.org for our
       centos hosts that need access to the mirror to consume.
+    pre-run: playbooks/openafs-rpm-package-build/pre.yaml
     run: playbooks/openafs-rpm-package-build/run.yaml
     files: &afs-files
       - playbooks/openafs-rpm-package-build