From 0d5f87e08f721db50700ce31a480ce63842cd81a Mon Sep 17 00:00:00 2001 From: kdhokte Date: Fri, 19 Nov 2021 18:14:21 -0500 Subject: [PATCH] update kubeadm configmap with new feature gate The script will run for every k8s upgrade as a part of the control-plane upgrade of the first master. It updates kubeadm-config configmap to configure kube-apiserver manifest with RemoveSelfLink=false. The kubelet config override file is consumed by kubeadm upgrade apply to configure cgroupDriver=cgroupfs. Kubernetes changed default cgroupDriver cgroupfs to systemd from k8s 1.21. We need to configure cgroupDriver with 'cgroupfs' in the kubelet config otherwise kubelet will fail to launch. Kubernetes changed kube-apiserver feature-gate RemoveSelfLink default value to true from k8s 1.20 onwards. This caused PVCs to not bound to PVs and eventually caused some system apps to fail. We need to configure the kube-apiserver feature-gates with RemoveSelfLink=false to avoid it until we can update the application code to handle the new behaviour. Test Plan: PASS: successful testing of the script during k8s version upgrades from 1.18 -> 1.19 -> 1.20 -> 1.21 on AIO-DX system. Verified that PVCs are bound to PVs after each upgrade. Verified that pods are up and running after each upgrade. Verified that kube-apiserver manifest and kubelet config file is configured as expected after each upgrade. PASS: 'build-pkgs --dep-test kubernetes-unversioned' package built with dependencies passed Story: 2008972 Task: 44037 Signed-off-by: kdhokte Change-Id: Ie53ebb5839eb0bd843f5bb2c577a4292969c0997 --- .../centos/files/kubelet_override.yaml | 4 + .../centos/files/upgrade_k8s_config.sh | 85 +++++++++++++++++++ .../centos/kubernetes-unversioned.spec | 15 ++++ 3 files changed, 104 insertions(+) create mode 100644 kubernetes/kubernetes-unversioned/centos/files/kubelet_override.yaml create mode 100755 kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh diff --git a/kubernetes/kubernetes-unversioned/centos/files/kubelet_override.yaml b/kubernetes/kubernetes-unversioned/centos/files/kubelet_override.yaml new file mode 100644 index 000000000..d6f2784aa --- /dev/null +++ b/kubernetes/kubernetes-unversioned/centos/files/kubelet_override.yaml @@ -0,0 +1,4 @@ +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: cgroupfs diff --git a/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh new file mode 100755 index 000000000..3aaa5edd8 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This will run for every k8s upgrade as a part of the control-plane upgrade of the first master. +# - updates kubeadm-config configmap to configure kube-apiserver manifest with RemoveSelfLink=false. +# - generates a kubelet config override file to configure cgroupDriver=cgroupfs. +# This is consumed by kubeadm upgrade apply +# +# Background: +# Kubernetes 1.21 changed cgroupDriver default to systemd (was cgroupfs). +# Kubernetes 1.20 changed feature-gates RemoveSelfLink default to true. + + +KUBEADM_CONFIGMAP_TMPFILE='/tmp/kubeadm_cm.yaml' + +function log { + logger -p local1.info "$1" +} + +# Update the configmap for kubeadm +function update_apiserver_configmap { + + log "Retrieving kubeadm configmap to temporary location: ${KUBEADM_CONFIGMAP_TMPFILE}" + counter=0 + RC=0 + RETRIES=10 + until [ $counter -gt $RETRIES ]; do + kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ + configmap kubeadm-config -o yaml > ${KUBEADM_CONFIGMAP_TMPFILE} + RC=$? + if [ "$RC" = "0" ] ; then + log "Kubeadm configmap retrieved." + break + fi + log "Error retrieving kubeadm configmap, retrying..." + sleep 5 + let "counter+=1" + done + + if [ $counter -gt $RETRIES ]; then + log "Failed to retrieve kubeadm configmap with error code [$RC]". + exit $RC + fi + + if ! grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; then + + log "Updating kube-apiserver feature-gates in retrieved kubeadm-config" + + if sed -i \ +'/^\s*feature-gates:\s*.*HugePageStorageMediumSize='\ +'true/ s/$/,RemoveSelfLink=false/' ${KUBEADM_CONFIGMAP_TMPFILE}; then + + if grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; + then + log "Successfully updated retrieved kubeadm-config" + if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \ + ${KUBEADM_CONFIGMAP_TMPFILE}; then + log 'Successfully replaced updated kubeadm configmap.' + else + RC=$? + log "Failed to replace updated kubeadm configmap with error code: [$RC]" + exit $RC + fi + else + log 'Failed to update kube-apiserver feature-gates with an unknown error' + exit -1 + fi + else + RC=$? + log "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]" + exit $RC + fi + else + log "Kubeadm configmap was already updated with RemoveSelfLink=false. Nothing to do." + fi + + rm -f ${KUBEADM_CONFIGMAP_TMPFILE} + +} + +update_apiserver_configmap +exit 0 diff --git a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec index cee00db5d..c9136ef0a 100644 --- a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec +++ b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec @@ -5,6 +5,7 @@ # %define debug_package %{nil} +%define local_sbindir /usr/local/sbin %global _k8s_name kubernetes @@ -35,6 +36,11 @@ Source1: %{con_repo}-v%{con_commit}.tar.gz # systemd resource control enable CPU and Memory accounting for cgroups Source2: kubernetes-accounting.conf +# kubelet config overrides parameters +Source3: kubelet_override.yaml + +Source4: upgrade_k8s_config.sh + Patch1: kubelet-service-remove-docker-dependency.patch BuildArch: noarch @@ -89,6 +95,11 @@ install -v -d -m 0755 %{buildroot}%{_tmpfilesdir} install -v -p -m 0644 -t %{buildroot}/%{_tmpfilesdir} contrib/init/systemd/tmpfiles.d/kubernetes.conf mkdir -p %{buildroot}/run install -v -d -m 0755 %{buildroot}/run/%{_k8s_name}/ +install -p -D -m 644 %{SOURCE3} %{buildroot}%{_sysconfdir}/%{_k8s_name}/kubelet_override.yaml + +install -d %{buildroot}%{local_sbindir} +# install execution scripts +install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh # install service files install -v -d -m 0755 %{buildroot}%{_unitdir} @@ -107,6 +118,9 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU %dir %{_curr_stage1} %dir %{_curr_stage2} +# the following are execution scripts +%{local_sbindir}/upgrade_k8s_config.sh + # the following are symlinks %{_bindir}/kubeadm %{_bindir}/kubelet @@ -123,6 +137,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU %config(noreplace) %{_sysconfdir}/%{_k8s_name}/config %config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet %config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet.kubeconfig +%config(noreplace) %{_sysconfdir}/%{_k8s_name}/kubelet_override.yaml %config(noreplace) %{_sysconfdir}/%{_k8s_name}/proxy %config(noreplace) %{_sysconfdir}/systemd/system.conf.d/kubernetes-accounting.conf %{_tmpfilesdir}/kubernetes.conf