From c9b781b7c022e619a8bed2c58e686c989c4ca9eb Mon Sep 17 00:00:00 2001 From: kdhokte Date: Tue, 1 Feb 2022 21:09:03 -0500 Subject: [PATCH] sanitize reserved cpus list before kubelet starts The script will run everytime before the kubelet service is started. It reads the reserved-cpus list for the kubelet from the service environment file and sanitizes it on the basis of online CPUs. If none of the reserved cpus is online, it removes the --reserved-cpus flag from the environment file which allows the kubelet to choose CPUs itself. Sanitizing the reserved-cpus list everytime before the kubelet starts assures that the kubelet will not fail to start due to unavailability of one or more CPUs in the list. By enabling or disabling CPU hyperthreading, available CPUs change. This change will make sure changing CPU hyperthreading setting will not lead to kubelet start failure after the system boots up. Test Plan: (On AIO-SX) PASS: Initial Hyperthreading state: enabled Host-lock->Reboot->Disable CPU hyperthreading and reboot->Host-unlock Observe kubelet does not fail to start before host-unlock. All pods states are as expected. Host-unlock succeeds. PASS: Initial Hyperthreading state: disabled Host-lock->Reboot->Enable CPU hyperthreading and reboot->Host-unlock Observe kubelet does not fail to start before host-unlock. All pods states are as expected. Host-unlock succeeds. PASS: Manually restart the Kubelet service. Observe that the kubelet does not fail to start. All pods states are as expected. PASS: Host-lock->Host unlock (without any config change). Observe that the kubelet does not fail to start. All pods states are as expected. PASS: Packages built successfully on both Debian and CentOS. Closes-Bug: 1955608 Change-Id: I699c5c36a56a50d4c48faa816edad69c17058079 Signed-off-by: Kaustubh Dhokte --- .../centos/files/kubeadm.conf | 1 + .../centos/files/kubeadm.conf | 1 + .../centos/files/kubeadm.conf | 1 + .../centos/files/kubeadm.conf | 1 + .../debian/deb_folder/kubeadm.conf | 1 + .../centos/files/kubeadm.conf | 1 + .../files/sanitize_kubelet_reserved_cpus.sh | 98 +++++++++++++++++++ .../centos/kubernetes-unversioned.spec | 5 + .../deb_folder/kubernetes-unversioned.install | 1 + .../debian/deb_folder/not-installed | 1 + .../debian/deb_folder/rules | 5 + .../sanitize_kubelet_reserved_cpus.sh | 98 +++++++++++++++++++ 12 files changed, 214 insertions(+) create mode 100755 kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh create mode 100755 kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh diff --git a/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf b/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf +++ b/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh b/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh new file mode 100755 index 000000000..1dcdbfdb5 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh @@ -0,0 +1,98 @@ +#! /bin/bash +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# The script will run everytime before the kubelet service is started. +# (Runs as a "ExecStartPre" action) +# +# It reads the reserved-cpus list for the kubelet from the kubelet +# environment file and sanitizes it on the basis of online CPUs. +# +# If none of the reserved cpus is online, it removes the --reserved-cpus flag +# from the environment file which allows the kubelet to choose CPUs itself +# + +ENVIRONMENT_FILE=$1 + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): $@" +} + + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): ERROR: $@" +} + +function sanitize_reserved_cpus { + kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading kubelet extra arguments. Error code: [${RC}]" + exit ${RC} + fi + + # Get reserved-cpus comma-separated-values string from environment file and strip double quotes + # format of kubelet_extra_args is: + # "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2 + # --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000" + if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then + reserved_cpus=${BASH_REMATCH[1]} + else + reserved_cpus="" + fi + if test -z "${reserved_cpus}"; then + LOG "No reserved-cpu list found for kubelet. Nothing to do." + exit 0 + fi + LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}" + + cpus_online=$(cat /sys/devices/system/cpu/online) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading online CPU list. Error code: [${RC}]" + exit ${RC} + fi + LOG "Online CPUs: ${cpus_online}" + + # Possible formats for reserved_cpus could be + # 0,2,4,6 + # 0-23,36-45 + # 0-4,6,9,13,23-34 + expanded_reserved_cpus=$(expand_sequence ${reserved_cpus}) + reserved_cpus_array=(${expanded_reserved_cpus//,/ }) + + sanitized_reserved_cpus="" + for element in "${reserved_cpus_array[@]}"; do + in_list ${element} ${cpus_online} + if [[ "$?" == "0" ]] ; then + sanitized_reserved_cpus+=",${element}" + fi + done + # Remove the extra leading ',' + sanitized_reserved_cpus=${sanitized_reserved_cpus#","} + LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}" + + if test -z "${sanitized_reserved_cpus}"; then + # Strip out --reserved-cpus option if no reserved-cpus are online + sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE} + else + # Replace existing reserved-cpus with sanitized list + sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE} + fi + RC="$?" + if [ ${RC} != "0" ]; then + ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]" + exit ${RC} + fi + LOG "Successfully updated reserved-cpus list for the kubelet." + +} + +source /etc/init.d/cpumap_functions.sh + +sanitize_reserved_cpus + +exit 0 diff --git a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec index c9136ef0a..f9944f10c 100644 --- a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec +++ b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec @@ -41,6 +41,8 @@ Source3: kubelet_override.yaml Source4: upgrade_k8s_config.sh +Source5: sanitize_kubelet_reserved_cpus.sh + Patch1: kubelet-service-remove-docker-dependency.patch BuildArch: noarch @@ -101,6 +103,8 @@ install -d %{buildroot}%{local_sbindir} # install execution scripts install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh +install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh + # install service files install -v -d -m 0755 %{buildroot}%{_unitdir} install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service @@ -120,6 +124,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU # the following are execution scripts %{local_sbindir}/upgrade_k8s_config.sh +%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh # the following are symlinks %{_bindir}/kubeadm diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install index 8e2cb1e78..717ad0279 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install @@ -5,3 +5,4 @@ etc/kubernetes/kubelet.kubeconfig etc/kubernetes/proxy etc/systemd/system.conf.d/kubernetes-accounting.conf usr/lib/tmpfiles.d/kubernetes.conf +usr/local/sbin/sanitize_kubelet_reserved_cpus.sh diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed b/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed index 666a84f38..ae7c9c81b 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed @@ -1,6 +1,7 @@ usr/bin/kubeadm usr/bin/kubelet usr/bin/kubelet-cgroup-setup.sh +usr/local/sbin/sanitize_kubelet_reserved_cpus.sh usr/bin/kubectl etc/systemd/system/kubelet.service.d/kubeadm.conf usr/share/bash-completion/completions/kubectl diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules index 18921fccd..70b19abdc 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules @@ -5,6 +5,7 @@ _k8s_name := kubernetes _bindir := /usr/bin +_local_sbindir := /usr/local/sbin _curr_stage1 := /usr/local/kubernetes/current/stage1 _curr_stage2 := /usr/local/kubernetes/current/stage2 @@ -60,6 +61,10 @@ override_dh_install: install -v -d -m 0755 ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d install -v -p -m 0644 -t ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d debian/kubernetes-accounting.conf + # install scripts + install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir} + install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}/sanitize_kubelet_reserved_cpus.sh + dh_install override_dh_usrlocal: diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh new file mode 100755 index 000000000..1dcdbfdb5 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh @@ -0,0 +1,98 @@ +#! /bin/bash +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# The script will run everytime before the kubelet service is started. +# (Runs as a "ExecStartPre" action) +# +# It reads the reserved-cpus list for the kubelet from the kubelet +# environment file and sanitizes it on the basis of online CPUs. +# +# If none of the reserved cpus is online, it removes the --reserved-cpus flag +# from the environment file which allows the kubelet to choose CPUs itself +# + +ENVIRONMENT_FILE=$1 + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): $@" +} + + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): ERROR: $@" +} + +function sanitize_reserved_cpus { + kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading kubelet extra arguments. Error code: [${RC}]" + exit ${RC} + fi + + # Get reserved-cpus comma-separated-values string from environment file and strip double quotes + # format of kubelet_extra_args is: + # "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2 + # --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000" + if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then + reserved_cpus=${BASH_REMATCH[1]} + else + reserved_cpus="" + fi + if test -z "${reserved_cpus}"; then + LOG "No reserved-cpu list found for kubelet. Nothing to do." + exit 0 + fi + LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}" + + cpus_online=$(cat /sys/devices/system/cpu/online) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading online CPU list. Error code: [${RC}]" + exit ${RC} + fi + LOG "Online CPUs: ${cpus_online}" + + # Possible formats for reserved_cpus could be + # 0,2,4,6 + # 0-23,36-45 + # 0-4,6,9,13,23-34 + expanded_reserved_cpus=$(expand_sequence ${reserved_cpus}) + reserved_cpus_array=(${expanded_reserved_cpus//,/ }) + + sanitized_reserved_cpus="" + for element in "${reserved_cpus_array[@]}"; do + in_list ${element} ${cpus_online} + if [[ "$?" == "0" ]] ; then + sanitized_reserved_cpus+=",${element}" + fi + done + # Remove the extra leading ',' + sanitized_reserved_cpus=${sanitized_reserved_cpus#","} + LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}" + + if test -z "${sanitized_reserved_cpus}"; then + # Strip out --reserved-cpus option if no reserved-cpus are online + sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE} + else + # Replace existing reserved-cpus with sanitized list + sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE} + fi + RC="$?" + if [ ${RC} != "0" ]; then + ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]" + exit ${RC} + fi + LOG "Successfully updated reserved-cpus list for the kubelet." + +} + +source /etc/init.d/cpumap_functions.sh + +sanitize_reserved_cpus + +exit 0