Clear pods in OutOfhugepages* state

Following an upgrade, some pods using
hugepages will still be in Running
state, but will have a replica that stays in
OutOfhugepages state.

k8s-pod-recovery can detect those pods and
delete them.

Closes-bug: 1943113
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Change-Id: Idba510cabd66cd8b796563e3e6efa9baa5b4401e

kubernetes/k8s-pod-recovery/centos/files/k8s-pod-recovery

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2020 Wind River Systems, Inc.
+# Copyright (c) 2020-2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -131,6 +131,35 @@ function _unknown_pods {
     fi
 }
 
+function _outofhugepages_pods {
+    # $1: actions <recover|verify>
+
+    # Target all namespaces and pods on this host
+    NAMESPACES=$(kubectl get ns | tail -n +2 | awk '{ print $1 }')
+
+    if [ "$1" == 'recover' ]; then
+        # Recovers pods that are: Running/OutOfhugepages
+        for ns in ${NAMESPACES[@]}; do
+            PODS=$(kubectl get pods -n $ns --field-selector spec.nodeName=${HOST} 2>/dev/null | awk /OutOfhugepages/'{print $1}')
+            for pod in $PODS ; do
+                LOG "OutOfhugepages pods: Recovering: $ns/$pod"
+                kubectl delete pods -n $ns $pod --wait=false
+            done
+        done
+    elif [ "$1" == 'verify' ]; then
+        for ns in ${NAMESPACES[@]}; do
+            PODS=$(kubectl get pods -n $ns --field-selector spec.nodeName=${HOST} 2>/dev/null | awk /OutOfhugepages/'{print $1}')
+            if [ -z "${PODS}" ]; then
+                LOG "OutOfhugepages pods: None present for namespace: $ns"
+            else
+                ERROR "OutOfhugepages pods: still present for namespace: $ns"
+            fi
+        done
+    else
+        ERROR "Unknown action: $1"
+    fi
+}
+
 function _node_affinity_pods {
     # $1: actions <recover|verify>
 
@@ -256,6 +285,7 @@ function _examine_pods {
     _unknown_pods $1
     _node_affinity_pods $1
     _force_reset_pods $1
+    _outofhugepages_pods $1
 }