Add DNS sanity checks to k8s deploy script

Check that k8s DNS is working, and terminate at the beginning if this is
not the case.

Change-Id: I30867671f39dd9d80f46f5a4381adc9d34df7ab7

tools/deployment/common/000-install-packages.sh

@@ -22,4 +22,5 @@ sudo apt-get install --no-install-recommends -y \
         nmap \
         curl \
         bc \
-        python3-pip
+        python3-pip \
+        dnsutils

tools/gate/deploy-k8s.sh

@@ -18,6 +18,7 @@ set -ex
 : "${MINIKUBE_VERSION:="v1.22.0"}"
 : "${CALICO_VERSION:="v3.20"}"
 : "${YQ_VERSION:="v4.6.0"}"
+: "${KUBE_DNS_IP="10.96.0.10"}"
 
 export DEBCONF_NONINTERACTIVE_SEEN=true
 export DEBIAN_FRONTEND=noninteractive
@@ -33,7 +34,7 @@ function configure_resolvconf {
   # to coredns via kubelet.resolv-conf extra param
   # 2 - /etc/resolv.conf - to be used for resolution on host
 
-  kube_dns_ip="10.96.0.10"
+  kube_dns_ip="${KUBE_DNS_IP}"
   # keep all nameservers from both resolv.conf excluding local addresses
   old_ns=$(grep -P --no-filename "^nameserver\s+(?!127\.0\.0\.|${kube_dns_ip})" \
            /etc/resolv.conf /run/systemd/resolve/resolv.conf | sort | uniq)
@@ -213,6 +214,13 @@ until kubectl --namespace=kube-system \
 done
 kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
 
+# Validate DNS now to save a lot of headaches later
+sleep 5
+if ! dig svc.cluster.local ${KUBE_DNS_IP} | grep ^cluster.local. >& /dev/null; then
+  echo k8s DNS Failure. Are you sure you disabled systemd-resolved before running this script?
+  exit 1
+fi
+
 # Remove stable repo, if present, to improve build time
 helm repo remove stable || true