From c7f5c9979c673489541087d2cab19fa08e622e0e Mon Sep 17 00:00:00 2001
From: Renis Makadia <renis.makadia@att.com>
Date: Wed, 13 Feb 2019 11:27:16 -0800
Subject: [PATCH] Add helm tests for Ceph Provisioners chart

- Adding helm tests for Ceph provisioner chart
- Helm test should only executed when deploying chart with
client_secrets: true.

Co-Authored-By: Chinasubbareddy Mallavarapu <cr3938@att.com>

Change-Id: I33421249246dfaf6ea4f835e76a74813dfb3b595
---
 .../templates/bin/_helm-tests.sh.tpl          | 127 ++++++++++++++++++
 .../templates/configmap-bin-provisioner.yaml  |   2 +
 .../templates/pod-helm-tests.yaml             | 108 +++++++++++++++
 ceph-provisioners/values.yaml                 |  14 ++
 .../multinode/035-ceph-ns-activate.sh         |   2 +
 .../openstack-support/025-ceph-ns-activate.sh |   2 +
 .../osh-infra-logging/025-ceph-ns-activate.sh |   2 +
 .../045-tenant-ceph-ns-activate.sh            |   2 +
 8 files changed, 259 insertions(+)
 create mode 100644 ceph-provisioners/templates/bin/_helm-tests.sh.tpl
 create mode 100644 ceph-provisioners/templates/pod-helm-tests.yaml

diff --git a/ceph-provisioners/templates/bin/_helm-tests.sh.tpl b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl
new file mode 100644
index 000000000..f0ade3e59
--- /dev/null
+++ b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+function reset_test_env()
+{
+  pvc_namespace=$1
+  pod_name=$2
+  pvc_name=$3
+  echo "--> Resetting POD and PVC before/after test"
+  if kubectl get pod -n $pvc_namespace $pod_name; then
+    kubectl delete pod -n $pvc_namespace $pod_name
+  fi
+
+  if kubectl get pvc -n $pvc_namespace $pvc_name; then
+    kubectl delete pvc -n $pvc_namespace $pvc_name;
+  fi
+}
+
+
+function storageclass_validation()
+{
+  pvc_namespace=$1
+  pod_name=$2
+  pvc_name=$3
+  storageclass=$4
+  echo "--> Starting validation"
+
+  # storageclass check
+  if ! kubectl get storageclass $storageclass; then
+    echo "Storageclass: $storageclass is not provisioned."
+    exit 1
+  fi
+
+  tee <<EOF | kubectl apply -n $pvc_namespace -f -
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: $pvc_name
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: $storageclass
+  resources:
+    requests:
+      storage: 3Gi
+EOF
+
+  # waiting for pvc to get create
+  sleep 30
+  if ! kubectl get pvc -n $pvc_namespace $pvc_name|grep Bound; then
+    echo "Storageclass is available but can't create PersistentVolumeClaim."
+    exit 1
+  fi
+
+  tee <<EOF | kubectl apply --namespace $pvc_namespace -f -
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: $pod_name
+spec:
+  containers:
+  - name: task-pv-storage
+    image: {{ .Values.images.tags.ceph_config_helper }}
+    command:
+    - "/bin/sh"
+    args:
+    - "-c"
+    - "touch /mnt/SUCCESS && exit 0 || exit 1"
+    volumeMounts:
+    - name: pvc
+      mountPath: "/mnt"
+      readOnly: false
+  restartPolicy: "Never"
+  volumes:
+  - name: pvc
+    persistentVolumeClaim:
+      claimName: $pvc_name
+EOF
+
+  # waiting for pod to get create
+  sleep 60
+  if ! kubectl get pods -n $pvc_namespace $pod_name; then
+    echo "Can not create POD with rbd storage class $storageclass based PersistentVolumeClaim."
+    echo 1
+  fi
+
+}
+
+
+reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME
+reset_test_env $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME
+
+if [ {{ .Values.storageclass.rbd.provision_storage_class }} == true ];
+then
+  echo "--> Checking RBD storage class."
+  storageclass={{ .Values.storageclass.rbd.metadata.name }}
+
+  storageclass_validation $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME $storageclass
+  reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME
+fi
+
+if [ {{ .Values.storageclass.cephfs.provision_storage_class }} == true ];
+then
+  echo "--> Checking cephfs storage class."
+  storageclass={{ .Values.storageclass.cephfs.metadata.name }}
+  storageclass_validation $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME $storageclass
+  reset_test_env $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME
+fi
diff --git a/ceph-provisioners/templates/configmap-bin-provisioner.yaml b/ceph-provisioners/templates/configmap-bin-provisioner.yaml
index 3163d50ad..248b366cd 100644
--- a/ceph-provisioners/templates/configmap-bin-provisioner.yaml
+++ b/ceph-provisioners/templates/configmap-bin-provisioner.yaml
@@ -26,4 +26,6 @@ data:
 {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   provisioner-rbd-namespace-client-key-cleaner.sh: |
 {{ tuple "bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  helm-tests.sh: |
+{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
diff --git a/ceph-provisioners/templates/pod-helm-tests.yaml b/ceph-provisioners/templates/pod-helm-tests.yaml
new file mode 100644
index 000000000..7cb9f1270
--- /dev/null
+++ b/ceph-provisioners/templates/pod-helm-tests.yaml
@@ -0,0 +1,108 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.deployment.client_secrets .Values.manifests.helm_tests }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := printf "%s-%s" $envAll.Release.Name "test" }}
+{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - persistentvolumes
+      - persistentvolumeclaims
+      - events
+      - pods
+    verbs:
+      - create
+      - get
+      - delete
+      - list
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{.Release.Name}}-test"
+  labels:
+{{ tuple $envAll "ceph" "provisioner-test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
+  restartPolicy: Never
+  serviceAccountName: {{ $serviceAccountName }}
+  initContainers:
+{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
+  containers:
+    - name: ceph-provisioner-helm-test
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 6 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
+{{ dict "envAll" $envAll "application" "test" "container" "test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
+      env:
+        - name: PVC_NAMESPACE
+          value: {{ .Release.Namespace }}
+        - name: RBD_TEST_POD_NAME
+          value: {{ .Values.pod.test_pod.rbd.name }}
+        - name: RBD_TEST_PVC_NAME
+          value: {{ .Values.pod.test_pod.rbd.pvc_name }}
+        - name: CEPHFS_TEST_POD_NAME
+          value: {{ .Values.pod.test_pod.cephfs.name }}
+        - name: CEPHFS_TEST_PVC_NAME
+          value: {{ .Values.pod.test_pod.cephfs.pvc_name }}
+      command:
+        - /tmp/helm-tests.sh
+      volumeMounts:
+        - name: ceph-provisioners-bin-clients
+          mountPath: /tmp/helm-tests.sh
+          subPath: helm-tests.sh
+          readOnly: true
+        - name: pod-tmp
+          mountPath: /tmp
+  volumes:
+    - name: ceph-provisioners-bin-clients
+      configMap:
+        name: {{ printf "%s-%s" $envAll.Release.Name "ceph-prov-bin-clients" | quote }}
+        defaultMode: 0555
+    - name: pod-tmp
+      emptyDir: {}
+{{- end }}
diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml
index 9e7a7c13f..7622bc6f7 100644
--- a/ceph-provisioners/values.yaml
+++ b/ceph-provisioners/values.yaml
@@ -49,6 +49,13 @@ labels:
     node_selector_value: enabled
 
 pod:
+  test_pod:
+    rbd:
+      name: rbd-prov-test-pod
+      pvc_name: rbd-prov-test-pvc
+    cephfs:
+      name: cephfs-prov-test-pod
+      pvc_name: cephfs-prov-test-pvc
   security_context:
     provisioner:
       pod:
@@ -88,6 +95,12 @@ pod:
         ceph_storage_keys_generator:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+    test:
+      pod:
+        runAsUser: 0
+      container:
+        test:
+          readOnlyRootFilesystem: true
   dns_policy: "ClusterFirstWithHostNet"
   replicas:
     cephfs_provisioner: 2
@@ -279,3 +292,4 @@ manifests:
   job_namespace_client_key_cleaner: true
   job_namespace_client_key: true
   storageclass: true
+  helm_tests: true
diff --git a/tools/deployment/multinode/035-ceph-ns-activate.sh b/tools/deployment/multinode/035-ceph-ns-activate.sh
index effbcf5ab..b0c432b63 100755
--- a/tools/deployment/multinode/035-ceph-ns-activate.sh
+++ b/tools/deployment/multinode/035-ceph-ns-activate.sh
@@ -50,3 +50,5 @@ helm upgrade --install ceph-osh-infra-config ./ceph-provisioners \
 
 #NOTE: Validate Deployment info
 helm status ceph-osh-infra-config
+
+helm test ceph-osh-infra-config
diff --git a/tools/deployment/openstack-support/025-ceph-ns-activate.sh b/tools/deployment/openstack-support/025-ceph-ns-activate.sh
index 3d19565df..d758d802c 100755
--- a/tools/deployment/openstack-support/025-ceph-ns-activate.sh
+++ b/tools/deployment/openstack-support/025-ceph-ns-activate.sh
@@ -50,6 +50,8 @@ helm upgrade --install ceph-openstack-config ./ceph-provisioners \
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh openstack
 
+helm test ceph-openstack-config
+
 #NOTE: Validate Deployment info
 kubectl get -n openstack jobs --show-all
 kubectl get -n openstack secrets
diff --git a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
index 0c9963080..c2d4c7e85 100755
--- a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
+++ b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
@@ -50,6 +50,8 @@ helm upgrade --install ceph-osh-infra-config ./ceph-provisioners \
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
+helm test ceph-osh-infra-config
+
 #NOTE: Validate Deployment info
 kubectl get -n osh-infra jobs --show-all
 kubectl get -n osh-infra secrets
diff --git a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
index bb20dc5dd..61008023e 100755
--- a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
+++ b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
@@ -75,3 +75,5 @@ helm upgrade --install tenant-ceph-openstack-config ./ceph-provisioners \
 
 #NOTE: Validate Deployment info
 helm status tenant-ceph-openstack-config
+
+helm test tenant-ceph-openstack-config