Kibana - Add hook to delete .kibana indices

This hook is enabled for post-delete and pre-upgrade triggers.
The indices deleted by this hook are Kibana's meta indices
  - .kibana
  - .kibana_1
  - .kibana_2
  etc

This is done to get around https://github.com/elastic/kibana/issues/58388
which sometimes prevents Kibana deployments from upgrading successfully.

Change-Id: I99ccc7de20c6dadb5154e4bb714dfd302a694a78

kibana/templates/bin/_flush_kibana_metadata.sh.tpl

@@ -0,0 +1,19 @@
+#!/bin/bash
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+set -ex
+echo "Deleting index created for metadata"
+
+curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  -XDELETE "${ELASTICSEARCH_ENDPOINT}/.kibana*"

kibana/templates/configmap-bin.yaml

@@ -26,6 +26,8 @@ data:
 {{ tuple "bin/_kibana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   create_kibana_index_patterns.sh: |
 {{ tuple "bin/_create_kibana_index_patterns.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  flush_kibana_metadata.sh: |
+{{ tuple "bin/_flush_kibana_metadata.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

kibana/templates/job-flush-kibana-metadata.yaml

@@ -0,0 +1,100 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+This hook is enabled for post-delete and pre-upgrade triggers.
+The indices deleted by this hook are Kibana's meta indices
+  - .kibana
+  - .kibana_1
+  - .kibana_2
+  etc
+
+This is done to get around https://github.com/elastic/kibana/issues/58388
+which sometimes prevents Kibana deployments from upgrading successfully.
+*/}}
+
+{{- if .Values.manifests.job_flush_kibana_metadata }}
+{{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+{{- $serviceAccountName := "flush-kibana-metadata" }}
+{{ tuple $envAll "flush_kibana_metadata" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: flush-kibana-metadata
+spec:
+  backoffLimit: {{ .Values.jobs.flush_kibana_metadata.backoffLimit }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "kibana" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+        "helm.sh/hook": post-delete, pre-upgrade
+        "helm.sh/hook-delete-policy": hook-succeeded
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ dict "envAll" $envAll "podName" "flush-kibana-metadata" "containerNames" (list "flush-kibana-metadata" "init")  | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      activeDeadlineSeconds: {{ .Values.jobs.flush_kibana_metadata.activeDeadlineSeconds }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "flush_kibana_metadata" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: flush-kibana-metadata
+{{ tuple $envAll "flush_kibana_metadata" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.flush_kibana_metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "flush_kibana_metadata" "container" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
+            - name: KIBANA_ENDPOINT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+            - name: ELASTICSEARCH_ENDPOINT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          command:
+            - /tmp/flush_kibana_metadata.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: pod-run
+              mountPath: /run
+            - name: kibana-bin
+              mountPath: /tmp/flush_kibana_metadata.sh
+              subPath: flush_kibana_metadata.sh
+              readOnly: false
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: pod-run
+          emptyDir:
+            medium: "Memory"
+        - name: kibana-bin
+          configMap:
+            name: kibana-bin
+            defaultMode: 0755
+{{- end }}

kibana/values.yaml

@@ -25,6 +25,7 @@ images:
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
     register_kibana_indexes: docker.io/openstackhelm/heat:newton-ubuntu_xenial
+    flush_kibana_metadata: docker.io/openstackhelm/heat:newton-ubuntu_xenial
   pull_policy: IfNotPresent
   local_registry:
     active: false
@@ -53,6 +54,13 @@ pod:
         register_kibana_indexes:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+    flush_kibana_metadata:
+      pod:
+        runAsUser: 1000
+      container:
+        flush_kibana_metadata:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
   affinity:
     anti:
       type:
@@ -102,6 +110,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      flush_kibana_metadata:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
   probes:
     kibana:
       kibana:
@@ -152,6 +167,15 @@ dependencies:
       services:
         - endpoint: internal
           service: kibana
+    flush_kibana_metadata:
+      services:
+        - endpoint: internal
+          service: kibana
+
+jobs:
+  flush_kibana_metadata:
+    backoffLimit: 6
+    activeDeadlineSeconds: 600
 
 conf:
   httpd: |
@@ -393,3 +417,4 @@ manifests:
   service: true
   service_ingress: true
   job_register_kibana_indexes: true
+  job_flush_kibana_metadata: true

kibana/values_overrides/apparmor.yaml

@@ -8,3 +8,6 @@ pod:
     register-kibana-indexes:
       register-kibana-indexes: runtime/default
       init: runtime/default
+    flush-kibana-metadata:
+      flush-kibana-metadata: runtime/default
+      init: runtime/default