Create Kibana Index Patterns

Use Kibana REST API to create Kibana index patterns and set a default
index pattern.
Script calling Kibana REST API is executed using a Job, and the specific
index patterns are configurable in values.yaml.

Change-Id: I1ca6dd9609e6d62d1ce749ee09e1490d51659709

kibana/templates/bin/_create_kibana_index_patterns.sh.tpl

@@ -0,0 +1,26 @@
+#!/bin/bash
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+set -ex
+
+{{- range .Values.conf.create_kibana_indexes.indexes }}
+curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/index-pattern/{{ . }}-*" -H 'Content-Type: application/json' \
+  -d '{"title":"{{ . }}-*","timeFieldName":"@timestamp","notExpandable":true}'
+{{- end}}
+curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/config/5.6.4" -H 'Content-Type: application/json' \
+  -d '{"defaultIndex" : "{{ .Values.conf.create_kibana_indexes.default_index }}-*"}'

kibana/templates/configmap-bin.yaml

@@ -26,6 +26,8 @@ data:
 {{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   kibana.sh: |
 {{ tuple "bin/_kibana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  create_kibana_index_patterns.sh: |
+{{ tuple "bin/_create_kibana_index_patterns.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

kibana/templates/job-register-kibana-indexes.yaml

@@ -0,0 +1,68 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_register_kibana_indexes }}
+{{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+{{- $serviceAccountName := "register-kibana-indexes" }}
+{{ tuple $envAll "register_kibana_indexes" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: register-kibana-indexes
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "kibana" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "register_kibana_indexes" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: register-kibana-indexes
+{{ tuple $envAll "register_kibana_indexes" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.register_kibana_indexes | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
+            - name: ELASTICSEARCH_ENDPOINT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          command:
+            - /tmp/create_kibana_index_patterns.sh
+          volumeMounts:
+            - name: kibana-bin
+              mountPath: /tmp/create_kibana_index_patterns.sh
+              subPath: create_kibana_index_patterns.sh
+              readOnly: false
+      volumes:
+        - name: kibana-bin
+          configMap:
+            name: kibana-bin
+            defaultMode: 0755
+{{- end }}

kibana/values.yaml

@@ -26,6 +26,7 @@ images:
     kibana: docker.io/kibana:5.6.4
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     image_repo_sync: docker.io/docker:17.07.0
+    register_kibana_indexes: docker.io/openstackhelm/heat:newton
   pull_policy: IfNotPresent
   local_registry:
     active: false
@@ -74,6 +75,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      register_kibana_indexes:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 secrets:
   elasticsearch:
@@ -101,6 +109,10 @@ dependencies:
       services:
         - endpoint: internal
           service: elasticsearch
+    register_kibana_indexes:
+      services:
+        - endpoint: internal
+          service: kibana
 
 conf:
   httpd: |
@@ -220,6 +232,12 @@ conf:
       port: 5601
       ssl:
         enabled: false
+  create_kibana_indexes:
+    enabled: true
+    indexes:
+      - logstash
+      - openstack
+    default_index: logstash
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -323,3 +341,4 @@ manifests:
   secret_ingress_tls: true
   service: true
   service_ingress: true
+  job_register_kibana_indexes: true