From ed7b5e4024f49c27db196c44f6abd8c2cf69282c Mon Sep 17 00:00:00 2001 From: "Huang, Scott (sh2725)" Date: Tue, 20 Nov 2018 14:36:11 -0500 Subject: [PATCH] Create Kibana Index Patterns Use Kibana REST API to create Kibana index patterns and set a default index pattern. Script calling Kibana REST API is executed using a Job, and the specific index patterns are configurable in values.yaml. Change-Id: I1ca6dd9609e6d62d1ce749ee09e1490d51659709 --- .../bin/_create_kibana_index_patterns.sh.tpl | 26 +++++++ kibana/templates/configmap-bin.yaml | 2 + .../job-register-kibana-indexes.yaml | 68 +++++++++++++++++++ kibana/values.yaml | 19 ++++++ 4 files changed, 115 insertions(+) create mode 100644 kibana/templates/bin/_create_kibana_index_patterns.sh.tpl create mode 100644 kibana/templates/job-register-kibana-indexes.yaml diff --git a/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl new file mode 100644 index 000000000..6d9b0f474 --- /dev/null +++ b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl @@ -0,0 +1,26 @@ +#!/bin/bash +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +set -ex + +{{- range .Values.conf.create_kibana_indexes.indexes }} +curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/index-pattern/{{ . }}-*" -H 'Content-Type: application/json' \ + -d '{"title":"{{ . }}-*","timeFieldName":"@timestamp","notExpandable":true}' +{{- end}} +curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/config/5.6.4" -H 'Content-Type: application/json' \ + -d '{"defaultIndex" : "{{ .Values.conf.create_kibana_indexes.default_index }}-*"}' diff --git a/kibana/templates/configmap-bin.yaml b/kibana/templates/configmap-bin.yaml index 61cadcdba..5342e2afe 100644 --- a/kibana/templates/configmap-bin.yaml +++ b/kibana/templates/configmap-bin.yaml @@ -26,6 +26,8 @@ data: {{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} kibana.sh: | {{ tuple "bin/_kibana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_kibana_index_patterns.sh: | +{{ tuple "bin/_create_kibana_index_patterns.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/kibana/templates/job-register-kibana-indexes.yaml b/kibana/templates/job-register-kibana-indexes.yaml new file mode 100644 index 000000000..beb5284bf --- /dev/null +++ b/kibana/templates/job-register-kibana-indexes.yaml @@ -0,0 +1,68 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_register_kibana_indexes }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $serviceAccountName := "register-kibana-indexes" }} +{{ tuple $envAll "register_kibana_indexes" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: register-kibana-indexes +spec: + template: + metadata: + labels: +{{ tuple $envAll "kibana" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "register_kibana_indexes" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: register-kibana-indexes +{{ tuple $envAll "register_kibana_indexes" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.register_kibana_indexes | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_ENDPOINT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + command: + - /tmp/create_kibana_index_patterns.sh + volumeMounts: + - name: kibana-bin + mountPath: /tmp/create_kibana_index_patterns.sh + subPath: create_kibana_index_patterns.sh + readOnly: false + volumes: + - name: kibana-bin + configMap: + name: kibana-bin + defaultMode: 0755 +{{- end }} diff --git a/kibana/values.yaml b/kibana/values.yaml index 9721ff707..7cf69b728 100644 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -26,6 +26,7 @@ images: kibana: docker.io/kibana:5.6.4 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 image_repo_sync: docker.io/docker:17.07.0 + register_kibana_indexes: docker.io/openstackhelm/heat:newton pull_policy: IfNotPresent local_registry: active: false @@ -74,6 +75,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + register_kibana_indexes: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" secrets: elasticsearch: @@ -101,6 +109,10 @@ dependencies: services: - endpoint: internal service: elasticsearch + register_kibana_indexes: + services: + - endpoint: internal + service: kibana conf: httpd: | @@ -220,6 +232,12 @@ conf: port: 5601 ssl: enabled: false + create_kibana_indexes: + enabled: true + indexes: + - logstash + - openstack + default_index: logstash endpoints: cluster_domain_suffix: cluster.local @@ -323,3 +341,4 @@ manifests: secret_ingress_tls: true service: true service_ingress: true + job_register_kibana_indexes: true