Merge "Add elasticsearch exporter"

elasticsearch/templates/monitoring/prometheus/bin/_elasticsearch-exporter.sh.tpl

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+COMMAND="${@:-start}"
+
+function start () {
+  exec /bin/elasticsearch_exporter \
+        -es.uri=$ELASTICSEARCH_URI \
+        -es.all={{ .Values.conf.prometheus_elasticsearch_exporter.es.all | quote }} \
+        -es.timeout={{ .Values.conf.prometheus_elasticsearch_exporter.es.timeout }} \
+        -web.telemetry-path={{ .Values.endpoints.prometheus_elasticsearch_exporter.path.default }}
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

elasticsearch/templates/monitoring/prometheus/exporter-configmap-bin.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.configmap_bin_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-exporter-bin
+data:
+  elasticsearch-exporter.sh: |
+{{ tuple "bin/_elasticsearch-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml

@@ -0,0 +1,80 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- if .Values.images.local_registry.active -}}
+{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.prometheus_elasticsearch_exporter .Values.conditional_dependencies.local_image_registry) -}}
+{{- else -}}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.prometheus_elasticsearch_exporter -}}
+{{- end -}}
+
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+
+{{- $serviceAccountName := "prometheus-elasticsearch-exporter" }}
+{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: prometheus-elasticsearch-exporter
+spec:
+  replicas: {{ .Values.pod.replicas.prometheus_elasticsearch_exporter }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_elasticsearch_exporter.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: elasticsearch-exporter
+{{ tuple $envAll "prometheus_elasticsearch_exporter" | include "helm-toolkit.snippets.image" | indent 10 }}
+          command:
+            - /tmp/elasticsearch-exporter.sh
+            - start
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/elasticsearch-exporter.sh
+                  - stop
+{{ tuple $envAll $envAll.Values.pod.resources.exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_URI
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_URI
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.network.prometheus_elasticsearch_exporter.port }}
+          volumeMounts:
+            - name: elasticsearch-exporter-bin
+              mountPath: /tmp/elasticsearch-exporter.sh
+              subPath: elasticsearch-exporter.sh
+              readOnly: true
+      volumes:
+        - name: elasticsearch-exporter-bin
+          configMap:
+            name: elasticsearch-exporter-bin
+            defaultMode: 0555
+{{- end }}

elasticsearch/templates/monitoring/prometheus/exporter-service.yaml

@@ -0,0 +1,37 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.elasticsearch_exporter }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "prometheus_elasticsearch_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "elasticsearch-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+{{- if .Values.monitoring.prometheus.enabled }}
+{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
+{{- end }}
+spec:
+  ports:
+  - name: metrics
+    port: {{ .Values.network.prometheus_elasticsearch_exporter.port }}
+  selector:
+{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

elasticsearch/templates/secret-admin-creds.yaml

@@ -17,6 +17,11 @@ limitations under the License.
 {{- if .Values.manifests.secret_admin }}
 {{- $envAll := . }}
 {{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
+
+{{- $elasticsearch_user := .Values.endpoints.elasticsearch.auth.admin.username }}
+{{- $elasticsearch_password := .Values.endpoints.elasticsearch.auth.admin.password }}
+{{- $elasticsearch_host := tuple "elasticsearch" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $elasticsearch_uri := printf "http://%s:%s@%s" $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
 ---
 apiVersion: v1
 kind: Secret
@@ -26,4 +31,5 @@ type: Opaque
 data:
   ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
   ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+  ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }}
 {{- end }}

elasticsearch/values.yaml

@@ -23,6 +23,7 @@ images:
     curator:  docker.io/bobrik/curator:5.2.0
     elasticsearch: docker.io/elasticsearch:5.6.4
     helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
+    prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
     snapshot_repository: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     image_repo_sync: docker.io/docker:17.07.0
@@ -44,6 +45,10 @@ dependencies:
     services: null
   elasticsearch_data:
     services: null
+  prometheus_elasticsearch_exporter:
+    services:
+      - service: elasticsearch
+        endpoint: internal
   curator:
     services: null
   image_repo_sync:
@@ -89,6 +94,8 @@ pod:
         timeout: 600
       client:
         timeout: 600
+      prometheus_elasticsearch_exporter:
+        timeout: 600
   mounts:
     elasticsearch:
       elasticsearch:
@@ -115,6 +122,13 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "2000m"
+    prometheus_elasticsearch_exporter:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
     jobs:
       curator:
         requests:
@@ -265,6 +279,10 @@ conf:
     override:
     prefix:
     append:
+  prometheus_elasticsearch_exporter:
+    es:
+      all: true
+      timeout: 20s
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -293,6 +311,25 @@ endpoints:
         default: 80
       discovery:
         default: 9300
+  prometheus_elasticsearch_exporter:
+    namespace: null
+    hosts:
+      default: elasticsearch-exporter
+    host_fqdn_override:
+      default: null
+    path:
+      default: /metrics
+    scheme:
+      default: 'http'
+    port:
+      metrics:
+        default: 9108
+
+monitoring:
+  prometheus:
+    enabled: false
+    elasticsearch_exporter:
+      scrape: true
 
 network:
   client:
@@ -310,6 +347,8 @@ network:
     node_port:
       enabled: false
       port: 30931
+  prometheus_elasticsearch_exporter:
+    port: 9108
 
 storage:
   elasticsearch:
@@ -341,6 +380,12 @@ manifests:
   helm_tests: true
   pvc_snapshots: false
   secret_admin: true
+  monitoring:
+    prometheus:
+      configmap_bin_exporter: true
+      deployment_exporter: true
+      service_exporter: true
+  pvc_snapshots: true
   service_data: true
   service_discovery: true
   service_logging: true

tools/gate/chart-deploys/default.yaml

@@ -203,6 +203,9 @@ charts:
           storage_class: openstack-helm-bootstrap
         filesystem_repository:
           storage_class: openstack-helm-bootstrap
+      monitoring:
+        prometheus:
+          enabled: true
 
   fluent_logging:
     chart_name: fluent-logging