Elasticsearch: Make templates job more generic

This change updates the logic in our create-elasticsearch-templates
job to support creation of a variety of different API objects.

Change-Id: I380a55b93e7aabb606e713c21d71a383fef78b3f

elasticsearch/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.1.0
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.1.4
+version: 0.1.5
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_create_template.sh.tpl

@@ -2,44 +2,9 @@
 
 set -ex
 
-{{ range $template, $fields := .Values.conf.templates }}
+{{ range $object := .Values.conf.api_objects }}
-
+curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+   -X{{ $object.method | default "PUT" | upper }} \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
+   "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
--H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
+   -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}'
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "{{$template}} template created!"
-else
-   echo "{{$template}} template not created!"
-fi
-
-{{ end }}
-
-{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_slm/policy/{{$policy_name}}" \
--H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "Policy {{$policy_name}} created!"
-else
-   echo "Policy {{$policy_name}} not created!"
-fi
-
-{{ end }}
-
-{{ range $policy_name, $fields := .Values.conf.index_policies }}
-
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ilm/policy/{{$policy_name}}" \
--H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "Policy {{$policy_name}} created!"
-else
-   echo "Policy {{$policy_name}} not created!"
-fi
-
 {{ end }}

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -50,21 +50,18 @@ function check_snapshot_repositories () {
   {{ end }}
 }
 
-{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+{{ if .Values.manifests.job_elasticsearch_templates }}
 # Tests whether elasticsearch has successfully generated the elasticsearch index mapping
 # templates defined by values.yaml
 function check_templates () {
-  {{ range $template, $fields := .Values.conf.templates }}
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
+  if [ "$total_hits" -gt 0 ]; then
-              | python -c "import sys, json; print(len(json.load(sys.stdin)))")
+     echo "PASS: Successful hits on templates!"
-  if [ "${{$template}}_total_hits" -gt 0 ]; then
-     echo "PASS: Successful hits on {{$template}} template!"
   else
-     echo "FAIL: No hits on query for {{$template}} template! Exiting";
+     echo "FAIL: No hits on query for templates! Exiting";
      exit 1;
   fi
-  {{ end }}
 }
 {{ end }}
 

elasticsearch/templates/configmap-etc-templates.yaml

@@ -1,35 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.configmap_etc_templates }}
-{{- $envAll := . }}
-
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: elasticsearch-templates-etc
-type: Opaque
-data:
-{{ range $template, $fields := .Values.conf.templates }}
-  {{ $template }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-  {{ $policy_name }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-{{ range $policy_name, $fields := .Values.conf.index_policies }}
-  {{ $policy_name }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-
-{{- end }}

elasticsearch/templates/job-elasticsearch-template.yaml

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+{{- if .Values.manifests.job_elasticsearch_templates }}
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
@@ -72,24 +72,6 @@ spec:
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
-            {{ range $template, $fields := .Values.conf.templates }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$template}}.json
-              subPath: {{$template}}.json
-              readOnly: true
-            {{ end }}
-            {{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$policy_name}}.json
-              subPath: {{$policy_name}}.json
-              readOnly: true
-            {{ end }}
-            {{ range $policy_name, $fields := .Values.conf.index_policies }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$policy_name}}.json
-              subPath: {{$policy_name}}.json
-              readOnly: true
-            {{ end }}
 {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -98,9 +80,5 @@ spec:
           configMap:
             name: elasticsearch-bin
             defaultMode: 0555
-        - name: elasticsearch-templates-etc
-          secret:
-            secretName: elasticsearch-templates-etc
-            defaultMode: 0444
 {{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/values.yaml

@@ -24,11 +24,11 @@ images:
     ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
     s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216
     s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
-    helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
+    helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
     prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
-    elasticsearch_templates: docker.io/openstackhelm/heat:newton
+    elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -746,8 +746,9 @@ conf:
       ca: null
       client_private_key: null
       client_cert: null
-  templates:
+  api_objects:
-    fluent:
+    - endpoint: _template/fluent
+      body:
         index_patterns: "logstash-*"
         settings:
           index:
@@ -774,8 +775,16 @@ conf:
                 pod_name:
                   type: keyword
                   index: false
-  snapshot_policies:
+    - endpoint: _ilm/policy/delete_all_indexes
-    non-security-snapshots:
+      body:
+        policy:
+          phases:
+            delete:
+              min_age: 14d
+              actions:
+                delete: {}
+    - endpoint: _slm/policy/non-security-snapshots
+      body:
         schedule: "0 30 1 * * ?"
         name: "<non-security-logs-snapshot-{now/d}>"
         repository: logstash_snapshots
@@ -791,7 +800,8 @@ conf:
           disable_action: false
         retention:
           expire_after: 29d
-    security-snapshots:
+    - endpoint: _slm/policy/security-snapshots
+      body:
         schedule: "0 30 1 * * ?"
         name: "<security-logs-snapshot-{now/d}>"
         repository: logstash_snapshots
@@ -807,14 +817,6 @@ conf:
           disable_action: false
         retention:
           expire_after: 179d
-  index_policies:
-    delete_all_indexes:
-      policy:
-        phases:
-          delete:
-            min_age: 14d
-            actions:
-              delete: {}
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry: