Elasticsearch: Make templates job more generic

This change updates the logic in our create-elasticsearch-templates
job to support creation of a variety of different API objects.

Change-Id: I380a55b93e7aabb606e713c21d71a383fef78b3f

elasticsearch/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.1.0
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.1.4
+version: 0.1.5
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_create_template.sh.tpl

@@ -2,44 +2,9 @@
 
 set -ex
 
-{{ range $template, $fields := .Values.conf.templates }}
-
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
--H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "{{$template}} template created!"
-else
-   echo "{{$template}} template not created!"
-fi
-
+{{ range $object := .Values.conf.api_objects }}
+curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+   -X{{ $object.method | default "PUT" | upper }} \
+   "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
+   -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}'
 {{ end }}
-
-{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_slm/policy/{{$policy_name}}" \
--H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "Policy {{$policy_name}} created!"
-else
-   echo "Policy {{$policy_name}} not created!"
-fi
-
-{{ end }}
-
-{{ range $policy_name, $fields := .Values.conf.index_policies }}
-
-result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
--XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ilm/policy/{{$policy_name}}" \
--H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
-| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
-if [ "$result" == "True" ]; then
-   echo "Policy {{$policy_name}} created!"
-else
-   echo "Policy {{$policy_name}} not created!"
-fi
-
-{{ end }}

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -50,21 +50,18 @@ function check_snapshot_repositories () {
   {{ end }}
 }
 
-{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+{{ if .Values.manifests.job_elasticsearch_templates }}
 # Tests whether elasticsearch has successfully generated the elasticsearch index mapping
 # templates defined by values.yaml
 function check_templates () {
-  {{ range $template, $fields := .Values.conf.templates }}
-  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
-              | python -c "import sys, json; print(len(json.load(sys.stdin)))")
-  if [ "${{$template}}_total_hits" -gt 0 ]; then
-     echo "PASS: Successful hits on {{$template}} template!"
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
+  if [ "$total_hits" -gt 0 ]; then
+     echo "PASS: Successful hits on templates!"
   else
-     echo "FAIL: No hits on query for {{$template}} template! Exiting";
+     echo "FAIL: No hits on query for templates! Exiting";
      exit 1;
   fi
-  {{ end }}
 }
 {{ end }}
 

elasticsearch/templates/configmap-etc-templates.yaml

@@ -1,35 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.configmap_etc_templates }}
-{{- $envAll := . }}
-
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: elasticsearch-templates-etc
-type: Opaque
-data:
-{{ range $template, $fields := .Values.conf.templates }}
-  {{ $template }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-  {{ $policy_name }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-{{ range $policy_name, $fields := .Values.conf.index_policies }}
-  {{ $policy_name }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
-
-{{- end }}

elasticsearch/templates/job-elasticsearch-template.yaml

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+{{- if .Values.manifests.job_elasticsearch_templates }}
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
@@ -72,24 +72,6 @@ spec:
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
-            {{ range $template, $fields := .Values.conf.templates }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$template}}.json
-              subPath: {{$template}}.json
-              readOnly: true
-            {{ end }}
-            {{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$policy_name}}.json
-              subPath: {{$policy_name}}.json
-              readOnly: true
-            {{ end }}
-            {{ range $policy_name, $fields := .Values.conf.index_policies }}
-            - name: elasticsearch-templates-etc
-              mountPath: /tmp/{{$policy_name}}.json
-              subPath: {{$policy_name}}.json
-              readOnly: true
-            {{ end }}
 {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -98,9 +80,5 @@ spec:
           configMap:
             name: elasticsearch-bin
             defaultMode: 0555
-        - name: elasticsearch-templates-etc
-          secret:
-            secretName: elasticsearch-templates-etc
-            defaultMode: 0444
 {{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/values.yaml

@@ -24,11 +24,11 @@ images:
     ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
     s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216
     s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
-    helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
+    helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
     prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
-    elasticsearch_templates: docker.io/openstackhelm/heat:newton
+    elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -746,75 +746,77 @@ conf:
       ca: null
       client_private_key: null
       client_cert: null
-  templates:
-    fluent:
-      index_patterns: "logstash-*"
-      settings:
-        index:
-          number_of_shards: 1
-      mappings:
-        properties:
-          kubernetes:
-            properties:
-              container_name:
-                type: keyword
-                index: false
-              docker_id:
-                type: keyword
-                index: false
-              host:
-                type: keyword
-                index: false
-              namespace_name:
-                type: keyword
-                index: false
-              pod_id:
-                type: keyword
-                index: false
-              pod_name:
-                type: keyword
-                index: false
-  snapshot_policies:
-    non-security-snapshots:
-      schedule: "0 30 1 * * ?"
-      name: "<non-security-logs-snapshot-{now/d}>"
-      repository: logstash_snapshots
-      config:
-        indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
-        ignore_unavailable: true
-        include_global_state: false
-        wait_for_completion: true
-        max_wait: 64800
-        wait_interval: 30
-        ignore_empty_list: true
-        continue_if_exception: true
-        disable_action: false
-      retention:
-        expire_after: 29d
-    security-snapshots:
-      schedule: "0 30 1 * * ?"
-      name: "<security-logs-snapshot-{now/d}>"
-      repository: logstash_snapshots
-      config:
-        indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
-        ignore_unavailable: true
-        include_global_state: false
-        wait_for_completion: true
-        max_wait: 18000
-        wait_interval: 30
-        ignore_empty_list: true
-        continue_if_exception: true
-        disable_action: false
-      retention:
-        expire_after: 179d
-  index_policies:
-    delete_all_indexes:
-      policy:
-        phases:
-          delete:
-            min_age: 14d
-            actions:
-              delete: {}
+  api_objects:
+    - endpoint: _template/fluent
+      body:
+        index_patterns: "logstash-*"
+        settings:
+          index:
+            number_of_shards: 1
+        mappings:
+          properties:
+            kubernetes:
+              properties:
+                container_name:
+                  type: keyword
+                  index: false
+                docker_id:
+                  type: keyword
+                  index: false
+                host:
+                  type: keyword
+                  index: false
+                namespace_name:
+                  type: keyword
+                  index: false
+                pod_id:
+                  type: keyword
+                  index: false
+                pod_name:
+                  type: keyword
+                  index: false
+    - endpoint: _ilm/policy/delete_all_indexes
+      body:
+        policy:
+          phases:
+            delete:
+              min_age: 14d
+              actions:
+                delete: {}
+    - endpoint: _slm/policy/non-security-snapshots
+      body:
+        schedule: "0 30 1 * * ?"
+        name: "<non-security-logs-snapshot-{now/d}>"
+        repository: logstash_snapshots
+        config:
+          indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
+          ignore_unavailable: true
+          include_global_state: false
+          wait_for_completion: true
+          max_wait: 64800
+          wait_interval: 30
+          ignore_empty_list: true
+          continue_if_exception: true
+          disable_action: false
+        retention:
+          expire_after: 29d
+    - endpoint: _slm/policy/security-snapshots
+      body:
+        schedule: "0 30 1 * * ?"
+        name: "<security-logs-snapshot-{now/d}>"
+        repository: logstash_snapshots
+        config:
+          indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
+          ignore_unavailable: true
+          include_global_state: false
+          wait_for_completion: true
+          max_wait: 18000
+          wait_interval: 30
+          ignore_empty_list: true
+          continue_if_exception: true
+          disable_action: false
+        retention:
+          expire_after: 179d
 endpoints:
   cluster_domain_suffix: cluster.local
   local_image_registry: