Elasticsearch/Kibana: Update version to 7.1.0

This updates the Elasticsearch and Kibana charts to deploy
version 7.1.0. This move required significant changes to both
charts, including: changing elasticsearch masters to a statefulset
to utilize reliable dns names for the discovery process, config
updates to reflect deprecated/updated/removed values, use the
kibana saved objects api for managing index patterns and setting
the default index, and updating the elasticsearch entrypoint
scripts to reflect the use of elastic-keystore for storing s3
credentials instead of defining them in the configuration file

Change-Id: I270d905f266fc15492e47d8376714ba80603e66d
Signed-off-by: Steve Wilkerson <sw5822@att.com>

elasticsearch/templates/bin/_elasticsearch.sh.tpl

@@ -15,12 +15,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-set -ex
+{{- $envAll := . }}
+
+set -e
 COMMAND="${@:-start}"
 
+function initiate_keystore () {
+  bin/elasticsearch-keystore create
+  echo ${S3_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.access_key
+  echo ${S3_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.secret_key
+}
+
 function start () {
   ulimit -l unlimited
-  exec /docker-entrypoint.sh elasticsearch
+  initiate_keystore
+  exec /usr/local/bin/docker-entrypoint.sh elasticsearch
 }
 
 function stop () {
@@ -42,10 +51,32 @@ function allocate_data_node () {
   echo "Node ${NODE_NAME} is ready to be used"
 }
 
+function start_master_node () {
+  ulimit -l unlimited
+  initiate_keystore
+  if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ];
+  then
+    {{ if empty $envAll.Values.conf.elasticsearch.config.cluster.initial_master_nodes -}}
+    {{- $_ := set $envAll.Values "__eligible_masters" ( list ) }}
+    {{- range $podInt := until ( atoi (print $envAll.Values.pod.replicas.master ) ) }}
+    {{- $eligibleMaster := printf "elasticsearch-master-%s" (toString $podInt) }}
+    {{- $__eligible_masters := append $envAll.Values.__eligible_masters $eligibleMaster }}
+    {{- $_ := set $envAll.Values "__eligible_masters" $__eligible_masters }}
+    {{- end -}}
+    {{- $masters := include "helm-toolkit.utils.joinListWithComma" $envAll.Values.__eligible_masters -}}
+    echo {{$masters}} >> {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt
+    exec /usr/local/bin/docker-entrypoint.sh elasticsearch -Ecluster.initial_master_nodes={{$masters}}
+    {{- end }}
+  else
+    exec /usr/local/bin/docker-entrypoint.sh elasticsearch
+  fi
+}
+
 function start_data_node () {
   ulimit -l unlimited
+  initiate_keystore
   allocate_data_node &
-  /docker-entrypoint.sh elasticsearch &
+  /usr/local/bin/docker-entrypoint.sh elasticsearch &
   function drain_data_node () {
     echo "Prepare to migrate data off node ${NODE_NAME}"
     echo "Move all data from node ${NODE_NAME}"

elasticsearch/templates/bin/_register-repository.sh.tpl

@@ -32,9 +32,7 @@ function register_snapshot_repository() {
       "settings": {
         "endpoint": "'"$RGW_HOST"'",
         "protocol": "http",
-        "bucket": "'"$S3_BUCKET"'",
+        "bucket": "'"$S3_BUCKET"'"
-        "access_key": "'"$S3_ACCESS_KEY"'",
-        "secret_key": "'"$S3_SECRET_KEY"'"
       }
     }' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
   if [ "$result" == "True" ];

elasticsearch/templates/configmap-etc-elasticsearch.yaml

@@ -17,23 +17,18 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc_elasticsearch }}
 {{- $envAll := . }}
 
-{{- if empty .Values.conf.elasticsearch.config.cloud.aws.access_key -}}
-{{- set .Values.conf.elasticsearch.config.cloud.aws "access_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.access_key -}}
-{{- end -}}
-
-{{- if empty .Values.conf.elasticsearch.config.cloud.aws.secret_key -}}
-{{- set .Values.conf.elasticsearch.config.cloud.aws "secret_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.secret_key -}}
-{{- end -}}
-
 {{- if empty .Values.endpoints.ceph_object_store.path.default -}}
 {{- set .Values.endpoints.ceph_object_store.path "default" .Values.conf.elasticsearch.snapshots.bucket -}}
 {{- end -}}
 
-{{- if empty .Values.conf.elasticsearch.config.cloud.aws.s3.endpoint -}}
+{{- if empty .Values.conf.elasticsearch.config.s3.client.default.endpoint -}}
 {{- $radosgw_host := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
-{{- $bucket_path := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" -}}
+{{- set .Values.conf.elasticsearch.config.s3.client.default "endpoint" $radosgw_host -}}
-{{- $s3_endpoint := printf "%s/%s" $radosgw_host $bucket_path -}}
+{{- end -}}
-{{- set .Values.conf.elasticsearch.config.cloud.aws.s3 "endpoint" $s3_endpoint -}}
+
+{{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}}
+{{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}}
+{{- set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
 {{- end -}}
 ---
 apiVersion: v1
@@ -46,4 +41,5 @@ data:
 #NOTE(portdirect): this must be last, to work round helm ~2.7 bug.
 {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }}
 {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.log4j2 "key" "log4j2.properties" "format" "Secret") | indent 2 }}
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.jvm_options "key" "jvm.options" "format" "Secret") | indent 2 }}
 {{- end }}

elasticsearch/templates/deployment-client.yaml

@@ -18,6 +18,7 @@ limitations under the License.
 {{- $envAll := . }}
 
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
 
 {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
 
@@ -177,7 +178,7 @@ spec:
             - name: NODE_MASTER
               value: "false"
             - name: NODE_INGEST
-              value: "false"
+              value: "true"
             - name: NODE_DATA
               value: "false"
             - name: HTTP_ENABLE
@@ -186,6 +187,19 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_ACCESS_KEY
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_SECRET_KEY
+{{- if .Values.pod.env.client }}
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
+{{- end }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -195,8 +209,6 @@ spec:
               mountPath: /tmp/elasticsearch.sh
               subPath: elasticsearch.sh
               readOnly: true
-            - name: elasticsearch-config
-              mountPath: /usr/share/elasticsearch/config
             - name: elasticsearch-etc
               mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
               subPath: elasticsearch.yml
@@ -205,14 +217,16 @@ spec:
               mountPath: /usr/share/elasticsearch/config/log4j2.properties
               subPath: log4j2.properties
               readOnly: true
+            - name: elasticsearch-etc
+              mountPath: /usr/share/elasticsearch/config/jvm.options
+              subPath: jvm.options
+              readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
-        - name: elasticsearch-config
-          emptyDir: {}
         - name: elasticsearch-logs
           emptyDir: {}
         - name: elasticsearch-bin

elasticsearch/templates/statefulset-data.yaml

@@ -18,6 +18,7 @@ limitations under the License.
 {{- $envAll := . }}
 
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
 
 {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
 
@@ -70,6 +71,7 @@ metadata:
 spec:
 {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }}
   serviceName: {{ tuple "elasticsearch" "data" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  podManagementPolicy: "Parallel"
   replicas: {{ .Values.pod.replicas.data }}
   selector:
     matchLabels:
@@ -101,6 +103,19 @@ spec:
             - sysctl
             - -w
             - vm.max_map_count={{ .Values.conf.init.max_map_count }}
+        - name: elasticsearch-perms
+{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - chown
+            - -R
+            - "elasticsearch:"
+            - {{ .Values.conf.elasticsearch.config.path.data }}
+          volumeMounts:
+            - name: storage
+              mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
       containers:
         - name: elasticsearch-data
 {{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
@@ -148,8 +163,21 @@ spec:
               value: "false"
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}"
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_ACCESS_KEY
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_SECRET_KEY
             - name: DISCOVERY_SERVICE
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+{{- if .Values.pod.env.data }}
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
+{{- end }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -159,8 +187,6 @@ spec:
               mountPath: /tmp/elasticsearch.sh
               subPath: elasticsearch.sh
               readOnly: true
-            - name: elasticsearch-config
-              mountPath: /usr/share/elasticsearch/config
             - name: elasticsearch-etc
               mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
               subPath: elasticsearch.yml
@@ -169,6 +195,10 @@ spec:
               mountPath: /usr/share/elasticsearch/config/log4j2.properties
               subPath: log4j2.properties
               readOnly: true
+            - name: elasticsearch-etc
+              mountPath: /usr/share/elasticsearch/config/jvm.options
+              subPath: jvm.options
+              readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
@@ -177,8 +207,6 @@ spec:
           emptyDir: {}
         - name: elasticsearch-logs
           emptyDir: {}
-        - name: elasticsearch-config
-          emptyDir: {}
         - name: elasticsearch-bin
           configMap:
             name: elasticsearch-bin
@@ -188,7 +216,7 @@ spec:
             secretName: elasticsearch-etc
             defaultMode: 0444
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
-{{- if not .Values.storage.enabled }}
+{{- if not .Values.storage.data.enabled }}
         - name: storage
           emptyDir: {}
 {{- else }}
@@ -196,10 +224,10 @@ spec:
     - metadata:
         name: storage
       spec:
-        accessModes: {{ .Values.storage.pvc.access_mode }}
+        accessModes: {{ .Values.storage.data.pvc.access_mode }}
         resources:
           requests:
-            storage: {{ .Values.storage.requests.storage  }}
+            storage: {{ .Values.storage.data.requests.storage  }}
-        storageClassName: {{ .Values.storage.storage_class }}
+        storageClassName: {{ .Values.storage.data.storage_class }}
 {{- end }}
 {{- end }}

elasticsearch/templates/statefulset-master.yaml

@@ -14,9 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if .Values.manifests.deployment_master }}
+{{- if .Values.manifests.statefulset_master }}
 {{- $envAll := . }}
 
+{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
+
 {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
 
 {{- $serviceAccountName := "elasticsearch-master" }}
@@ -58,7 +60,7 @@ rules:
     - get
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: elasticsearch-master
   annotations:
@@ -66,6 +68,8 @@ metadata:
   labels:
 {{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
+  serviceName: {{ tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  podManagementPolicy: "Parallel"
   replicas: {{ .Values.pod.replicas.master }}
   selector:
     matchLabels:
@@ -98,6 +102,19 @@ spec:
           - sysctl
           - -w
           - vm.max_map_count={{ .Values.conf.init.max_map_count }}
+        - name: elasticsearch-perms
+{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - chown
+            - -R
+            - "elasticsearch:"
+            - {{ .Values.conf.elasticsearch.config.path.data }}
+          volumeMounts:
+            - name: storage
+              mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
       containers:
         - name: elasticsearch-master
 {{ dict "envAll" $envAll "application" "master" "container" "elasticsearch_master" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
@@ -105,7 +122,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.master | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
             - /tmp/elasticsearch.sh
-            - start
+            - start_master_node
           lifecycle:
             preStop:
               exec:
@@ -141,6 +158,19 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}"
+            - name: S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_ACCESS_KEY
+            - name: S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $s3UserSecret }}
+                  key: S3_SECRET_KEY
+{{- if .Values.pod.env.master }}
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
+{{- end }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -150,8 +180,6 @@ spec:
               mountPath: /tmp/elasticsearch.sh
               subPath: elasticsearch.sh
               readOnly: true
-            - name: elasticsearch-config
-              mountPath: /usr/share/elasticsearch/config
             - name: elasticsearch-etc
               mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
               subPath: elasticsearch.yml
@@ -160,6 +188,10 @@ spec:
               mountPath: /usr/share/elasticsearch/config/log4j2.properties
               subPath: log4j2.properties
               readOnly: true
+            - name: elasticsearch-etc
+              mountPath: /usr/share/elasticsearch/config/jvm.options
+              subPath: jvm.options
+              readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
@@ -168,8 +200,6 @@ spec:
           emptyDir: {}
         - name: elasticsearch-logs
           emptyDir: {}
-        - name: elasticsearch-config
-          emptyDir: {}
         - name: elasticsearch-bin
           configMap:
             name: elasticsearch-bin
@@ -178,7 +208,19 @@ spec:
           secret:
             secretName: elasticsearch-etc
             defaultMode: 0444
+{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
+{{- if not .Values.storage.master.enabled }}
         - name: storage
           emptyDir: {}
-{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
+{{- else }}
+  volumeClaimTemplates:
+    - metadata:
+        name: storage
+      spec:
+        accessModes: {{ .Values.storage.master.pvc.access_mode }}
+        resources:
+          requests:
+            storage: {{ .Values.storage.master.requests.storage  }}
+        storageClassName: {{ .Values.storage.master.storage_class }}
+{{- end }}
 {{- end }}

elasticsearch/values.yaml

@@ -20,8 +20,8 @@ images:
   tags:
     apache_proxy: docker.io/httpd:2.4
     memory_init: docker.io/openstackhelm/heat:newton-ubuntu_xenial
+    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191119
     curator:  docker.io/bobrik/curator:5.6.0
-    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:5_6_4-20191119
     ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
     s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119
     s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
@@ -123,6 +123,10 @@ dependencies:
         - elasticsearch-register-snapshot-repository
 
 pod:
+  env:
+    client: null
+    data: null
+    master: null
   mandatory_access_control:
     type: apparmor
     elasticsearch-master:
@@ -154,7 +158,7 @@ pod:
             add:
               - IPC_LOCK
               - SYS_RESOURCE
-          readOnlyRootFilesystem: true
+          readOnlyRootFilesystem: false
     master:
       pod:
         runAsUser: 0
@@ -168,7 +172,7 @@ pod:
             add:
               - IPC_LOCK
               - SYS_RESOURCE
-          readOnlyRootFilesystem: true
+          readOnlyRootFilesystem: false
     es_cluster_wait:
       pod:
         runAsUser: 0
@@ -418,6 +422,7 @@ conf:
     LoadModule unixd_module modules/mod_unixd.so
     LoadModule status_module modules/mod_status.so
     LoadModule autoindex_module modules/mod_autoindex.so
+    LoadModule rewrite_module modules/mod_rewrite.so
 
     <IfModule unixd_module>
     User daemon
@@ -480,10 +485,6 @@ conf:
           Require valid-user
       </Location>
 
-      # Restrict access to the Elasticsearch Update API endpoint to prevent modification of indexed documents
-      <Location /*/_doc/*/_update*>
-          Require all denied
-      </Location>
       # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents
       <Location /*/_update_by_query*>
           Require all denied
@@ -492,30 +493,49 @@ conf:
       <Location /*/_delete_by_query*>
           Require all denied
       </Location>
-
-      # Prohibit DELETE methods on the document API endpoint
-      <Location /*/_doc/*>
-          AllowMethods GET POST OPTIONS
-          ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
-          ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
-          AuthName "Elasticsearch"
-          AuthType Basic
-          AuthBasicProvider file ldap
-          AuthUserFile /usr/local/apache2/conf/.htpasswd
-          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
-          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
-          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
-          Require valid-user
-      </Location>
     </VirtualHost>
   log4j2: |
     status = error
     appender.console.type = Console
     appender.console.name = console
     appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+    appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n
     rootLogger.level = info
     rootLogger.appenderRef.console.ref = console
+  jvm_options: |
+    -Xms1g
+    -Xmx1g
+    -XX:+UseConcMarkSweepGC
+    -XX:CMSInitiatingOccupancyFraction=75
+    -XX:+UseCMSInitiatingOccupancyOnly
+    -Des.networkaddress.cache.ttl=60
+    -Des.networkaddress.cache.negative.ttl=10
+    -XX:+AlwaysPreTouch
+    -Xss1m
+    -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -XX:-OmitStackTraceInFastThrow
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Djava.io.tmpdir=${ES_TMPDIR}
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:HeapDumpPath=data
+    -XX:ErrorFile=logs/hs_err_pid%p.log
+    8:-XX:+PrintGCDetails
+    8:-XX:+PrintGCDateStamps
+    8:-XX:+PrintTenuringDistribution
+    8:-XX:+PrintGCApplicationStoppedTime
+    8:-Xloggc:logs/gc.log
+    8:-XX:+UseGCLogFileRotation
+    8:-XX:NumberOfGCLogFiles=32
+    8:-XX:GCLogFileSize=64m
+    9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+    9-:-Djava.locale.providers=COMPAT
+    10-:-XX:UseAVX=2
   init:
     max_map_count: 262144
   ceph:
@@ -636,21 +656,17 @@ conf:
       cluster:
         name: elasticsearch
       discovery:
-        zen:
+        # NOTE(srwilkers): This gets configured dynamically via endpoint lookups
-          ping.unicast.hosts: ${DISCOVERY_SERVICE}
+        seed_hosts: null
-          minimum_master_nodes: 2
-      http:
-        enabled: ${HTTP_ENABLE}
-        compression: true
       network:
         host: 0.0.0.0
-      cloud:
+      s3:
-        aws:
+        client:
-          protocol: http
+          default:
-          s3:
             # NOTE(srwilkers): This gets configured dynamically via endpoint
             # lookups
             endpoint: null
+            protocol: http
       node:
         ingest: ${NODE_INGEST}
         master: ${NODE_MASTER}
@@ -658,8 +674,8 @@ conf:
         name: ${NODE_NAME}
         max_local_storage_nodes: 3
       path:
-        data: /usr/share/elasticsearch/data
+        data: /data
-        logs: /usr/share/elasticsearch/logs
+        logs: /logs
     snapshots:
       enabled: false
       # NOTE(srwilkers): The path for the radosgw s3 endpoint gets populated
@@ -680,33 +696,32 @@ conf:
       timeout: 20s
   templates:
     fluent:
-      template: "logstash-*"
       index_patterns: "logstash-*"
       settings:
-        number_of_shards: 1
+        index:
+          number_of_shards: 1
       mappings:
-        fluent:
+        properties:
-          properties:
+          kubernetes:
-            kubernetes:
+            properties:
-              properties:
+              container_name:
-                container_name:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
+              docker_id:
-                docker_id:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
+              host:
-                host:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
+              namespace_name:
-                namespace_name:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
+              pod_id:
-                pod_id:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
+              pod_name:
-                pod_name:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -829,13 +844,22 @@ network:
       port: 30920
 
 storage:
-  enabled: true
+  data:
-  pvc:
+    enabled: true
-    name: pvc-elastic
+    pvc:
-    access_mode: [ "ReadWriteOnce" ]
+      name: pvc-elastic
-  requests:
+      access_mode: [ "ReadWriteOnce" ]
-    storage: 5Gi
+    requests:
-  storage_class: general
+      storage: 5Gi
+    storage_class: general
+  master:
+    enabled: true
+    pvc:
+      name: pvc-elastic
+      access_mode: [ "ReadWriteOnce" ]
+    requests:
+      storage: 1Gi
+    storage_class: general
 
 
 manifests:
@@ -847,7 +871,6 @@ manifests:
   cron_curator: true
   cron_verify_repositories: true
   deployment_client: true
-  deployment_master: true
   ingress: true
   job_cluster_wait: true
   job_elasticsearch_templates: true
@@ -870,3 +893,4 @@ manifests:
   service_ingress: true
   service_logging: true
   statefulset_data: true
+  statefulset_master: true

kibana/templates/bin/_create_kibana_index_patterns.sh.tpl

@@ -18,9 +18,13 @@ set -ex
 
 {{- range .Values.conf.create_kibana_indexes.indexes }}
 curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/index-pattern/{{ . }}-*" -H 'Content-Type: application/json' \
+  -XPOST "${KIBANA_ENDPOINT}/api/saved_objects/index-pattern/{{ . }}*" -H 'kbn-xsrf: true' \
-  -d '{"title":"{{ . }}-*","timeFieldName":"@timestamp","notExpandable":true}'
+  -H 'Content-Type: application/json' -d \
-{{- end }}
+  '{"attributes":{"title":"{{ . }}-*","timeFieldName":"@timestamp"}}'
+
+{{ end }}
+
 curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-  -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/config/5.6.4" -H 'Content-Type: application/json' \
+  -XPOST "${KIBANA_ENDPOINT}/api/kibana/settings/defaultIndex" -H 'kbn-xsrf: true' \
-  -d '{"defaultIndex" : "{{ .Values.conf.create_kibana_indexes.default_index }}-*"}'
+  -H 'Content-Type: application/json' -d \
+  '{"value" : "{{ .Values.conf.create_kibana_indexes.default_index }}*"}'

kibana/templates/bin/_kibana.sh.tpl

@@ -15,14 +15,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-set -ex
+set -e
 COMMAND="${@:-start}"
 
 function start () {
-  exec kibana \
+  exec /usr/share/kibana/bin/kibana \
-    --elasticsearch.url="$ELASTICSEARCH_URL" \
+    --elasticsearch.hosts="${ELASTICSEARCH_HOSTS}" \
-    --elasticsearch.username="$ELASTICSEARCH_USERNAME" \
+    --elasticsearch.username="${ELASTICSEARCH_USERNAME}" \
-    --elasticsearch.password="$ELASTICSEARCH_PASSWORD"
+    --elasticsearch.password="${ELASTICSEARCH_PASSWORD}"
 }
 
 function stop () {

kibana/templates/deployment.yaml

@@ -18,6 +18,13 @@ limitations under the License.
 {{- $envAll := . }}
 
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+{{- $esUser := .Values.endpoints.elasticsearch.auth.admin.username }}
+{{- $esPass := .Values.endpoints.elasticsearch.auth.admin.password }}
+{{- $authHeader := printf "%s:%s" $esUser $esPass | b64enc }}
+
+{{- $esScheme := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+{{- $esSvc := tuple "elasticsearch" "default" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $esHosts := printf "%s://%s" $esScheme $esSvc }}
 
 {{- $serviceAccountName := "kibana" }}
 {{ tuple $envAll "kibana" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -101,9 +108,18 @@ spec:
           ports:
             - name: kibana
               containerPort: {{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          readinessProbe:
+            httpGet:
+              path: /status
+              port: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              httpHeaders:
+                - name: Authorization
+                  value: Basic {{ $authHeader }}
+            initialDelaySeconds: 20
+            periodSeconds: 10
           env:
-            - name: ELASTICSEARCH_URL
+            - name: ELASTICSEARCH_HOSTS
-              value: {{ tuple "elasticsearch" "default" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+              value: {{ $esHosts }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -124,6 +140,8 @@ spec:
               subPath: kibana.sh
               readOnly: true
             - name: pod-etc-kibana
+              mountPath: /usr/share/kibana/config
+            - name: pod-optimize-kibana
               mountPath: /usr/share/kibana/optimize
             - name: kibana-etc
               mountPath: /usr/share/kibana/config/kibana.yml
@@ -137,6 +155,8 @@ spec:
             medium: "Memory"
         - name: pod-etc-kibana
           emptyDir: {}
+        - name: pod-optimize-kibana
+          emptyDir: {}
         - name: kibana-bin
           configMap:
             name: kibana-bin

kibana/templates/job-register-kibana-indexes.yaml

@@ -53,6 +53,8 @@ spec:
                 secretKeyRef:
                   name: {{ $esUserSecret }}
                   key: ELASTICSEARCH_PASSWORD
+            - name: KIBANA_ENDPOINT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
             - name: ELASTICSEARCH_ENDPOINT
               value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
           command:

kibana/values.yaml

@@ -23,7 +23,7 @@ labels:
 images:
   tags:
     apache_proxy: docker.io/httpd:2.4
-    kibana: docker.io/kibana:5.6.4
+    kibana: docker.elastic.co/kibana/kibana-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
     register_kibana_indexes: docker.io/openstackhelm/heat:newton-ubuntu_xenial
@@ -38,17 +38,19 @@ pod:
   security_context:
     dashboard:
       pod:
-        runAsUser: 999
+        runAsUser: 1000
       container:
         apache_proxy:
           runAsUser: 0
           readOnlyRootFilesystem: false
         kibana:
+          fsGroup: 1000
+          runAsNonRoot: true
           allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
+          readOnlyRootFilesystem: false
     register_kibana_indexes:
       pod:
-        runAsUser: 999
+        runAsUser: 1000
       container:
         register_kibana_indexes:
           allowPrivilegeEscalation: false
@@ -250,7 +252,6 @@ conf:
       startupTimeout: 5000
     kibana:
       defaultAppId: discover
-      index: .kibana
     logging:
       quiet: false
       silent: false
@@ -258,13 +259,16 @@ conf:
     ops:
       interval: 5000
     server:
+      rewriteBasePath: false
       host: localhost
+      name: kibana
       maxPayloadBytes: 1048576
       port: 5601
       ssl:
         enabled: false
   create_kibana_indexes:
     enabled: true
+    version: 7.1.0
     indexes:
       - logstash
       - openstack

tools/deployment/armada/manifests/armada-lma.yaml

@@ -316,7 +316,7 @@ data:
             secret_key: ${RADOSGW_S3_ELASTICSEARCH_SECRET_KEY}
     pod:
       replicas:
-        data: 1
+        data: 2
         master: 2
     labels:
       elasticsearch:
@@ -341,74 +341,71 @@ data:
           settings:
             number_of_shards: 1
           mappings:
-            syslog:
+            properties:
-              properties:
+              cluster:
-                cluster:
+                type: keyword
-                  type: keyword
+              app:
-                app:
+                type: keyword
-                  type: keyword
+              pid:
-                pid:
+                type: integer
-                  type: integer
+              host:
-                host:
+                type: keyword
-                  type: keyword
+              log:
-                log:
+                type: text
-                  type: text
         oslo_openstack_fluentd:
           template: "openstack-*"
           index_patterns: "openstack-*"
           settings:
             number_of_shards: 1
           mappings:
-            oslo_openstack_fluentd:
+            properties:
-              properties:
+              extra:
-                extra:
+                properties:
-                  properties:
+                  project:
-                    project:
+                    type: text
-                      type: text
+                    norms: false
-                      norms: false
+                  version:
-                    version:
+                    type: text
-                      type: text
+                    norms: false
-                      norms: false
+              filename:
-                filename:
+                type: text
-                  type: text
+                norms: false
-                  norms: false
+              funcname:
-                funcname:
+                type: text
-                  type: text
+                norms: false
-                  norms: false
+              message:
-                message:
+                type: text
-                  type: text
+                norms: false
-                  norms: false
+              process_name:
-                process_name:
+                type: keyword
-                  type: keyword
+                index: false
-                  index: false
         docker_fluentd:
           template: "logstash-*"
           index_patterns: "logstash-*"
           settings:
             number_of_shards: 1
           mappings:
-            docker_fluentd:
+            properties:
-              properties:
+              kubernetes:
-                kubernetes:
+                properties:
-                  properties:
+                  container_name:
-                    container_name:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
+                  docker_id:
-                    docker_id:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
+                  host:
-                    host:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
+                  namespace_name:
-                    namespace_name:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
+                  pod_id:
-                    pod_id:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
+                  pod_name:
-                    pod_name:
+                    type: keyword
-                      type: keyword
+                    index: false
-                      index: false
       curator:
         action_file:
           actions:

tools/deployment/multinode/120-elasticsearch.sh

@@ -26,7 +26,7 @@ jobs:
     cron: "*/3 * * * *"
 pod:
   replicas:
-    data: 1
+    data: 2
     master: 2
 conf:
   elasticsearch:

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -29,7 +29,7 @@ monitoring:
     enabled: true
 pod:
   replicas:
-    data: 1
+    data: 2
     master: 2
 conf:
   elasticsearch: