From fd4bf572111b75f1a73396c6d3b714259b53a024 Mon Sep 17 00:00:00 2001
From: "Lo, Chi (cl566n)" <cl566n@att.com>
Date: Mon, 26 Apr 2021 11:42:04 -0700
Subject: [PATCH] Enable TLS for Elasticsearch

The change enables:

(1) TLS for the Elasticsearch transport networking layer. The
    transport networking layer is used for internal communication
    between nodes in a cluster.

(2) TLS path between Elasticsearch and Ceph-rgw host.

Change-Id: Ifb6cb5db19bc5db2c8cb914f6a5887cf3d0f9434
---
 elasticsearch/Chart.yaml                       |  2 +-
 .../templates/bin/_create_s3_buckets.sh.tpl    |  2 +-
 .../templates/bin/_elasticsearch.sh.tpl        |  7 ++++++-
 elasticsearch/templates/deployment-client.yaml | 10 ++++++++++
 .../templates/deployment-gateway.yaml          | 11 +++++++++++
 elasticsearch/templates/job-s3-bucket.yaml     |  4 ++++
 elasticsearch/templates/statefulset-data.yaml  |  6 ++++--
 .../templates/statefulset-master.yaml          | 13 +++++++++++--
 elasticsearch/values_overrides/tls.yaml        | 11 +++++++++++
 helm-toolkit/Chart.yaml                        |  2 +-
 .../manifests/_job-s3-bucket.yaml.tpl          | 18 ++++++++++++++++++
 releasenotes/notes/elasticsearch.yaml          |  1 +
 releasenotes/notes/helm-toolkit.yaml           |  1 +
 13 files changed, 80 insertions(+), 8 deletions(-)

diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml
index 2baf03ce1..8cb3ff4fd 100644
--- a/elasticsearch/Chart.yaml
+++ b/elasticsearch/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.6.2
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.5
+version: 0.2.6
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch
diff --git a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl
index 1b09067bd..ed9ed1f07 100644
--- a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl
+++ b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl
@@ -54,7 +54,7 @@ CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 if [ "$RGW_PROTO" = "http" ]; then
   CONNECTION_ARGS+=" --no-ssl"
 else
-  CONNECTION_ARGS+=" --no-check-certificate"
+  CONNECTION_ARGS+=" ${TLS_OPTION}"
 fi
 
 USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
diff --git a/elasticsearch/templates/bin/_elasticsearch.sh.tpl b/elasticsearch/templates/bin/_elasticsearch.sh.tpl
index 008e80510..778f27657 100644
--- a/elasticsearch/templates/bin/_elasticsearch.sh.tpl
+++ b/elasticsearch/templates/bin/_elasticsearch.sh.tpl
@@ -19,7 +19,6 @@ set -e
 COMMAND="${@:-start}"
 
 function initiate_keystore () {
-  set -ex
   bin/elasticsearch-keystore create
 
   {{- if .Values.conf.elasticsearch.snapshots.enabled }}
@@ -30,6 +29,12 @@ function initiate_keystore () {
   echo ${{$secret_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
   {{- end }}
   {{- end }}
+
+  {{- if .Values.manifests.certificates }}
+  {{- $alias := .Values.secrets.tls.elasticsearch.elasticsearch.internal }}
+  /usr/share/elasticsearch/jdk/bin/keytool -storepasswd -cacerts -new ${ELASTICSEARCH_PASSWORD} -storepass changeit
+  /usr/share/elasticsearch/jdk/bin/keytool -importcert -alias {{$alias}} -cacerts -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD}
+  {{- end }}
 }
 
 function start () {
diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml
index a87e8e72e..ed66fd926 100644
--- a/elasticsearch/templates/deployment-client.yaml
+++ b/elasticsearch/templates/deployment-client.yaml
@@ -172,6 +172,15 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
+{{- if .Values.manifests.certificates }}
+            - name: JAVA_KEYSTORE_CERT_PATH
+              value: "/usr/share/elasticsearch/config/ca.crt"
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
+{{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
@@ -204,6 +213,7 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
diff --git a/elasticsearch/templates/deployment-gateway.yaml b/elasticsearch/templates/deployment-gateway.yaml
index e66a1e2a6..6348509a0 100644
--- a/elasticsearch/templates/deployment-gateway.yaml
+++ b/elasticsearch/templates/deployment-gateway.yaml
@@ -115,6 +115,15 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
+{{- if .Values.manifests.certificates }}
+            - name: JAVA_KEYSTORE_CERT_PATH
+              value: "/usr/share/elasticsearch/config/ca.crt"
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
+{{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
@@ -144,6 +153,7 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -160,5 +170,6 @@ spec:
             defaultMode: 0444
         - name: storage
           emptyDir: {}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}
diff --git a/elasticsearch/templates/job-s3-bucket.yaml b/elasticsearch/templates/job-s3-bucket.yaml
index cff2133ca..8ea633d8d 100644
--- a/elasticsearch/templates/job-s3-bucket.yaml
+++ b/elasticsearch/templates/job-s3-bucket.yaml
@@ -15,5 +15,9 @@ limitations under the License.
 {{- if and (.Values.manifests.job_s3_bucket) (.Values.conf.elasticsearch.snapshots.enabled) }}
 {{- $esBucket := .Values.conf.elasticsearch.snapshots.bucket }}
 {{- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" "s3Bucket" $esBucket -}}
+{{- if .Values.manifests.certificates }}
+{{- $_ := set $s3BucketJob "tlsCertificateSecret" .Values.secrets.tls.elasticsearch.elasticsearch.internal -}}
+{{- $_ := set $s3BucketJob "tlsCertificatePath" "/etc/elasticsearch/certs/ca.crt" -}}
+{{- end }}
 {{ $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" }}
 {{- end -}}
diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml
index 2b7bc32a5..b6befc0ac 100644
--- a/elasticsearch/templates/statefulset-data.yaml
+++ b/elasticsearch/templates/statefulset-data.yaml
@@ -117,7 +117,9 @@ spec:
               value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
 {{- if .Values.manifests.certificates }}
             - name: CACERT_OPTION
-              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+              value: "--cacert /usr/share/elasticsearch/config/ca.crt"
+            - name: JAVA_KEYSTORE_CERT_PATH
+              value: "/usr/share/elasticsearch/config/ca.crt"
 {{- end }}
             - name: NODE_MASTER
               value: "false"
@@ -162,7 +164,7 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
-{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
diff --git a/elasticsearch/templates/statefulset-master.yaml b/elasticsearch/templates/statefulset-master.yaml
index e9f7e541e..fe41e48c2 100644
--- a/elasticsearch/templates/statefulset-master.yaml
+++ b/elasticsearch/templates/statefulset-master.yaml
@@ -15,8 +15,6 @@ limitations under the License.
 {{- if .Values.manifests.statefulset_master }}
 {{- $envAll := . }}
 
-{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
-
 {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
 
 {{- $serviceAccountName := "elasticsearch-master" }}
@@ -122,6 +120,15 @@ spec:
               value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
             - name: ES_JAVA_OPTS
               value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}"
+{{- if .Values.manifests.certificates }}
+            - name: JAVA_KEYSTORE_CERT_PATH
+              value: "/usr/share/elasticsearch/config/ca.crt"
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.elasticsearch.user }}
+                  key: ELASTICSEARCH_PASSWORD
+{{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
@@ -151,6 +158,7 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -165,6 +173,7 @@ spec:
           secret:
             secretName: elasticsearch-etc
             defaultMode: 0444
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- if not .Values.storage.master.enabled }}
         - name: storage
diff --git a/elasticsearch/values_overrides/tls.yaml b/elasticsearch/values_overrides/tls.yaml
index 85b99bf01..50f4f5b97 100644
--- a/elasticsearch/values_overrides/tls.yaml
+++ b/elasticsearch/values_overrides/tls.yaml
@@ -133,6 +133,17 @@ conf:
       SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
       SSLHonorCipherOrder     on
     </VirtualHost>
+  elasticsearch:
+    config:
+      xpack:
+        security:
+          transport:
+            ssl:
+              enabled: true
+              verification_mode: certificate
+              key: /usr/share/elasticsearch/config/tls.key
+              certificate: /usr/share/elasticsearch/config/tls.crt
+              certificate_authorities: ["/usr/share/elasticsearch/config/ca.crt"]
 manifests:
   certificates: true
 ...
diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml
index db0e735ae..f9fe2138b 100644
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.12
+version: 0.2.13
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
diff --git a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
index a70c6c1b4..b26bdb4f2 100644
--- a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
+++ b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
@@ -31,6 +31,8 @@ limitations under the License.
 {{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
 {{- $s3UserSecret := index $envAll.Values.secrets.rgw $serviceName -}}
 {{- $s3Bucket := index . "s3Bucket" | default $serviceName }}
+{{- $tlsCertificateSecret := index . "tlsCertificateSecret" -}}
+{{- $tlsCertificatePath := index . "tlsCertificatePath" -}}
 
 {{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "s3-bucket" }}
 {{ tuple $envAll "s3_bucket" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -73,6 +75,10 @@ spec:
             - -c
             - /tmp/create-s3-bucket.sh
           env:
+{{- if and ($tlsCertificatePath) ($tlsCertificateSecret) }}
+            - name: TLS_OPTION
+              value: {{ printf "--ca-certs=%s" $tlsCertificatePath | quote }}
+{{- end }}
 {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
 {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
 {{- end }}
@@ -96,6 +102,12 @@ spec:
               subPath: key
               readOnly: true
             {{ end }}
+{{- if and ($tlsCertificatePath) ($tlsCertificateSecret) }}
+            - name: {{ $tlsCertificateSecret }}
+              mountPath: {{ $tlsCertificatePath }}
+              subPath: ca.crt
+              readOnly: true
+{{- end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -120,4 +132,10 @@ spec:
           secret:
             secretName: pvc-ceph-client-key
         {{ end }}
+{{- if and ($tlsCertificatePath) ($tlsCertificateSecret) }}
+        - name: {{ $tlsCertificateSecret }}
+          secret:
+            secretName: {{ $tlsCertificateSecret }}
+            defaultMode: 292
+{{- end }}
 {{- end -}}
diff --git a/releasenotes/notes/elasticsearch.yaml b/releasenotes/notes/elasticsearch.yaml
index daadf34f6..3fd8df180 100644
--- a/releasenotes/notes/elasticsearch.yaml
+++ b/releasenotes/notes/elasticsearch.yaml
@@ -15,4 +15,5 @@ elasticsearch:
   - 0.2.3 Add configurable backoffLimit to templates job
   - 0.2.4 Update helm-test script
   - 0.2.5 Enable TLS with Kibana
+  - 0.2.6 Enable TLS path between nodes in cluster and TLS path between ceph-rgw
 ...
diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml
index ab03ff5b8..7e270fcf5 100644
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@@ -19,4 +19,5 @@ helm-toolkit:
   - 0.2.10 Add more S3 configuration options
   - 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9
   - 0.2.12 Remove hook-delete-policy
+  - 0.2.13 Modify connection args for s3 bucket creation when TLS is enabled
 ...