From 95e3c21df4229252897514c0244e774e9319f251 Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Wed, 11 Mar 2020 23:35:06 -0500 Subject: [PATCH] Settings for Remote Elasticsearch Clusters This change adds a new Deployment to the Elasticsearch chart to add a set of "gateway" nodes to the Elasticsearch cluster. These nodes will facilitate Elasticsearch remote cluster, for features such as cross cluster search. Co-Authored-By: David Smith Change-Id: Ic4ac988a922a12addce3c65e0ef4099d46bbc784 --- .../templates/deployment-client.yaml | 2 + .../templates/deployment-gateway.yaml | 171 ++++++++++++++++++ .../templates/secret-ingress-tls.yaml | 17 ++ elasticsearch/templates/service-gateway.yaml | 30 +++ elasticsearch/templates/statefulset-data.yaml | 2 + .../templates/statefulset-master.yaml | 2 + elasticsearch/values.yaml | 52 ++++++ .../values_overrides/remote-cluster.yaml | 30 +++ 8 files changed, 306 insertions(+) create mode 100644 elasticsearch/templates/deployment-gateway.yaml create mode 100644 elasticsearch/templates/secret-ingress-tls.yaml create mode 100644 elasticsearch/templates/service-gateway.yaml create mode 100644 elasticsearch/values_overrides/remote-cluster.yaml diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index 2031778ee..0f9c833ab 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -146,6 +146,8 @@ spec: value: "true" - name: NODE_DATA value: "false" + - name: NODE_GATEWAY + value: "false" - name: HTTP_ENABLE value: "true" - name: DISCOVERY_SERVICE diff --git a/elasticsearch/templates/deployment-gateway.yaml b/elasticsearch/templates/deployment-gateway.yaml new file mode 100644 index 000000000..3bbac928b --- /dev/null +++ b/elasticsearch/templates/deployment-gateway.yaml @@ -0,0 +1,171 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.network.remote_clustering.enabled }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} + +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elasticsearch-remote-gateway" }} +{{ tuple $envAll "elasticsearch_gateway" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch-gateway + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + replicas: {{ .Values.pod.replicas.gateway }} + selector: + matchLabels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "elasticsearch-gateway" "containerNames" (list "elasticsearch-remote-gateway") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "gateway" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.gateway.node_selector_key }}: {{ .Values.labels.gateway.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.client.timeout | default "600" }} + initContainers: +{{ tuple $envAll "elasticsearch" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: memory-map-increase +{{ tuple $envAll "memory_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "gateway" "container" "memory_map_increase" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - sysctl + - -w + - vm.max_map_count={{ .Values.conf.init.max_map_count }} + containers: + - name: elasticsearch-gateway +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.gateway | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "gateway" "container" "elasticsearch_gateway" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/elasticsearch.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/elasticsearch.sh + - stop + ports: + - name: transport + containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + livenessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_MASTER + value: "false" + - name: NODE_INGEST + value: "true" + - name: NODE_DATA + value: "false" + - name: NODE_GATEWAY + value: "true" + - name: HTTP_ENABLE + value: "false" + - name: DISCOVERY_SERVICE + value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: ES_JAVA_OPTS + value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}" + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_SECRET_KEY +{{- if .Values.pod.env.gateway }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-logs + mountPath: {{ .Values.conf.elasticsearch.config.path.logs }} + - name: elasticsearch-bin + mountPath: /tmp/elasticsearch.sh + subPath: elasticsearch.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-logs + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 + - name: elasticsearch-etc + secret: + secretName: elasticsearch-etc + defaultMode: 0444 + - name: storage + emptyDir: {} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elasticsearch/templates/secret-ingress-tls.yaml b/elasticsearch/templates/secret-ingress-tls.yaml new file mode 100644 index 000000000..d739cdc25 --- /dev/null +++ b/elasticsearch/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "elasticsearch" "backendService" "elasticsearch" ) }} +{{- end }} diff --git a/elasticsearch/templates/service-gateway.yaml b/elasticsearch/templates/service-gateway.yaml new file mode 100644 index 000000000..27b4f1de4 --- /dev/null +++ b/elasticsearch/templates/service-gateway.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.network.remote_clustering.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elasticsearch" "gateway" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: transport + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + nodePort: {{ .Values.network.remote_clustering.node_port.port }} + selector: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + type: NodePort +{{- end }} diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml index 6a44d1796..e67a7c70a 100644 --- a/elasticsearch/templates/statefulset-data.yaml +++ b/elasticsearch/templates/statefulset-data.yaml @@ -124,6 +124,8 @@ spec: value: "false" - name: NODE_DATA value: "true" + - name: NODE_GATEWAY + value: "false" - name: HTTP_ENABLE value: "false" - name: ES_JAVA_OPTS diff --git a/elasticsearch/templates/statefulset-master.yaml b/elasticsearch/templates/statefulset-master.yaml index e2916563f..cfa297100 100644 --- a/elasticsearch/templates/statefulset-master.yaml +++ b/elasticsearch/templates/statefulset-master.yaml @@ -117,6 +117,8 @@ spec: value: "false" - name: NODE_DATA value: "false" + - name: NODE_GATEWAY + value: "false" - name: HTTP_ENABLE value: "false" - name: DISCOVERY_SERVICE diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 7252d71fd..4a4b21941 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -58,6 +58,9 @@ labels: test: node_selector_key: openstack-control-plane node_selector_value: enabled + gateway: + node_selector_key: openstack-control-plane + node_selector_value: enabled dependencies: dynamic: @@ -84,6 +87,10 @@ dependencies: - endpoint: discovery service: elasticsearch jobs: null + elasticsearch_gateway: + services: + - endpoint: discovery + service: elasticsearch elasticsearch_data: services: - endpoint: internal @@ -136,6 +143,18 @@ pod: client: null data: null master: null + gateway: null + secrets: null + mandatory_access_control: + type: apparmor + elasticsearch-master: + elasticsearch-master: runtime/default + elasticsearch-data: + elasticsearch-data: runtime/default + elasticsearch-client: + elasticsearch-client: runtime/default + elasticsearch-gateway: + elasticsearch-gateway: runtime/default security_context: exporter: pod: @@ -209,6 +228,22 @@ pod: # recovery scenarios when the data pods are unexpectedly lost due to # node outages and shard/index recovery is required readOnlyRootFilesystem: false + gateway: + pod: + runAsUser: 0 + container: + memory_map_increase: + privileged: true + readOnlyRootFilesystem: true + apache_proxy: + readOnlyRootFilesystem: false + elasticsearch_gateway: + privileged: true + capabilities: + add: + - IPC_LOCK + - SYS_RESOURCE + readOnlyRootFilesystem: false affinity: anti: type: @@ -221,6 +256,7 @@ pod: master: 3 data: 3 client: 3 + gateway: 3 lifecycle: upgrades: statefulsets: @@ -282,6 +318,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + gateway: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" jobs: curator: requests: @@ -656,6 +699,8 @@ conf: memory_lock: true cluster: name: elasticsearch + remote: + connect: ${NODE_GATEWAY} discovery: # NOTE(srwilkers): This gets configured dynamically via endpoint lookups seed_hosts: null @@ -749,6 +794,7 @@ endpoints: data: elasticsearch-data default: elasticsearch-logging discovery: elasticsearch-discovery + gateway: elasticsaerch-gateway public: elasticsearch host_fqdn_override: default: null @@ -763,6 +809,7 @@ endpoints: default: null scheme: default: http + gateway: tcp port: client: default: 9200 @@ -843,6 +890,10 @@ network: node_port: enabled: false port: 30920 + remote_clustering: + enabled: false + node_port: + port: 30930 storage: data: @@ -889,6 +940,7 @@ manifests: network_policy_exporter: false service_exporter: true network_policy: false + secret_ingress_tls: true service_data: true service_discovery: true service_ingress: true diff --git a/elasticsearch/values_overrides/remote-cluster.yaml b/elasticsearch/values_overrides/remote-cluster.yaml new file mode 100644 index 000000000..093c3cd53 --- /dev/null +++ b/elasticsearch/values_overrides/remote-cluster.yaml @@ -0,0 +1,30 @@ +# Can't use these settings at startup yet becuse of +# https://github.com/elastic/elasticsearch/issues/27006 +# conf: +# elasticsearch: +# config: +# cluster: +# remote: +# remote_elasticsearch: +# seeds: +# - elasticsearch-gateway-1.remote_host:9301 +# - elasticsearch-gateway-2.remote_host:9301 +# - elasticsearch-gateway-3.remote_host:9301 +# skip_unavailale: true +network: + remote_clustering: + enabled: true + +manifests: + cron_curator: false + cron_verify_repositories: false + job_snapshot_repository: false +pod: + replicas: + master: 2 + data: 1 + client: 1 + gateway: 1 +images: + tags: + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_6_2-centos_7