openstack/openstack-helm-infra
Code Issues Proposed changes

Upgrade ElasticSearch and Kibana to v8.9.0

Browse Source 
Change-Id: I5ce965a2abf40bad14f0a8a505c8f3000f110d37
This commit is contained in:
Leontii Istomin 2023-08-15 14:35:23 -05:00
parent b76240f1df
commit 4a74ff2ba9
13 changed files with 49 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elasticsearch/Chart.yaml
View File

@ -12,10 +12,10 @@
---
apiVersion: v1
appVersion: v7.6.2
appVersion: v8.9.0
description: OpenStack-Helm ElasticSearch
name: elasticsearch
version: 0.2.24
version: 0.2.25
home: https://www.elastic.co/
sources:
- https://github.com/elastic/elasticsearch

16
elasticsearch/templates/bin/_elasticsearch.sh.tpl
View File

@ -19,26 +19,26 @@ set -e
COMMAND="${@:-start}"
function initiate_keystore () {
bin/elasticsearch-keystore create
elasticsearch-keystore create
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- range $client, $settings := .Values.storage.s3.clients -}}
{{- $access_key := printf "%s_S3_ACCESS_KEY" ( $client | replace "-" "_" | upper) }}
{{- $secret_key := printf "%s_S3_SECRET_KEY" ( $client | replace "-" "_" | upper) }}
echo ${{$access_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
echo ${{$secret_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
echo ${{$access_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
echo ${{$secret_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
{{- end }}
{{- end }}
{{- if .Values.manifests.certificates }}
{{- $alias := .Values.secrets.tls.elasticsearch.elasticsearch.internal }}
/usr/share/elasticsearch/jdk/bin/keytool -storepasswd -cacerts -new ${ELASTICSEARCH_PASSWORD} -storepass changeit
/usr/share/elasticsearch/jdk/bin/keytool -importcert -alias {{$alias}} -cacerts -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD}
JAVA_KEYTOOL_PATH=/usr/share/elasticsearch/jdk/bin/keytool
TRUSTSTORE_PATH=/usr/share/elasticsearch/config/elasticsearch-java-truststore
${JAVA_KEYTOOL_PATH} -importcert -alias {{$alias}} -keystore ${TRUSTSTORE_PATH} -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD}
${JAVA_KEYTOOL_PATH} -storepasswd -keystore ${TRUSTSTORE_PATH} -new ${ELASTICSEARCH_PASSWORD} -storepass ${ELASTICSEARCH_PASSWORD}
{{- end }}
}
function start () {
ulimit -l unlimited
initiate_keystore
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
}
@ -76,7 +76,6 @@ function allocate_data_node () {
}
function start_master_node () {
ulimit -l unlimited
initiate_keystore
if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ];
then
@ -97,7 +96,6 @@ function start_master_node () {
}
function start_data_node () {
ulimit -l unlimited
initiate_keystore
allocate_data_node &
/usr/local/bin/docker-entrypoint.sh elasticsearch &

10
elasticsearch/templates/deployment-client.yaml
View File

@ -159,14 +159,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NODE_MASTER
value: "false"
- name: NODE_INGEST
value: "true"
- name: NODE_DATA
value: "false"
- name: NODE_GATEWAY
value: "false"
- name: node.roles
value: "[ingest]"
- name: HTTP_ENABLE
value: "true"
- name: DISCOVERY_SERVICE

10
elasticsearch/templates/deployment-gateway.yaml
View File

@ -101,14 +101,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NODE_MASTER
value: "false"
- name: NODE_INGEST
value: "true"
- name: NODE_DATA
value: "false"
- name: NODE_GATEWAY
value: "true"
- name: node.roles
value: "[ingest, gateway]"
- name: HTTP_ENABLE
value: "false"
- name: DISCOVERY_SERVICE

12
elasticsearch/templates/statefulset-data.yaml
View File

@ -76,7 +76,7 @@ spec:
command:
- chown
- -R
- "elasticsearch:"
- "1000:1000"
- {{ .Values.conf.elasticsearch.config.path.data }}
volumeMounts:
- name: storage
@ -124,14 +124,8 @@ spec:
- name: JAVA_KEYSTORE_CERT_PATH
value: "/usr/share/elasticsearch/config/ca.crt"
{{- end }}
- name: NODE_MASTER
value: "false"
- name: NODE_INGEST
value: "false"
- name: NODE_DATA
value: "true"
- name: NODE_GATEWAY
value: "false"
- name: node.roles
value: "[data]"
- name: HTTP_ENABLE
value: "false"
- name: ES_JAVA_OPTS

12
elasticsearch/templates/statefulset-master.yaml
View File

@ -73,7 +73,7 @@ spec:
command:
- chown
- -R
- "elasticsearch:"
- "1000:1000"
- {{ .Values.conf.elasticsearch.config.path.data }}
volumeMounts:
- name: storage
@ -109,14 +109,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NODE_MASTER
value: "true"
- name: NODE_INGEST
value: "false"
- name: NODE_DATA
value: "false"
- name: NODE_GATEWAY
value: "false"
- name: node.roles
value: "[master]"
- name: HTTP_ENABLE
value: "false"
- name: DISCOVERY_SERVICE

55
elasticsearch/values.yaml
View File

@ -19,16 +19,16 @@ images:
tags:
apache_proxy: docker.io/library/httpd:2.4
memory_init: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
elasticsearch: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
elasticsearch: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0
curator: docker.io/bobrik/curator:5.8.1
ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_focal_17.2.6-1-20230508
s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_17.2.6-1-20230508
elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0
image_repo_sync: docker.io/library/docker:17.07.0
pull_policy: "IfNotPresent"
local_registry:
@ -170,11 +170,8 @@ pod:
apache_proxy:
readOnlyRootFilesystem: false
elasticsearch_client:
privileged: true
capabilities:
add:
- IPC_LOCK
- SYS_RESOURCE
runAsUser: 1000
runAsGroup: 1000
readOnlyRootFilesystem: false
master:
pod:
@ -186,11 +183,8 @@ pod:
elasticsearch_perms:
readOnlyRootFilesystem: true
elasticsearch_master:
privileged: true
capabilities:
add:
- IPC_LOCK
- SYS_RESOURCE
runAsUser: 1000
runAsGroup: 1000
readOnlyRootFilesystem: false
snapshot_repository:
pod:
@ -214,11 +208,8 @@ pod:
elasticsearch_perms:
readOnlyRootFilesystem: true
elasticsearch_data:
privileged: true
capabilities:
add:
- IPC_LOCK
- SYS_RESOURCE
runAsUser: 1000
runAsGroup: 1000
# NOTE: This was changed from true to false to account for
# recovery scenarios when the data pods are unexpectedly lost due to
# node outages and shard/index recovery is required
@ -233,11 +224,8 @@ pod:
apache_proxy:
readOnlyRootFilesystem: false
elasticsearch_gateway:
privileged: true
capabilities:
add:
- IPC_LOCK
- SYS_RESOURCE
runAsUser: 1000
runAsGroup: 1000
readOnlyRootFilesystem: false
curator:
pod:
@ -300,7 +288,7 @@ pod:
liveness:
enabled: true
params:
initialDelaySeconds: 30
initialDelaySeconds: 60
periodSeconds: 10
mounts:
elasticsearch:
@ -581,6 +569,10 @@ conf:
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
{{- if .Values.manifests.certificates }}
-Djavax.net.ssl.trustStore=/usr/share/elasticsearch/config/elasticsearch-java-truststore
-Djavax.net.ssl.trustStorePassword={{ .Values.endpoints.elasticsearch.auth.admin.password }}
{{- end }}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
@ -713,12 +705,15 @@ conf:
blacklist: ['elasticsearch', 'urllib3']
elasticsearch:
config:
xpack:
security:
enabled: false
bootstrap:
memory_lock: true
# As far as we run the pod as non-root, we can't make locking memory unlimited.
# configure the memory locking limits on host itself of disable swap completely.
memory_lock: false
cluster:
name: elasticsearch
remote:
connect: ${NODE_GATEWAY}
discovery:
# NOTE(srwilkers): This gets configured dynamically via endpoint lookups
seed_hosts: null
@ -726,12 +721,6 @@ conf:
host: 0.0.0.0
s3:
client: {}
node:
ingest: ${NODE_INGEST}
master: ${NODE_MASTER}
data: ${NODE_DATA}
name: ${NODE_NAME}
max_local_storage_nodes: 3
path:
data: /data
logs: /logs

1
elasticsearch/values_overrides/tls.yaml
View File

@ -137,6 +137,7 @@ conf:
config:
xpack:
security:
enabled: true
transport:
ssl:
enabled: true

4
kibana/Chart.yaml
View File

@ -12,10 +12,10 @@
---
apiVersion: v1
appVersion: v7.1.0
appVersion: v8.9.0
description: OpenStack-Helm Kibana
name: kibana
version: 0.1.12
version: 0.1.13
home: https://www.elastic.co/products/kibana
sources:
- https://github.com/elastic/kibana

14
kibana/values.yaml
View File

@ -22,7 +22,7 @@ labels:
images:
tags:
apache_proxy: docker.io/library/httpd:2.4
kibana: docker.elastic.co/kibana/kibana-oss:7.1.0
kibana: docker.elastic.co/kibana/kibana:8.9.0
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
image_repo_sync: docker.io/library/docker:17.07.0
register_kibana_indexes: docker.io/openstackhelm/heat:wallaby-ubuntu_focal
@ -286,29 +286,19 @@ conf:
kibana:
elasticsearch:
pingTimeout: 1500
preserveHost: true
requestTimeout: 30000
shardTimeout: 0
startupTimeout: 5000
kibana:
defaultAppId: discover
logging:
quiet: false
silent: false
verbose: false
ops:
interval: 5000
server:
rewriteBasePath: false
host: localhost
name: kibana
maxPayloadBytes: 1048576
maxPayload: 1048576
port: 5601
ssl:
enabled: false
create_kibana_indexes:
enabled: true
version: 7.1.0
indexes:
base:
- logstash

1
releasenotes/notes/elasticsearch.yaml
View File

@ -34,4 +34,5 @@ elasticsearch:
- 0.2.22 Update all Ceph images to Focal
- 0.2.23 Add configurable liveness probe for elasticsearch client
- 0.2.24 Update Ceph to 17.2.6
- 0.2.25 Update ElasticSearch to 8.9.0
...

1
releasenotes/notes/kibana.yaml
View File

@ -13,4 +13,5 @@ kibana:
- 0.1.10 Update image defaults
- 0.1.11 Added OCI registry authentication
- 0.1.12 Added feedback http_code 200 for kibana indexes
- 0.1.13 Update Kibana to 8.9.0
...

2
tools/deployment/osh-infra-logging/050-elasticsearch.sh
View File

@ -45,7 +45,7 @@ conf:
slm_policy:
endpoint: _slm/policy/snapshots
body:
schedule: "0 */3 * * * ?"
schedule: "0 */15 * * * ?"
name: "<snapshot-{now/d}>"
repository: ceph-rgw
config: