openstack/openstack-helm-infra
Code Issues Proposed changes

Merge "Elasticsearch/Kibana: Update version to 7.1.0"

Browse Source
This commit is contained in:
Zuul 2019-12-06 18:16:05 +00:00 committed by Gerrit Code Review
commit bb7c2787c3
15 changed files with 342 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
elasticsearch/templates/bin/_elasticsearch.sh.tpl
View File

@ -15,12 +15,21 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
{{- $envAll := . }}
set -e
COMMAND="${@:-start}"
function initiate_keystore () {
bin/elasticsearch-keystore create
echo ${S3_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.access_key
echo ${S3_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.secret_key
}
function start () {
ulimit -l unlimited
exec /docker-entrypoint.sh elasticsearch
initiate_keystore
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
}
function stop () {
@ -42,10 +51,32 @@ function allocate_data_node () {
echo "Node ${NODE_NAME} is ready to be used"
}
function start_master_node () {
ulimit -l unlimited
initiate_keystore
if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ];
then
{{ if empty $envAll.Values.conf.elasticsearch.config.cluster.initial_master_nodes -}}
{{- $_ := set $envAll.Values "__eligible_masters" ( list ) }}
{{- range $podInt := until ( atoi (print $envAll.Values.pod.replicas.master ) ) }}
{{- $eligibleMaster := printf "elasticsearch-master-%s" (toString $podInt) }}
{{- $__eligible_masters := append $envAll.Values.__eligible_masters $eligibleMaster }}
{{- $_ := set $envAll.Values "__eligible_masters" $__eligible_masters }}
{{- end -}}
{{- $masters := include "helm-toolkit.utils.joinListWithComma" $envAll.Values.__eligible_masters -}}
echo {{$masters}} >> {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt
exec /usr/local/bin/docker-entrypoint.sh elasticsearch -Ecluster.initial_master_nodes={{$masters}}
{{- end }}
else
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
fi
}
function start_data_node () {
ulimit -l unlimited
initiate_keystore
allocate_data_node &
/docker-entrypoint.sh elasticsearch &
/usr/local/bin/docker-entrypoint.sh elasticsearch &
function drain_data_node () {
echo "Prepare to migrate data off node ${NODE_NAME}"
echo "Move all data from node ${NODE_NAME}"

4
elasticsearch/templates/bin/_register-repository.sh.tpl
View File

@ -32,9 +32,7 @@ function register_snapshot_repository() {
"settings": {
"endpoint": "'"$RGW_HOST"'",
"protocol": "http",
"bucket": "'"$S3_BUCKET"'",
"access_key": "'"$S3_ACCESS_KEY"'",
"secret_key": "'"$S3_SECRET_KEY"'"
"bucket": "'"$S3_BUCKET"'"
}
}' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
if [ "$result" == "True" ];

20
elasticsearch/templates/configmap-etc-elasticsearch.yaml
View File

@ -17,23 +17,18 @@ limitations under the License.
{{- if .Values.manifests.configmap_etc_elasticsearch }}
{{- $envAll := . }}
{{- if empty .Values.conf.elasticsearch.config.cloud.aws.access_key -}}
{{- set .Values.conf.elasticsearch.config.cloud.aws "access_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.access_key -}}
{{- end -}}
{{- if empty .Values.conf.elasticsearch.config.cloud.aws.secret_key -}}
{{- set .Values.conf.elasticsearch.config.cloud.aws "secret_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.secret_key -}}
{{- end -}}
{{- if empty .Values.endpoints.ceph_object_store.path.default -}}
{{- set .Values.endpoints.ceph_object_store.path "default" .Values.conf.elasticsearch.snapshots.bucket -}}
{{- end -}}
{{- if empty .Values.conf.elasticsearch.config.cloud.aws.s3.endpoint -}}
{{- if empty .Values.conf.elasticsearch.config.s3.client.default.endpoint -}}
{{- $radosgw_host := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
{{- $bucket_path := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" -}}
{{- $s3_endpoint := printf "%s/%s" $radosgw_host $bucket_path -}}
{{- set .Values.conf.elasticsearch.config.cloud.aws.s3 "endpoint" $s3_endpoint -}}
{{- set .Values.conf.elasticsearch.config.s3.client.default "endpoint" $radosgw_host -}}
{{- end -}}
{{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}}
{{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}}
{{- set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
{{- end -}}
---
apiVersion: v1
@ -46,4 +41,5 @@ data:
#NOTE(portdirect): this must be last, to work round helm ~2.7 bug.
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.log4j2 "key" "log4j2.properties" "format" "Secret") | indent 2 }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.jvm_options "key" "jvm.options" "format" "Secret") | indent 2 }}
{{- end }}

24
elasticsearch/templates/deployment-client.yaml
View File

@ -18,6 +18,7 @@ limitations under the License.
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
@ -177,7 +178,7 @@ spec:
- name: NODE_MASTER
value: "false"
- name: NODE_INGEST
value: "false"
value: "true"
- name: NODE_DATA
value: "false"
- name: HTTP_ENABLE
@ -186,6 +187,19 @@ spec:
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
{{- if .Values.pod.env.client }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
{{- end }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
@ -195,8 +209,6 @@ spec:
mountPath: /tmp/elasticsearch.sh
subPath: elasticsearch.sh
readOnly: true
- name: elasticsearch-config
mountPath: /usr/share/elasticsearch/config
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
subPath: elasticsearch.yml
@ -205,14 +217,16 @@ spec:
mountPath: /usr/share/elasticsearch/config/log4j2.properties
subPath: log4j2.properties
readOnly: true
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/jvm.options
subPath: jvm.options
readOnly: true
- name: storage
mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: pod-tmp
emptyDir: {}
- name: elasticsearch-config
emptyDir: {}
- name: elasticsearch-logs
emptyDir: {}
- name: elasticsearch-bin

44
elasticsearch/templates/statefulset-data.yaml
View File

@ -18,6 +18,7 @@ limitations under the License.
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
@ -70,6 +71,7 @@ metadata:
spec:
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }}
serviceName: {{ tuple "elasticsearch" "data" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
podManagementPolicy: "Parallel"
replicas: {{ .Values.pod.replicas.data }}
selector:
matchLabels:
@ -101,6 +103,19 @@ spec:
- sysctl
- -w
- vm.max_map_count={{ .Values.conf.init.max_map_count }}
- name: elasticsearch-perms
{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
runAsUser: 0
command:
- chown
- -R
- "elasticsearch:"
- {{ .Values.conf.elasticsearch.config.path.data }}
volumeMounts:
- name: storage
mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
containers:
- name: elasticsearch-data
{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
@ -148,8 +163,21 @@ spec:
value: "false"
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
- name: DISCOVERY_SERVICE
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- if .Values.pod.env.data }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
{{- end }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
@ -159,8 +187,6 @@ spec:
mountPath: /tmp/elasticsearch.sh
subPath: elasticsearch.sh
readOnly: true
- name: elasticsearch-config
mountPath: /usr/share/elasticsearch/config
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
subPath: elasticsearch.yml
@ -169,6 +195,10 @@ spec:
mountPath: /usr/share/elasticsearch/config/log4j2.properties
subPath: log4j2.properties
readOnly: true
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/jvm.options
subPath: jvm.options
readOnly: true
- name: storage
mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
@ -177,8 +207,6 @@ spec:
emptyDir: {}
- name: elasticsearch-logs
emptyDir: {}
- name: elasticsearch-config
emptyDir: {}
- name: elasticsearch-bin
configMap:
name: elasticsearch-bin
@ -188,7 +216,7 @@ spec:
secretName: elasticsearch-etc
defaultMode: 0444
{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
{{- if not .Values.storage.enabled }}
{{- if not .Values.storage.data.enabled }}
- name: storage
emptyDir: {}
{{- else }}
@ -196,10 +224,10 @@ spec:
- metadata:
name: storage
spec:
accessModes: {{ .Values.storage.pvc.access_mode }}
accessModes: {{ .Values.storage.data.pvc.access_mode }}
resources:
requests:
storage: {{ .Values.storage.requests.storage }}
storageClassName: {{ .Values.storage.storage_class }}
storage: {{ .Values.storage.data.requests.storage }}
storageClassName: {{ .Values.storage.data.storage_class }}
{{- end }}
{{- end }}

58
elasticsearch/templates/deployment-master.yaml → elasticsearch/templates/statefulset-master.yaml
View File

@ -14,9 +14,11 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.deployment_master }}
{{- if .Values.manifests.statefulset_master }}
{{- $envAll := . }}
{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }}
{{- $serviceAccountName := "elasticsearch-master" }}
@ -58,7 +60,7 @@ rules:
- get
---
apiVersion: apps/v1
kind: Deployment
kind: StatefulSet
metadata:
name: elasticsearch-master
annotations:
@ -66,6 +68,8 @@ metadata:
labels:
{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
serviceName: {{ tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
podManagementPolicy: "Parallel"
replicas: {{ .Values.pod.replicas.master }}
selector:
matchLabels:
@ -98,6 +102,19 @@ spec:
- sysctl
- -w
- vm.max_map_count={{ .Values.conf.init.max_map_count }}
- name: elasticsearch-perms
{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
runAsUser: 0
command:
- chown
- -R
- "elasticsearch:"
- {{ .Values.conf.elasticsearch.config.path.data }}
volumeMounts:
- name: storage
mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
containers:
- name: elasticsearch-master
{{ dict "envAll" $envAll "application" "master" "container" "elasticsearch_master" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
@ -105,7 +122,7 @@ spec:
{{ tuple $envAll $envAll.Values.pod.resources.master | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/elasticsearch.sh
- start
- start_master_node
lifecycle:
preStop:
exec:
@ -141,6 +158,19 @@ spec:
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
{{- if .Values.pod.env.master }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
{{- end }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
@ -150,8 +180,6 @@ spec:
mountPath: /tmp/elasticsearch.sh
subPath: elasticsearch.sh
readOnly: true
- name: elasticsearch-config
mountPath: /usr/share/elasticsearch/config
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
subPath: elasticsearch.yml
@ -160,6 +188,10 @@ spec:
mountPath: /usr/share/elasticsearch/config/log4j2.properties
subPath: log4j2.properties
readOnly: true
- name: elasticsearch-etc
mountPath: /usr/share/elasticsearch/config/jvm.options
subPath: jvm.options
readOnly: true
- name: storage
mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
@ -168,8 +200,6 @@ spec:
emptyDir: {}
- name: elasticsearch-logs
emptyDir: {}
- name: elasticsearch-config
emptyDir: {}
- name: elasticsearch-bin
configMap:
name: elasticsearch-bin
@ -178,7 +208,19 @@ spec:
secret:
secretName: elasticsearch-etc
defaultMode: 0444
{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
{{- if not .Values.storage.master.enabled }}
- name: storage
emptyDir: {}
{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
{{- else }}
volumeClaimTemplates:
- metadata:
name: storage
spec:
accessModes: {{ .Values.storage.master.pvc.access_mode }}
resources:
requests:
storage: {{ .Values.storage.master.requests.storage }}
storageClassName: {{ .Values.storage.master.storage_class }}
{{- end }}
{{- end }}

158
elasticsearch/values.yaml
View File

@ -20,8 +20,8 @@ images:
tags:
apache_proxy: docker.io/httpd:2.4
memory_init: docker.io/openstackhelm/heat:newton-ubuntu_xenial
elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191119
curator: docker.io/bobrik/curator:5.6.0
elasticsearch: docker.io/openstackhelm/elasticsearch-s3:5_6_4-20191119
ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119
s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
@ -123,6 +123,10 @@ dependencies:
- elasticsearch-register-snapshot-repository
pod:
env:
client: null
data: null
master: null
mandatory_access_control:
type: apparmor
elasticsearch-master:
@ -154,7 +158,7 @@ pod:
add:
- IPC_LOCK
- SYS_RESOURCE
readOnlyRootFilesystem: true
readOnlyRootFilesystem: false
master:
pod:
runAsUser: 0
@ -168,7 +172,7 @@ pod:
add:
- IPC_LOCK
- SYS_RESOURCE
readOnlyRootFilesystem: true
readOnlyRootFilesystem: false
es_cluster_wait:
pod:
runAsUser: 0
@ -418,6 +422,7 @@ conf:
LoadModule unixd_module modules/mod_unixd.so
LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so
LoadModule rewrite_module modules/mod_rewrite.so
<IfModule unixd_module>
User daemon
@ -480,10 +485,6 @@ conf:
Require valid-user
</Location>
# Restrict access to the Elasticsearch Update API endpoint to prevent modification of indexed documents
<Location /*/_doc/*/_update*>
Require all denied
</Location>
# Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents
<Location /*/_update_by_query*>
Require all denied
@ -492,30 +493,49 @@ conf:
<Location /*/_delete_by_query*>
Require all denied
</Location>
# Prohibit DELETE methods on the document API endpoint
<Location /*/_doc/*>
AllowMethods GET POST OPTIONS
ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
AuthName "Elasticsearch"
AuthType Basic
AuthBasicProvider file ldap
AuthUserFile /usr/local/apache2/conf/.htpasswd
AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
Require valid-user
</Location>
</VirtualHost>
log4j2: |
status = error
appender.console.type = Console
appender.console.name = console
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n
rootLogger.level = info
rootLogger.appenderRef.console.ref = console
jvm_options: |
-Xms1g
-Xmx1g
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-Des.networkaddress.cache.ttl=60
-Des.networkaddress.cache.negative.ttl=10
-XX:+AlwaysPreTouch
-Xss1m
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-XX:-OmitStackTraceInFastThrow
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
9-:-Djava.locale.providers=COMPAT
10-:-XX:UseAVX=2
init:
max_map_count: 262144
ceph:
@ -636,21 +656,17 @@ conf:
cluster:
name: elasticsearch
discovery:
zen:
ping.unicast.hosts: ${DISCOVERY_SERVICE}
minimum_master_nodes: 2
http:
enabled: ${HTTP_ENABLE}
compression: true
# NOTE(srwilkers): This gets configured dynamically via endpoint lookups
seed_hosts: null
network:
host: 0.0.0.0
cloud:
aws:
protocol: http
s3:
s3:
client:
default:
# NOTE(srwilkers): This gets configured dynamically via endpoint
# lookups
endpoint: null
protocol: http
node:
ingest: ${NODE_INGEST}
master: ${NODE_MASTER}
@ -658,8 +674,8 @@ conf:
name: ${NODE_NAME}
max_local_storage_nodes: 3
path:
data: /usr/share/elasticsearch/data
logs: /usr/share/elasticsearch/logs
data: /data
logs: /logs
snapshots:
enabled: false
# NOTE(srwilkers): The path for the radosgw s3 endpoint gets populated
@ -680,33 +696,32 @@ conf:
timeout: 20s
templates:
fluent:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
index:
number_of_shards: 1
mappings:
fluent:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
endpoints:
cluster_domain_suffix: cluster.local
@ -829,13 +844,22 @@ network:
port: 30920
storage:
enabled: true
pvc:
name: pvc-elastic
access_mode: [ "ReadWriteOnce" ]
requests:
storage: 5Gi
storage_class: general
data:
enabled: true
pvc:
name: pvc-elastic
access_mode: [ "ReadWriteOnce" ]
requests:
storage: 5Gi
storage_class: general
master:
enabled: true
pvc:
name: pvc-elastic
access_mode: [ "ReadWriteOnce" ]
requests:
storage: 1Gi
storage_class: general
manifests:
@ -847,7 +871,6 @@ manifests:
cron_curator: true
cron_verify_repositories: true
deployment_client: true
deployment_master: true
ingress: true
job_cluster_wait: true
job_elasticsearch_templates: true
@ -870,3 +893,4 @@ manifests:
service_ingress: true
service_logging: true
statefulset_data: true
statefulset_master: true

14
kibana/templates/bin/_create_kibana_index_patterns.sh.tpl
View File

@ -18,9 +18,13 @@ set -ex
{{- range .Values.conf.create_kibana_indexes.indexes }}
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/index-pattern/{{ . }}-*" -H 'Content-Type: application/json' \
-d '{"title":"{{ . }}-*","timeFieldName":"@timestamp","notExpandable":true}'
{{- end }}
-XPOST "${KIBANA_ENDPOINT}/api/saved_objects/index-pattern/{{ . }}*" -H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' -d \
'{"attributes":{"title":"{{ . }}-*","timeFieldName":"@timestamp"}}'
{{ end }}
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/config/5.6.4" -H 'Content-Type: application/json' \
-d '{"defaultIndex" : "{{ .Values.conf.create_kibana_indexes.default_index }}-*"}'
-XPOST "${KIBANA_ENDPOINT}/api/kibana/settings/defaultIndex" -H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' -d \
'{"value" : "{{ .Values.conf.create_kibana_indexes.default_index }}*"}'

10
kibana/templates/bin/_kibana.sh.tpl
View File

@ -15,14 +15,14 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
set -e
COMMAND="${@:-start}"
function start () {
exec kibana \
--elasticsearch.url="$ELASTICSEARCH_URL" \
--elasticsearch.username="$ELASTICSEARCH_USERNAME" \
--elasticsearch.password="$ELASTICSEARCH_PASSWORD"
exec /usr/share/kibana/bin/kibana \
--elasticsearch.hosts="${ELASTICSEARCH_HOSTS}" \
--elasticsearch.username="${ELASTICSEARCH_USERNAME}" \
--elasticsearch.password="${ELASTICSEARCH_PASSWORD}"
}
function stop () {

24
kibana/templates/deployment.yaml
View File

@ -18,6 +18,13 @@ limitations under the License.
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $esUser := .Values.endpoints.elasticsearch.auth.admin.username }}
{{- $esPass := .Values.endpoints.elasticsearch.auth.admin.password }}
{{- $authHeader := printf "%s:%s" $esUser $esPass | b64enc }}
{{- $esScheme := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
{{- $esSvc := tuple "elasticsearch" "default" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $esHosts := printf "%s://%s" $esScheme $esSvc }}
{{- $serviceAccountName := "kibana" }}
{{ tuple $envAll "kibana" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@ -101,9 +108,18 @@ spec:
ports:
- name: kibana
containerPort: {{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
readinessProbe:
httpGet:
path: /status
port: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
httpHeaders:
- name: Authorization
value: Basic {{ $authHeader }}
initialDelaySeconds: 20
periodSeconds: 10
env:
- name: ELASTICSEARCH_URL
value: {{ tuple "elasticsearch" "default" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
- name: ELASTICSEARCH_HOSTS
value: {{ $esHosts }}
- name: ELASTICSEARCH_USERNAME
valueFrom:
secretKeyRef:
@ -124,6 +140,8 @@ spec:
subPath: kibana.sh
readOnly: true
- name: pod-etc-kibana
mountPath: /usr/share/kibana/config
- name: pod-optimize-kibana
mountPath: /usr/share/kibana/optimize
- name: kibana-etc
mountPath: /usr/share/kibana/config/kibana.yml
@ -137,6 +155,8 @@ spec:
medium: "Memory"
- name: pod-etc-kibana
emptyDir: {}
- name: pod-optimize-kibana
emptyDir: {}
- name: kibana-bin
configMap:
name: kibana-bin

2
kibana/templates/job-register-kibana-indexes.yaml
View File

@ -53,6 +53,8 @@ spec:
secretKeyRef:
name: {{ $esUserSecret }}
key: ELASTICSEARCH_PASSWORD
- name: KIBANA_ENDPOINT
value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
- name: ELASTICSEARCH_ENDPOINT
value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
command:

14
kibana/values.yaml
View File

@ -23,7 +23,7 @@ labels:
images:
tags:
apache_proxy: docker.io/httpd:2.4
kibana: docker.io/kibana:5.6.4
kibana: docker.elastic.co/kibana/kibana-oss:7.1.0
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
image_repo_sync: docker.io/docker:17.07.0
register_kibana_indexes: docker.io/openstackhelm/heat:newton-ubuntu_xenial
@ -38,17 +38,19 @@ pod:
security_context:
dashboard:
pod:
runAsUser: 999
runAsUser: 1000
container:
apache_proxy:
runAsUser: 0
readOnlyRootFilesystem: false
kibana:
fsGroup: 1000
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
readOnlyRootFilesystem: false
register_kibana_indexes:
pod:
runAsUser: 999
runAsUser: 1000
container:
register_kibana_indexes:
allowPrivilegeEscalation: false
@ -250,7 +252,6 @@ conf:
startupTimeout: 5000
kibana:
defaultAppId: discover
index: .kibana
logging:
quiet: false
silent: false
@ -258,13 +259,16 @@ conf:
ops:
interval: 5000
server:
rewriteBasePath: false
host: localhost
name: kibana
maxPayloadBytes: 1048576
port: 5601
ssl:
enabled: false
create_kibana_indexes:
enabled: true
version: 7.1.0
indexes:
- logstash
- openstack

111
tools/deployment/armada/manifests/armada-lma.yaml
View File

@ -316,7 +316,7 @@ data:
secret_key: ${RADOSGW_S3_ELASTICSEARCH_SECRET_KEY}
pod:
replicas:
data: 1
data: 2
master: 2
labels:
elasticsearch:
@ -341,74 +341,71 @@ data:
settings:
number_of_shards: 1
mappings:
syslog:
properties:
cluster:
type: keyword
app:
type: keyword
pid:
type: integer
host:
type: keyword
log:
type: text
properties:
cluster:
type: keyword
app:
type: keyword
pid:
type: integer
host:
type: keyword
log:
type: text
oslo_openstack_fluentd:
template: "openstack-*"
index_patterns: "openstack-*"
settings:
number_of_shards: 1
mappings:
oslo_openstack_fluentd:
properties:
extra:
properties:
project:
type: text
norms: false
version:
type: text
norms: false
filename:
type: text
norms: false
funcname:
type: text
norms: false
message:
type: text
norms: false
process_name:
type: keyword
index: false
properties:
extra:
properties:
project:
type: text
norms: false
version:
type: text
norms: false
filename:
type: text
norms: false
funcname:
type: text
norms: false
message:
type: text
norms: false
process_name:
type: keyword
index: false
docker_fluentd:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
docker_fluentd:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
curator:
action_file:
actions:

2
tools/deployment/multinode/120-elasticsearch.sh
View File

@ -26,7 +26,7 @@ jobs:
cron: "*/3 * * * *"
pod:
replicas:
data: 1
data: 2
master: 2
conf:
elasticsearch:

2
tools/deployment/osh-infra-logging/050-elasticsearch.sh
View File

@ -29,7 +29,7 @@ monitoring:
enabled: true
pod:
replicas:
data: 1
data: 2
master: 2
conf:
elasticsearch: