Add template for Fluent logging index
Fluent-logging stores logs on a elasticsearch by default. Elasticsearch stores all fields as tokens by default, but some fields shoud be stored for purposes such as retrieval, without splitting. Mapping in elasticsearch is used to define a property of fields and Template can defines the mapping for an index. fluent-logging use it to define the index structure. Specific index type can be defined on the value file. Change-Id: Id597111f478fcddf709b36d2db9ac5a5d6d8206f
This commit is contained in:
parent
3713266f4c
commit
66919d28ef
@ -108,3 +108,34 @@ section):
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
# This function generates elasticsearch template files with entries in the
|
||||
# fluent-logging values.yaml. It results in a configuration section with the
|
||||
# following format (for as many key/value pairs defined in values for a section):
|
||||
# {
|
||||
# key: value
|
||||
# key: {
|
||||
# key: { ... }
|
||||
# }
|
||||
# }
|
||||
# The configuration schema can be found here:
|
||||
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
|
||||
|
||||
{{- define "fluent_logging.to_elasticsearch_template" -}}
|
||||
{
|
||||
{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
{{- define "fluent_logging.recursive_tuple" -}}
|
||||
{{- range $key, $value := . -}}
|
||||
,
|
||||
{{- if or (kindIs "map" $value) }}
|
||||
{{ $key | quote -}}:{
|
||||
{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
|
||||
}
|
||||
{{- else }}
|
||||
{{ $key | quote -}}:{{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
13
fluent-logging/templates/bin/_create_template.sh.tpl
Normal file
13
fluent-logging/templates/bin/_create_template.sh.tpl
Normal file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml
|
||||
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/template.xml)
|
||||
if [ "$result" == "True" ]; then
|
||||
echo "template created!"
|
||||
else
|
||||
echo "template not created!"
|
||||
fi
|
@ -46,7 +46,22 @@ function check_kubernetes_tag () {
|
||||
fi
|
||||
}
|
||||
|
||||
# Tests whether fluent-logging has successfully generate template_fluent_logging template
|
||||
# defined by value.yaml
|
||||
function check_template () {
|
||||
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \
|
||||
| python -c "import sys, json; print len(json.load(sys.stdin))")
|
||||
if [ "$total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on template, provided by fluent-logging!"
|
||||
else
|
||||
echo "FAIL: No hits on query for template_fluent_logging template! Exiting";
|
||||
exit 1;
|
||||
fi
|
||||
}
|
||||
|
||||
# Sleep for at least the buffer flush time to allow for indices to be populated
|
||||
sleep 30
|
||||
check_template
|
||||
check_logstash_index
|
||||
check_kubernetes_tag
|
||||
|
@ -28,6 +28,8 @@ data:
|
||||
{{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
helm-tests.sh: |
|
||||
{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
create_template.sh: |
|
||||
{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
image-repo-sync.sh: |+
|
||||
{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
|
||||
{{- end }}
|
||||
|
@ -28,4 +28,6 @@ data:
|
||||
{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
|
||||
td-agent.conf: |+
|
||||
{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
|
||||
template.xml.raw: |+
|
||||
{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }}
|
||||
{{- end }}
|
||||
|
74
fluent-logging/templates/job-elasticsearch-template.yaml
Normal file
74
fluent-logging/templates/job-elasticsearch-template.yaml
Normal file
@ -0,0 +1,74 @@
|
||||
{{/*
|
||||
Copyright 2017 The Openstack-Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/}}
|
||||
|
||||
{{- if .Values.manifests.job_elasticsearch_template }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.elasticsearch_template }}
|
||||
{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }}
|
||||
{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: elasticsearch-template
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{ tuple $envAll "fluent" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector:
|
||||
{{ .Values.labels.elasticsearch_template.node_selector_key }}: {{ .Values.labels.elasticsearch_template.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_elasticsearch_template_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
containers:
|
||||
- name: elasticsearch-template
|
||||
image: {{ .Values.images.tags.elasticsearch_template }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
||||
env:
|
||||
- name: ELASTICSEARCH_HOST
|
||||
value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
|
||||
- name: ELASTICSEARCH_PORT
|
||||
value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
|
||||
- name: ELASTICSEARCH_USERNAME
|
||||
value: {{ .Values.endpoints.elasticsearch.auth.admin.username }}
|
||||
- name: ELASTICSEARCH_PASSWORD
|
||||
value: {{ .Values.endpoints.elasticsearch.auth.admin.password }}
|
||||
command:
|
||||
- /tmp/create_template.sh
|
||||
volumeMounts:
|
||||
- name: fluent-logging-bin
|
||||
mountPath: /tmp/create_template.sh
|
||||
subPath: create_template.sh
|
||||
readOnly: true
|
||||
- name: fluent-logging-etc
|
||||
mountPath: /tmp/template.xml.raw
|
||||
subPath: template.xml.raw
|
||||
readOnly: true
|
||||
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 10 }}{{ end }}
|
||||
volumes:
|
||||
- name: fluent-logging-bin
|
||||
configMap:
|
||||
name: fluent-logging-bin
|
||||
defaultMode: 0555
|
||||
- name: fluent-logging-etc
|
||||
configMap:
|
||||
name: fluent-logging-etc
|
||||
defaultMode: 0666
|
||||
{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 6 }}{{ end }}
|
||||
{{- end }}
|
@ -28,6 +28,9 @@ labels:
|
||||
prometheus_fluentd_exporter:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
elasticsearch_template:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
|
||||
images:
|
||||
tags:
|
||||
@ -36,6 +39,7 @@ images:
|
||||
prometheus_fluentd_exporter: docker.io/srwilkers/fluentd_exporter:v0.1
|
||||
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
|
||||
helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
|
||||
elasticsearch_template: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
|
||||
image_repo_sync: docker.io/docker:17.07.0
|
||||
pull_policy: IfNotPresent
|
||||
local_registry:
|
||||
@ -64,9 +68,15 @@ dependencies:
|
||||
- service: kafka
|
||||
endpoint: public
|
||||
fluentbit:
|
||||
jobs:
|
||||
- elasticsearch-template
|
||||
services:
|
||||
- service: fluentd
|
||||
endpoint: internal
|
||||
elasticsearch-template:
|
||||
services:
|
||||
- service: elasticsearch
|
||||
endpoint: internal
|
||||
tests:
|
||||
services:
|
||||
- service: elasticsearch
|
||||
@ -156,6 +166,50 @@ conf:
|
||||
log:
|
||||
format: "logger:stdout?json=true"
|
||||
level: "info"
|
||||
template:
|
||||
template: "logstash-*"
|
||||
settings:
|
||||
number_of_shards: 5
|
||||
number_of_replicas: 1
|
||||
mappings:
|
||||
flb_type:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
host:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
labels:
|
||||
properties:
|
||||
app:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
application:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
component:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
release_group:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: not_analyzed
|
||||
log:
|
||||
type: text
|
||||
|
||||
endpoints:
|
||||
cluster_domain_suffix: cluster.local
|
||||
@ -319,6 +373,9 @@ pod:
|
||||
fluentbit:
|
||||
fluent_tests:
|
||||
fluent_tests:
|
||||
elasticsearch_template:
|
||||
init_container:
|
||||
elasticsearch_template:
|
||||
|
||||
manifests:
|
||||
configmap_bin: true
|
||||
@ -334,3 +391,4 @@ manifests:
|
||||
service_exporter: true
|
||||
secret_elasticsearch: true
|
||||
service_fluentd: true
|
||||
job_elasticsearch_template: true
|
||||
|
Loading…
x
Reference in New Issue
Block a user