Add template for Fluent logging index

Fluent-logging stores logs on a elasticsearch by default. Elasticsearch
stores all fields as tokens by default, but some fields shoud be stored
for purposes such as retrieval, without splitting. Mapping in elasticsearch
is used to define a property of fields and Template can defines the mapping
for an index. fluent-logging use it to define the index structure. Specific
index type can be defined on the value file.

Change-Id: Id597111f478fcddf709b36d2db9ac5a5d6d8206f
This commit is contained in:
sungil 2018-01-25 15:58:29 +09:00
parent 3713266f4c
commit 66919d28ef
7 changed files with 195 additions and 0 deletions

@ -108,3 +108,34 @@ section):
{{- end -}}
{{- end -}}
{{- end -}}
# This function generates elasticsearch template files with entries in the
# fluent-logging values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# {
# key: value
# key: {
# key: { ... }
# }
# }
# The configuration schema can be found here:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
{{- define "fluent_logging.to_elasticsearch_template" -}}
{
{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
}
{{- end }}
{{- define "fluent_logging.recursive_tuple" -}}
{{- range $key, $value := . -}}
,
{{- if or (kindIs "map" $value) }}
{{ $key | quote -}}:{
{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
}
{{- else }}
{{ $key | quote -}}:{{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}

@ -0,0 +1,13 @@
#!/bin/bash
set -ex
sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \
-H 'Content-Type: application/json' -d @/tmp/template.xml)
if [ "$result" == "True" ]; then
echo "template created!"
else
echo "template not created!"
fi

@ -46,7 +46,22 @@ function check_kubernetes_tag () {
fi
}
# Tests whether fluent-logging has successfully generate template_fluent_logging template
# defined by value.yaml
function check_template () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \
| python -c "import sys, json; print len(json.load(sys.stdin))")
if [ "$total_hits" -gt 0 ]; then
echo "PASS: Successful hits on template, provided by fluent-logging!"
else
echo "FAIL: No hits on query for template_fluent_logging template! Exiting";
exit 1;
fi
}
# Sleep for at least the buffer flush time to allow for indices to be populated
sleep 30
check_template
check_logstash_index
check_kubernetes_tag

@ -28,6 +28,8 @@ data:
{{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
helm-tests.sh: |
{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create_template.sh: |
{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
image-repo-sync.sh: |+
{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
{{- end }}

@ -28,4 +28,6 @@ data:
{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
td-agent.conf: |+
{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
template.xml.raw: |+
{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }}
{{- end }}

@ -0,0 +1,74 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_elasticsearch_template }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.elasticsearch_template }}
{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }}
{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: elasticsearch-template
spec:
template:
metadata:
labels:
{{ tuple $envAll "fluent" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.elasticsearch_template.node_selector_key }}: {{ .Values.labels.elasticsearch_template.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies $mounts_elasticsearch_template_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: elasticsearch-template
image: {{ .Values.images.tags.elasticsearch_template }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: ELASTICSEARCH_HOST
value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
- name: ELASTICSEARCH_PORT
value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: ELASTICSEARCH_USERNAME
value: {{ .Values.endpoints.elasticsearch.auth.admin.username }}
- name: ELASTICSEARCH_PASSWORD
value: {{ .Values.endpoints.elasticsearch.auth.admin.password }}
command:
- /tmp/create_template.sh
volumeMounts:
- name: fluent-logging-bin
mountPath: /tmp/create_template.sh
subPath: create_template.sh
readOnly: true
- name: fluent-logging-etc
mountPath: /tmp/template.xml.raw
subPath: template.xml.raw
readOnly: true
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 10 }}{{ end }}
volumes:
- name: fluent-logging-bin
configMap:
name: fluent-logging-bin
defaultMode: 0555
- name: fluent-logging-etc
configMap:
name: fluent-logging-etc
defaultMode: 0666
{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 6 }}{{ end }}
{{- end }}

@ -28,6 +28,9 @@ labels:
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
elasticsearch_template:
node_selector_key: openstack-control-plane
node_selector_value: enabled
images:
tags:
@ -36,6 +39,7 @@ images:
prometheus_fluentd_exporter: docker.io/srwilkers/fluentd_exporter:v0.1
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
elasticsearch_template: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
image_repo_sync: docker.io/docker:17.07.0
pull_policy: IfNotPresent
local_registry:
@ -64,9 +68,15 @@ dependencies:
- service: kafka
endpoint: public
fluentbit:
jobs:
- elasticsearch-template
services:
- service: fluentd
endpoint: internal
elasticsearch-template:
services:
- service: elasticsearch
endpoint: internal
tests:
services:
- service: elasticsearch
@ -156,6 +166,50 @@ conf:
log:
format: "logger:stdout?json=true"
level: "info"
template:
template: "logstash-*"
settings:
number_of_shards: 5
number_of_replicas: 1
mappings:
flb_type:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: not_analyzed
docker_id:
type: keyword
index: not_analyzed
host:
type: keyword
index: not_analyzed
labels:
properties:
app:
type: keyword
index: not_analyzed
application:
type: keyword
index: not_analyzed
component:
type: keyword
index: not_analyzed
release_group:
type: keyword
index: not_analyzed
namespace_name:
type: keyword
index: not_analyzed
pod_id:
type: keyword
index: not_analyzed
pod_name:
type: keyword
index: not_analyzed
log:
type: text
endpoints:
cluster_domain_suffix: cluster.local
@ -319,6 +373,9 @@ pod:
fluentbit:
fluent_tests:
fluent_tests:
elasticsearch_template:
init_container:
elasticsearch_template:
manifests:
configmap_bin: true
@ -334,3 +391,4 @@ manifests:
service_exporter: true
secret_elasticsearch: true
service_fluentd: true
job_elasticsearch_template: true