openstack/openstack-helm-infra
Code Issues Proposed changes

Merge "Fluent-logging: Support creation of arbitrary number of templates"

Browse Source
This commit is contained in:
Zuul 2018-06-05 15:29:38 +00:00 committed by Gerrit Code Review
commit b88e34ada6
6 changed files with 72 additions and 97 deletions

35
fluent-logging/templates/_helpers.tpl

@ -108,38 +108,3 @@ section):
{{- end -}}
{{- end -}}
{{- end -}}
# This function generates elasticsearch template files with entries in the
# fluent-logging values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# {
# key: value
# key: {
# key: { ... }
# }
# }
# The configuration schema can be found here:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
{{- define "fluent_logging.to_elasticsearch_template" -}}
{
{{- include "fluent_logging.recursive_tuple" . | indent 2 }}
}
{{- end }}
{{- define "fluent_logging.recursive_tuple" -}}
{{- range $key, $value := . -}}
,
{{- if or (kindIs "map" $value) }}
{{ $key | quote -}}:{
{{- include "fluent_logging.recursive_tuple" $value | indent 2 }}
}
{{- else }}
{{- if eq $key "index_patterns"}}
{{ $key | quote -}}: [{{ $value | quote }}]
{{- else }}
{{ $key | quote -}}:{{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

13
fluent-logging/templates/bin/_create_template.sh.tpl

@ -2,13 +2,16 @@
set -ex
sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml
{{ range $template, $fields := .Values.conf.templates }}
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \
-H 'Content-Type: application/json' -d @/tmp/template.xml \
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
-H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
| python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
if [ "$result" == "True" ]; then
echo "template created!"
echo "{{$template}} template created!"
else
echo "template not created!"
echo "{{$template}} template not created!"
fi
{{ end }}

20
fluent-logging/templates/bin/_helm-tests.sh.tpl

@ -46,22 +46,24 @@ function check_kubernetes_tag () {
fi
}
# Tests whether fluent-logging has successfully generate template_fluent_logging template
# defined by value.yaml
function check_template () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \
# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
# templates defined by values.yaml
function check_templates () {
{{ range $template, $fields := .Values.conf.templates }}
{{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
| python -c "import sys, json; print len(json.load(sys.stdin))")
if [ "$total_hits" -gt 0 ]; then
echo "PASS: Successful hits on template, provided by fluent-logging!"
if [ "${{$template}}_total_hits" -gt 0 ]; then
echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
else
echo "FAIL: No hits on query for template_fluent_logging template! Exiting";
echo "FAIL: No hits on query for {{$template}} template! Exiting";
exit 1;
fi
{{ end }}
}
# Sleep for at least the buffer flush time to allow for indices to be populated
sleep 30
check_template
check_templates
check_logstash_index
check_kubernetes_tag

6
fluent-logging/templates/configmap-etc.yaml

@ -28,6 +28,8 @@ data:
{{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
td-agent.conf: |
{{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }}
template.xml.raw: |
{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }}
{{ range $template, $fields := .Values.conf.templates }}
{{ $template }}.json: |
{{ toJson $fields | indent 4 }}
{{ end }}
{{- end }}

6
fluent-logging/templates/job-elasticsearch-template.yaml

@ -65,10 +65,12 @@ spec:
mountPath: /tmp/create_template.sh
subPath: create_template.sh
readOnly: true
{{ range $template, $fields := .Values.conf.templates }}
- name: fluent-logging-etc
mountPath: /tmp/template.xml.raw
subPath: template.xml.raw
mountPath: /tmp/{{$template}}.json
subPath: {{$template}}.json
readOnly: true
{{ end }}
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: fluent-logging-bin

89
fluent-logging/values.yaml

@ -166,50 +166,51 @@ conf:
log:
format: "logger:stdout?json=true"
level: "info"
template:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
fluent:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
labels:
properties:
app:
type: keyword
index: false
application:
type: keyword
index: false
component:
type: keyword
index: false
release_group:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
log:
type: text
templates:
fluent:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
fluent:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
labels:
properties:
app:
type: keyword
index: false
application:
type: keyword
index: false
component:
type: keyword
index: false
release_group:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
log:
type: text
endpoints:
cluster_domain_suffix: cluster.local