openstack/openstack-helm-infra
Code Issues Proposed changes

Fluentd: Update configuration defaults and overrides

Browse Source 
This updates the default fluentd configuration to include
recommended settings for preventing the elasticsearch plugin from
reloading the connection after 10000 requests (default for the
ruby gem). This also updates the configuration overrides for the
fluentd-daemonset deployment to provide input parity with the
default fluentbit configuration by adding inputs for the docker
and kubelet systemd units, inputs for ceph, libvirt, kernel logs,
and auth logs on the host.  Finally, this updates the fluentd
template to include environment variables for the host name and
the fluentd pod name so they can be added to logged events through
fluentd filter plugins

Change-Id: I21f7a89a325c44f8b058ff01a20191bea1a210b4
Signed-off-by: Steve Wilkerson <sw5822@att.com>
This commit is contained in:
Steve Wilkerson 2019-05-31 15:05:54 -05:00 committed by Steve Wilkerson
parent 2f0c2e328d
commit aee9708b06
3 changed files with 195 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
fluentd/templates/deployment-fluentd.yaml
View File

@ -40,7 +40,6 @@ kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ $rcControllerName | quote }}
rules:
rules:
- apiGroups:
- ""
@ -140,6 +139,14 @@ spec:
initialDelaySeconds: 60
timeoutSeconds: 10
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: FLUENTD_PORT
value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: ELASTICSEARCH_HOST

16
fluentd/values.yaml
View File

@ -192,6 +192,9 @@ conf:
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
@ -211,6 +214,9 @@ conf:
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix qemu
@ -230,6 +236,9 @@ conf:
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
@ -249,6 +258,9 @@ conf:
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
@ -267,8 +279,10 @@ conf:
retry_forever false
retry_max_interval 30
</buffer>
flush_interval 15s
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"

211
tools/deployment/common/fluentd-daemonset.sh
View File

@ -47,6 +47,12 @@ conf:
@type monitor_agent
</source>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<source>
<parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ
@ -58,30 +64,125 @@ conf:
@type tail
</source>
<source>
@type tail
tag ceph.*
path /var/log/ceph/*/*.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type tail
tag libvirt.*
path /var/log/libvirt/**.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type tail
tag kernel
path /var/log/kern.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type tail
tag auth
path /var/log/auth.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "docker.service" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<filter kubernetes.**>
@type kubernetes_metadata
</filter>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<filter ceph.**>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<filter libvirt.**>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<filter kernel>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<filter auth>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<match fluent.**>
@type null
</match>
<match libvirt>
<match libvirt.**>
<buffer>
chunk_limit_size 500K
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 16
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
@ -91,38 +192,22 @@ conf:
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match qemu>
<match ceph.**>
<buffer>
chunk_limit_size 500K
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 16
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix qemu
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 500K
flush_interval 5s
flush_thread_count 8
queue_limit_length 16
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
include_tag_key true
logstash_format true
logstash_prefix journal
logstash_prefix ceph
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
@ -131,14 +216,17 @@ conf:
<match kernel>
<buffer>
chunk_limit_size 500K
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 16
queue_limit_length 32
retry_forever false
retry_max_interval 30
disable_chunk_backup true
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
@ -148,23 +236,68 @@ conf:
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<match auth>
<buffer>
chunk_limit_size 500K
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 16
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
flush_interval 15s
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix auth
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
type_name fluent
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
EOF