From aee9708b06f5d6dd47fc53fdc000ca56097517d7 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 31 May 2019 15:05:54 -0500 Subject: [PATCH] Fluentd: Update configuration defaults and overrides This updates the default fluentd configuration to include recommended settings for preventing the elasticsearch plugin from reloading the connection after 10000 requests (default for the ruby gem). This also updates the configuration overrides for the fluentd-daemonset deployment to provide input parity with the default fluentbit configuration by adding inputs for the docker and kubelet systemd units, inputs for ceph, libvirt, kernel logs, and auth logs on the host. Finally, this updates the fluentd template to include environment variables for the host name and the fluentd pod name so they can be added to logged events through fluentd filter plugins Change-Id: I21f7a89a325c44f8b058ff01a20191bea1a210b4 Signed-off-by: Steve Wilkerson --- fluentd/templates/deployment-fluentd.yaml | 9 +- fluentd/values.yaml | 16 +- tools/deployment/common/fluentd-daemonset.sh | 211 +++++++++++++++---- 3 files changed, 195 insertions(+), 41 deletions(-) diff --git a/fluentd/templates/deployment-fluentd.yaml b/fluentd/templates/deployment-fluentd.yaml index 71d638e9a..b6ae52823 100644 --- a/fluentd/templates/deployment-fluentd.yaml +++ b/fluentd/templates/deployment-fluentd.yaml @@ -40,7 +40,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ $rcControllerName | quote }} -rules: rules: - apiGroups: - "" @@ -140,6 +139,14 @@ spec: initialDelaySeconds: 60 timeoutSeconds: 10 env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: FLUENTD_PORT value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_HOST diff --git a/fluentd/values.yaml b/fluentd/values.yaml index 536f95cda..12513fd79 100644 --- a/fluentd/values.yaml +++ b/fluentd/values.yaml @@ -192,6 +192,9 @@ conf: retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix libvirt @@ -211,6 +214,9 @@ conf: retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix qemu @@ -230,6 +236,9 @@ conf: retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix journal @@ -249,6 +258,9 @@ conf: retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix kernel @@ -267,8 +279,10 @@ conf: retry_forever false retry_max_interval 30 - flush_interval 15s host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true password "#{ENV['ELASTICSEARCH_PASSWORD']}" diff --git a/tools/deployment/common/fluentd-daemonset.sh b/tools/deployment/common/fluentd-daemonset.sh index 6659e81e0..102bb8bbc 100755 --- a/tools/deployment/common/fluentd-daemonset.sh +++ b/tools/deployment/common/fluentd-daemonset.sh @@ -47,6 +47,12 @@ conf: @type monitor_agent + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + + time_format %Y-%m-%dT%H:%M:%S.%NZ @@ -58,30 +64,125 @@ conf: @type tail + + @type tail + tag ceph.* + path /var/log/ceph/*/*.log + read_from_head true + + @type none + + + + + @type tail + tag libvirt.* + path /var/log/libvirt/**.log + read_from_head true + + @type none + + + + + @type tail + tag kernel + path /var/log/kern.log + read_from_head true + + @type none + + + + + @type tail + tag auth + path /var/log/auth.log + read_from_head true + + @type none + + + + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + read_from_head true + + + fields_strip_underscores true + fields_lowercase true + + + + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + read_from_head true + + + fields_strip_underscores true + fields_lowercase true + + + @type kubernetes_metadata - - bind 0.0.0.0 - port "#{ENV['FLUENTD_PORT']}" - @type forward - + + @type record_transformer + + hostname "#{ENV['NODE_NAME']}" + fluentd_pod "#{ENV['POD_NAME']}" + + + + + @type record_transformer + + hostname "#{ENV['NODE_NAME']}" + fluentd_pod "#{ENV['POD_NAME']}" + + + + + @type record_transformer + + hostname "#{ENV['NODE_NAME']}" + fluentd_pod "#{ENV['POD_NAME']}" + + + + + @type record_transformer + + hostname "#{ENV['NODE_NAME']}" + fluentd_pod "#{ENV['POD_NAME']}" + + @type null - + - chunk_limit_size 500K + chunk_limit_size 512K flush_interval 5s flush_thread_count 8 - queue_limit_length 16 + queue_limit_length 32 retry_forever false retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix libvirt @@ -91,38 +192,22 @@ conf: user "#{ENV['ELASTICSEARCH_USERNAME']}" - + - chunk_limit_size 500K + chunk_limit_size 512K flush_interval 5s flush_thread_count 8 - queue_limit_length 16 + queue_limit_length 32 retry_forever false retry_max_interval 30 host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true - logstash_prefix qemu - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - logstash_prefix journal + logstash_prefix ceph password "#{ENV['ELASTICSEARCH_PASSWORD']}" port "#{ENV['ELASTICSEARCH_PORT']}" @type elasticsearch @@ -131,14 +216,17 @@ conf: - chunk_limit_size 500K + chunk_limit_size 512K flush_interval 5s flush_thread_count 8 - queue_limit_length 16 + queue_limit_length 32 retry_forever false - retry_max_interval 30 + disable_chunk_backup true host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true logstash_prefix kernel @@ -148,23 +236,68 @@ conf: user "#{ENV['ELASTICSEARCH_USERNAME']}" - + - chunk_limit_size 500K + chunk_limit_size 512K flush_interval 5s flush_thread_count 8 - queue_limit_length 16 + queue_limit_length 32 retry_forever false retry_max_interval 30 - flush_interval 15s host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true + include_tag_key true + logstash_format true + logstash_prefix auth + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 512K + flush_interval 5s + flush_thread_count 8 + queue_limit_length 32 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true + include_tag_key true + logstash_format true + logstash_prefix journal + password "#{ENV['ELASTICSEARCH_PASSWORD']}" + port "#{ENV['ELASTICSEARCH_PORT']}" + @type elasticsearch + user "#{ENV['ELASTICSEARCH_USERNAME']}" + + + + + chunk_limit_size 512K + flush_interval 5s + flush_thread_count 8 + queue_limit_length 32 + retry_forever false + retry_max_interval 30 + + host "#{ENV['ELASTICSEARCH_HOST']}" + reload_connections false + reconnect_on_error true + reload_on_failure true include_tag_key true logstash_format true password "#{ENV['ELASTICSEARCH_PASSWORD']}" port "#{ENV['ELASTICSEARCH_PORT']}" @type elasticsearch - type_name fluent user "#{ENV['ELASTICSEARCH_USERNAME']}" EOF