Fluentd: Update configuration defaults and overrides

This updates the default fluentd configuration to include
recommended settings for preventing the elasticsearch plugin from
reloading the connection after 10000 requests (default for the
ruby gem). This also updates the configuration overrides for the
fluentd-daemonset deployment to provide input parity with the
default fluentbit configuration by adding inputs for the docker
and kubelet systemd units, inputs for ceph, libvirt, kernel logs,
and auth logs on the host.  Finally, this updates the fluentd
template to include environment variables for the host name and
the fluentd pod name so they can be added to logged events through
fluentd filter plugins

Change-Id: I21f7a89a325c44f8b058ff01a20191bea1a210b4
Signed-off-by: Steve Wilkerson <sw5822@att.com>

fluentd/templates/deployment-fluentd.yaml

@@ -40,7 +40,6 @@ kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: {{ $rcControllerName | quote }}
-rules:
 rules:
   - apiGroups:
       - ""
@@ -140,6 +139,14 @@ spec:
             initialDelaySeconds: 60
             timeoutSeconds: 10
           env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: FLUENTD_PORT
               value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_HOST

fluentd/values.yaml

@@ -192,6 +192,9 @@ conf:
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix libvirt
@@ -211,6 +214,9 @@ conf:
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix qemu
@@ -230,6 +236,9 @@ conf:
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix journal
@@ -249,6 +258,9 @@ conf:
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix kernel
@@ -267,8 +279,10 @@ conf:
           retry_forever false
           retry_max_interval 30
         </buffer>
-        flush_interval 15s
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         password "#{ENV['ELASTICSEARCH_PASSWORD']}"

tools/deployment/common/fluentd-daemonset.sh

@@ -47,6 +47,12 @@ conf:
         @type monitor_agent
       </source>
 
+      <source>
+        bind 0.0.0.0
+        port "#{ENV['FLUENTD_PORT']}"
+        @type forward
+      </source>
+
       <source>
         <parse>
           time_format %Y-%m-%dT%H:%M:%S.%NZ
@@ -58,30 +64,125 @@ conf:
         @type tail
       </source>
 
+      <source>
+        @type tail
+        tag ceph.*
+        path /var/log/ceph/*/*.log
+        read_from_head true
+        <parse>
+          @type none
+        </parse>
+      </source>
+
+      <source>
+        @type tail
+        tag libvirt.*
+        path /var/log/libvirt/**.log
+        read_from_head true
+        <parse>
+          @type none
+        </parse>
+      </source>
+
+      <source>
+        @type tail
+        tag kernel
+        path /var/log/kern.log
+        read_from_head true
+        <parse>
+          @type none
+        </parse>
+      </source>
+
+      <source>
+        @type tail
+        tag auth
+        path /var/log/auth.log
+        read_from_head true
+        <parse>
+          @type none
+        </parse>
+      </source>
+
+      <source>
+        @type systemd
+        tag journal.*
+        path /var/log/journal
+        matches [{ "_SYSTEMD_UNIT": "docker.service" }]
+        read_from_head true
+
+        <entry>
+          fields_strip_underscores true
+          fields_lowercase true
+        </entry>
+      </source>
+
+      <source>
+        @type systemd
+        tag journal.*
+        path /var/log/journal
+        matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+        read_from_head true
+
+        <entry>
+          fields_strip_underscores true
+          fields_lowercase true
+        </entry>
+      </source>
+
       <filter kubernetes.**>
         @type kubernetes_metadata
       </filter>
 
-      <source>
+      <filter ceph.**>
-        bind 0.0.0.0
+        @type record_transformer
-        port "#{ENV['FLUENTD_PORT']}"
+        <record>
-        @type forward
+          hostname "#{ENV['NODE_NAME']}"
-      </source>
+          fluentd_pod "#{ENV['POD_NAME']}"
+        </record>
+      </filter>
+
+      <filter libvirt.**>
+        @type record_transformer
+        <record>
+          hostname "#{ENV['NODE_NAME']}"
+          fluentd_pod "#{ENV['POD_NAME']}"
+        </record>
+      </filter>
+
+      <filter kernel>
+        @type record_transformer
+        <record>
+          hostname "#{ENV['NODE_NAME']}"
+          fluentd_pod "#{ENV['POD_NAME']}"
+        </record>
+      </filter>
+
+      <filter auth>
+        @type record_transformer
+        <record>
+          hostname "#{ENV['NODE_NAME']}"
+          fluentd_pod "#{ENV['POD_NAME']}"
+        </record>
+      </filter>
 
       <match fluent.**>
         @type null
       </match>
 
-      <match libvirt>
+      <match libvirt.**>
         <buffer>
-          chunk_limit_size 500K
+          chunk_limit_size 512K
           flush_interval 5s
           flush_thread_count 8
-          queue_limit_length 16
+          queue_limit_length 32
           retry_forever false
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix libvirt
@@ -91,38 +192,22 @@ conf:
         user "#{ENV['ELASTICSEARCH_USERNAME']}"
       </match>
 
-      <match qemu>
+      <match ceph.**>
         <buffer>
-          chunk_limit_size 500K
+          chunk_limit_size 512K
           flush_interval 5s
           flush_thread_count 8
-          queue_limit_length 16
+          queue_limit_length 32
           retry_forever false
           retry_max_interval 30
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
-        logstash_prefix qemu
+        logstash_prefix ceph
-        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
-        port "#{ENV['ELASTICSEARCH_PORT']}"
-        @type elasticsearch
-        user "#{ENV['ELASTICSEARCH_USERNAME']}"
-      </match>
-
-      <match journal.**>
-        <buffer>
-          chunk_limit_size 500K
-          flush_interval 5s
-          flush_thread_count 8
-          queue_limit_length 16
-          retry_forever false
-          retry_max_interval 30
-        </buffer>
-        host "#{ENV['ELASTICSEARCH_HOST']}"
-        include_tag_key true
-        logstash_format true
-        logstash_prefix journal
         password "#{ENV['ELASTICSEARCH_PASSWORD']}"
         port "#{ENV['ELASTICSEARCH_PORT']}"
         @type elasticsearch
@@ -131,14 +216,17 @@ conf:
 
       <match kernel>
         <buffer>
-          chunk_limit_size 500K
+          chunk_limit_size 512K
           flush_interval 5s
           flush_thread_count 8
-          queue_limit_length 16
+          queue_limit_length 32
           retry_forever false
-          retry_max_interval 30
+          disable_chunk_backup true
         </buffer>
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         logstash_prefix kernel
@@ -148,23 +236,68 @@ conf:
         user "#{ENV['ELASTICSEARCH_USERNAME']}"
       </match>
 
-      <match **>
+      <match auth>
         <buffer>
-          chunk_limit_size 500K
+          chunk_limit_size 512K
           flush_interval 5s
           flush_thread_count 8
-          queue_limit_length 16
+          queue_limit_length 32
           retry_forever false
           retry_max_interval 30
         </buffer>
-        flush_interval 15s
         host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
+        include_tag_key true
+        logstash_format true
+        logstash_prefix auth
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match journal.**>
+        <buffer>
+          chunk_limit_size 512K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 32
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
+        include_tag_key true
+        logstash_format true
+        logstash_prefix journal
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match **>
+        <buffer>
+          chunk_limit_size 512K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 32
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
         include_tag_key true
         logstash_format true
         password "#{ENV['ELASTICSEARCH_PASSWORD']}"
         port "#{ENV['ELASTICSEARCH_PORT']}"
         @type elasticsearch
-        type_name fluent
         user "#{ENV['ELASTICSEARCH_USERNAME']}"
       </match>
 EOF