From d89a89c26bd0348e48049f9cddfeb5219dce9282 Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Mon, 17 Jun 2019 15:59:57 +0000
Subject: [PATCH] Format internal Fluentd logs
This change formats internal Fluent logs in a similar way to other
logs. It makes it easier for a user to identify issues with Fluent
parsing logs. Any failure to parse a log will be ingested into the
logging framework and can easily be located by searching for
'pattern not match' or by filtering for Fluent log warnings.
Change-Id: Iea6d12c07a2f4152f2038d3de2ef589479b3332b
---
.../conf/filter/00-record_transformer.conf.j2 | 21 +++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
index 7f173dedfa..911320798f 100644
--- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
@@ -36,6 +36,27 @@
</record>
</filter>
+# Rename internal Fluent message field to match other logs. This removes
+# all other fields by default, including the original message field. This is
+# intented to avoid duplication of the log message and to prevent passing
+# invalid dimensions to Monasca, if it is enabled. Note that if this step
+# is moved to the format folder, then it will applied after the second step
+# below which will break the logic.
+<filter fluent.**>
+ @type parser
+ key_name message
+ format /^(?<Payload>.*)$/
+</filter>
+
+<filter fluent.**>
+ @type record_transformer
+ <record>
+ Hostname "#{Socket.gethostname}"
+ programname ${tag_parts[0]}
+ log_level ${tag_parts[1]}
+ </record>
+</filter>
+
{% if enable_monasca | bool %}
# Kolla configures Fluentd to extract timestamps from OpenStack service
# logs, however these timestamps are not saved in the event and are not