From 233a187a76c69e4c892a4ad70e710ee67836dfdd Mon Sep 17 00:00:00 2001
From: Isaac Prior <isaac@stackhpc.com>
Date: Thu, 3 Oct 2019 16:29:39 +0100
Subject: [PATCH] Parse MariaDB log messages in different formats into Fluentd
MariaDB logs contain two different log message formats, one output
from mysqld and one from mysqld_safe. This patch splits the message
formats by tag and parses them separately.
Change-Id: I58857be67ae387eeda7487811a6af85b0f95970c
Closes-Bug: #1845629
---
ansible/roles/common/tasks/config.yml | 2 ++
.../conf/filter/01-rewrite-0.12.conf.j2 | 7 +++++
.../conf/filter/01-rewrite-0.14.conf.j2 | 15 +++++++++++
.../templates/conf/filter/02-parser.conf.j2 | 27 +++++++++++++++++++
.../templates/conf/input/02-mariadb.conf.j2 | 7 ++---
5 files changed, 55 insertions(+), 3 deletions(-)
create mode 100644 ansible/roles/common/templates/conf/filter/02-parser.conf.j2
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 2da793ba33..056ecda10f 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -236,6 +236,8 @@
dest: 00-record_transformer
- src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
+ - src: 02-parser
+ dest: 02-parser
when:
- enable_fluentd | bool
- item.src ~ '.conf' not in customised_filter_files
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
index edb201d928..c8c821b3a0 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
@@ -37,3 +37,10 @@
rewriterule34 programname ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$ openstack_python
rewriterule35 programname .+ unmatched
</match>
+
+# Retag log messages from MariaDB according to log format
+<match infra.mariadb>
+ @type rewrite_tag_filter
+ rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe
+ rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld
+</match>
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
index a85735bdbc..8cafb7a721 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
@@ -187,3 +187,18 @@
tag unmatched
</rule>
</match>
+
+# Retag log messages from MariaDB according to log format
+<match infra.mariadb>
+ @type rewrite_tag_filter
+ <rule>
+ key Payload
+ pattern /^\d{6}/
+ tag infra.mariadb.mysqld_safe
+ </rule>
+ <rule>
+ key Payload
+ pattern /^\d{4}-\d{2}-\d{2}/
+ tag infra.mariadb.mysqld
+ </rule>
+</match>
diff --git a/ansible/roles/common/templates/conf/filter/02-parser.conf.j2 b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2
new file mode 100644
index 0000000000..5936a3cf54
--- /dev/null
+++ b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2
@@ -0,0 +1,27 @@
+# Parse MariaDB logs with 6 digit date format (mysqld_safe)
+<filter infra.mariadb.mysqld_safe>
+ @type parser
+ format /^(?<Timestamp>\d{6} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?<Payload>mysqld_safe .*)/
+ time_format %y%m%d %k:%M:%S
+ time_key Timestamp
+ key_name Payload
+ reserve_data true
+</filter>
+
+# Parse MariaDB logs with 8 digit date format (mysqld)
+<filter infra.mariadb.mysqld>
+ @type parser
+ format /^(?<Timestamp>\d{4}-\d{2}-\d{2} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?<Payload>\w+ +(\[(?<log_level>\w+)\]|\w+: +(?<log_level>\w+):).*)/
+ time_format %Y-%m-%d %k:%M:%S
+ time_key Timestamp
+ key_name Payload
+ reserve_data true
+</filter>
+
+# Re-add timestamp record now that the log date has been parsed
+<filter infra.mariadb.*>
+ @type record_transformer
+ <record>
+ timestamp ${time}
+ </record>
+</filter>
diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
index 3933a38568..98797b09b8 100644
--- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
+++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
@@ -1,11 +1,12 @@
+# mysqld and its wrapper script mysqld_safe output logs using a different timestamp.
+# Defer parsing the logs until the different formats have been retagged.
<source>
@type tail
path /var/log/kolla/mariadb/mariadb.log
pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
tag infra.mariadb
format multiline
- format_firstline /^\d{6}/
- format1 /^(?<time>\d{6} \d{1,2}:\d{1,2}:\d{1,2}) (\[(?<log_level>\S+)\]|mysqld_safe) (?<Payload>.*)/
- time_format %y%m%d %k:%M:%S
+ format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
+ format1 /^(?<Payload>.*)/
enable_watch_timer false
</source>