From 38a6d30411be2b8bb116bfb765b054309a3a9c91 Mon Sep 17 00:00:00 2001
From: Bertrand Lallau <bertrand.lallau@thalesgroup.com>
Date: Thu, 17 Aug 2017 18:05:20 +0200
Subject: [PATCH] Fluentd: manage multiline log pattern
Actual Fluentd config doesn't manage multiline log pattern like:
2017-08-16 21:41:03.097 7 WARNING oslo_config.cfg [-] Option "firewall_driver" from group "DEFAULT" is deprecated for removal (
nova-network is deprecated, as are any related configuration options.
). Its value may be silently ignored in the future.
This log will generate 1 Fluentd event for each line (in previous
example 3 events).
This is really a wrong behaviour and made Kibana interface not friendly.
This patch uses a multiline parser.
Change-Id: Iee337645d1dc42876ec79be08ee3715f11a53d87
Closes-Bug: #1711399
---
ansible/roles/common/tasks/config.yml | 1 -
.../roles/common/templates/conf/input/00-global.conf.j2 | 7 ++++++-
ansible/roles/common/templates/fluentd.json.j2 | 6 ------
3 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 27024db028..8eccc94f20 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -60,7 +60,6 @@
with_items:
- "apache_access"
- "wsgi_access"
- - "wsgi_python"
notify:
- Restart fluentd container
diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2
index b1fa69d92f..013785250a 100644
--- a/ansible/roles/common/templates/conf/input/00-global.conf.j2
+++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2
@@ -45,5 +45,10 @@
"/var/log/kolla/*/*-error.log"]
pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
tag kolla.*
- format /^(?<message>.*)$/
+ format multiline
+ format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+ format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+ time_key Timestamp
+ time_format %F %T.%L
+ ignore_repeated_permission_error true
</source>
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 95cacc5def..b72dd226c8 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -64,12 +64,6 @@
"dest": "{{ fluentd_dir }}/format/wsgi_access.conf",
"owner": "{{ fluentd_user }}",
"perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/format/wsgi_python.conf",
- "dest": "{{ fluentd_dir }}/format/wsgi_python.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
},{% if enable_elasticsearch | bool or
( elasticsearch_address != kolla_internal_vip_address ) | bool %}
{