From b97832dd4ffa5fba06c38afb44cce522dbafefdc Mon Sep 17 00:00:00 2001
From: Michal Nasiadka <mnasiadka@gmail.com>
Date: Thu, 6 Aug 2020 12:38:36 +0200
Subject: [PATCH] Refactor fluentd syslog logging
Co-Authored-By: Mark Goddard <mark@stackhpc.com>
Change-Id: I75ca59d981bcd2dd51faa296ab0b4223a891f5cb
---
ansible/roles/common/defaults/main.yml | 24 ++
.../conf/filter/00-record_transformer.conf.j2 | 27 +--
.../templates/conf/output/00-local.conf.j2 | 212 +-----------------
.../roles/common/templates/fluentd.json.j2 | 27 +--
4 files changed, 40 insertions(+), 250 deletions(-)
diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index f99da8609d..b7239d4719 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -71,6 +71,30 @@ syslog_haproxy_facility: "local1"
syslog_glance_tls_proxy_facility: "local2"
syslog_neutron_tls_proxy_facility: "local4"
+syslog_facilities:
+ - name: "swift"
+ enabled: "{{ enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) }}"
+ facility: "{{ syslog_swift_facility }}"
+ logdir: "swift"
+ logfile: "swift_latest"
+ output_tag: true
+ output_time: true
+ - name: "haproxy"
+ enabled: "{{ enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] }}"
+ facility: "{{ syslog_haproxy_facility }}"
+ logdir: "haproxy"
+ logfile: "haproxy_latest"
+ - name: "glance_tls_proxy"
+ enabled: "{{ glance_enable_tls_backend | bool and inventory_hostname in groups['glance-api'] }}"
+ facility: "{{ syslog_glance_tls_proxy_facility }}"
+ logdir: "glance-tls-proxy"
+ logfile: "glance-tls-proxy"
+ - name: "neutron_tls_proxy"
+ enabled: "{{ neutron_enable_tls_backend | bool and inventory_hostname in groups['neutron-server'] }}"
+ facility: "{{ syslog_neutron_tls_proxy_facility }}"
+ logdir: "neutron-tls-proxy"
+ logfile: "neutron-tls-proxy"
+
kolla_toolbox_default_volumes:
- "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
index 2d5fef5bbd..723a37dfc8 100644
--- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
@@ -22,33 +22,14 @@
</record>
</filter>
-<filter syslog.local0.**>
+{% for item in syslog_facilities | selectattr('enabled') %}
+<filter syslog.{{ item.facility }}.**>
@type record_transformer
<record>
- programname swift
- </record>
-</filter>
-
-<filter syslog.local1.**>
- @type record_transformer
- <record>
- programname haproxy
- </record>
-</filter>
-
-<filter syslog.{{ syslog_glance_tls_proxy_facility }}.**>
- @type record_transformer
- <record>
- programname glance-tls-proxy
- </record>
-</filter>
-
-<filter syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
- @type record_transformer
- <record>
- programname neutron-tls-proxy
+ programname {{ item.logdir }}
</record>
</filter>
+{% endfor %}
# Rename internal Fluent message field to match other logs. This removes
# all other fields by default, including the original message field. This is
diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index 4b8067e0db..33e821ac74 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -1,81 +1,17 @@
-{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
-<match syslog.{{ syslog_swift_facility }}.**>
+{% for item in syslog_facilities | selectattr('enabled') %}
+<match syslog.{{ item.facility }}.**>
@type copy
<store>
@type file
- path /var/log/kolla/swift/swift_latest
+ path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}
append true
# Disable timestamp in filename for logs
<buffer []>
- path /var/log/kolla/swift/swift_latest.*.buffer
+ path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}.*.buffer
</buffer>
- </store>
-{% if log_direct_to_elasticsearch %}
- <store>
- @type elasticsearch
- host {{ elasticsearch_address }}
- port {{ elasticsearch_port }}
- scheme {{ fluentd_elasticsearch_scheme }}
-{% if fluentd_elasticsearch_path != '' %}
- path {{ fluentd_elasticsearch_path }}
-{% endif %}
-{% if fluentd_elasticsearch_scheme == 'https' %}
- ssl_version {{ fluentd_elasticsearch_ssl_version }}
- ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
-{% if fluentd_elasticsearch_cacert | length > 0 %}
- ca_file {{ fluentd_elasticsearch_cacert }}
-{% endif %}
-{% endif %}
-{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
- user {{ fluentd_elasticsearch_user }}
- password {{ fluentd_elasticsearch_password }}
-{% endif %}
- logstash_format true
- logstash_prefix {{ kibana_log_prefix }}
- flush_interval 15s
- reconnect_on_error true
- buffer_type file
- buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
- suppress_type_name true
- </store>
-{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
- <store>
- @type monasca
- keystone_url {{ keystone_internal_url }}
- monasca_api {{ monasca_log_api_internal_endpoint }}
- monasca_api_version v2.0
- username {{ monasca_agent_user }}
- password {{ monasca_agent_password }}
- domain_id default
- project_name {{ monasca_control_plane_project }}
- message_field_name Payload
- buffer_type file
- buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_swift_facility }}.*
- max_retry_wait 1800s
- disable_retry_limit true
- <buffer>
- chunk_limit_size 8m
- </buffer>
- </store>
-{% endif %}
-</match>
-{% endif %}
-
-{% if enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] %}
-<match syslog.{{ syslog_haproxy_facility }}.**>
- @type copy
- <store>
- @type file
- path /var/log/kolla/haproxy/haproxy_latest
- append true
- # Disable timestamp in filename for logs
- <buffer []>
- path /var/log/kolla/haproxy/haproxy_latest.*.buffer
- </buffer>
- # Don't prepend syslog tag or timestamp to log output
<format>
- output_tag false
- output_time false
+ output_tag {{ item.output_tag | default(false) | lower }}
+ output_time {{ item.output_time | default(false) | lower }}
</format>
</store>
{% if log_direct_to_elasticsearch %}
@@ -103,7 +39,7 @@
flush_interval 15s
reconnect_on_error true
buffer_type file
- buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
+ buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
suppress_type_name true
</store>
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
@@ -118,7 +54,7 @@
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
- buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_haproxy_facility }}.*
+ buffer_path /var/lib/fluentd/data/monasca.buffer/{{ item.facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
@@ -127,134 +63,4 @@
</store>
{% endif %}
</match>
-{% endif %}
-
-{% if glance_enable_tls_backend | bool %}
-<match syslog.{{ syslog_glance_tls_proxy_facility }}.**>
- @type copy
- <store>
- @type file
- path /var/log/kolla/glance-tls-proxy/glance-tls-proxy
- append true
- # Disable timestamp in filename for logs
- <buffer []>
- path /var/log/kolla/glance-tls-proxy/glance-tls-proxy.*.buffer
- </buffer>
- # Don't prepend syslog tag or timestamp to log output
- <format>
- output_tag false
- output_time false
- </format>
- </store>
-{% if log_direct_to_elasticsearch %}
- <store>
- @type elasticsearch
- host {{ elasticsearch_address }}
- port {{ elasticsearch_port }}
- scheme {{ fluentd_elasticsearch_scheme }}
-{% if fluentd_elasticsearch_path != '' %}
- path {{ fluentd_elasticsearch_path }}
-{% endif %}
-{% if fluentd_elasticsearch_scheme == 'https' %}
- ssl_version {{ fluentd_elasticsearch_ssl_version }}
- ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
-{% endif %}
-{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
- user {{ fluentd_elasticsearch_user }}
- password {{ fluentd_elasticsearch_password }}
-{% endif %}
- logstash_format true
- logstash_prefix {{ kibana_log_prefix }}
- flush_interval 15s
- reconnect_on_error true
- buffer_type file
- buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
- suppress_type_name true
- </store>
-{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
- <store>
- @type monasca
- keystone_url {{ keystone_internal_url }}
- monasca_api {{ monasca_log_api_internal_endpoint }}
- monasca_api_version v2.0
- username {{ monasca_agent_user }}
- password {{ monasca_agent_password }}
- domain_id default
- project_name {{ monasca_control_plane_project }}
- message_field_name Payload
- buffer_type file
- buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.*
- max_retry_wait 1800s
- disable_retry_limit true
- <buffer>
- chunk_limit_size 8m
- </buffer>
- </store>
-{% endif %}
-</match>
-{% endif %}
-
-{% if neutron_enable_tls_backend | bool %}
-<match syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
- @type copy
- <store>
- @type file
- path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy
- append true
- # Disable timestamp in filename for logs
- <buffer []>
- path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy.*.buffer
- </buffer>
- # Don't prepend syslog tag or timestamp to log output
- <format>
- output_tag false
- output_time false
- </format>
- </store>
-{% if log_direct_to_elasticsearch %}
- <store>
- @type elasticsearch
- host {{ elasticsearch_address }}
- port {{ elasticsearch_port }}
- scheme {{ fluentd_elasticsearch_scheme }}
-{% if fluentd_elasticsearch_path != '' %}
- path {{ fluentd_elasticsearch_path }}
-{% endif %}
-{% if fluentd_elasticsearch_scheme == 'https' %}
- ssl_version {{ fluentd_elasticsearch_ssl_version }}
- ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
-{% endif %}
-{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
- user {{ fluentd_elasticsearch_user }}
- password {{ fluentd_elasticsearch_password }}
-{% endif %}
- logstash_format true
- logstash_prefix {{ kibana_log_prefix }}
- flush_interval 15s
- reconnect_on_error true
- buffer_type file
- buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
- suppress_type_name true
- </store>
-{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
- <store>
- @type monasca
- keystone_url {{ keystone_internal_url }}
- monasca_api {{ monasca_log_api_internal_endpoint }}
- monasca_api_version v2.0
- username {{ monasca_agent_user }}
- password {{ monasca_agent_password }}
- domain_id default
- project_name {{ monasca_control_plane_project }}
- message_field_name Payload
- buffer_type file
- buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
- max_retry_wait 1800s
- disable_retry_limit true
- <buffer>
- chunk_limit_size 8m
- </buffer>
- </store>
-{% endif %}
-</match>
-{% endif %}
+{% endfor %}
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 6c091e09a0..712182c14a 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -19,34 +19,13 @@
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
-{% if enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] %}
+{% for facility in syslog_facilities | selectattr('enabled') %}
{
- "path": "/var/log/kolla/haproxy",
+ "path": "/var/log/kolla/{{ facility.logdir }}",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
-{% endif %}
-{% if glance_enable_tls_backend | bool %}
- {
- "path": "/var/log/kolla/glance-tls-proxy",
- "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
- "recurse": true
- },
-{% endif %}
-{% if neutron_enable_tls_backend | bool %}
- {
- "path": "/var/log/kolla/neutron-tls-proxy",
- "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
- "recurse": true
- },
-{% endif %}
-{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
- {
- "path": "/var/log/kolla/swift",
- "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
- "recurse": true
- },
-{% endif %}
+{% endfor %}
{
"path": "/var/lib/fluentd/data",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",