openstack/kolla-ansible
Code Issues Proposed changes

Refactor fluentd syslog logging

Browse Source 
Co-Authored-By: Mark Goddard <mark@stackhpc.com>

Change-Id: I75ca59d981bcd2dd51faa296ab0b4223a891f5cb
This commit is contained in:
Michal Nasiadka 2020-08-06 12:38:36 +02:00 committed by Mark Goddard
parent 654edefca3
commit b97832dd4f
4 changed files with 40 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ansible/roles/common/defaults/main.yml
View File

@ -71,6 +71,30 @@ syslog_haproxy_facility: "local1"
syslog_glance_tls_proxy_facility: "local2"
syslog_neutron_tls_proxy_facility: "local4"
syslog_facilities:
- name: "swift"
enabled: "{{ enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) }}"
facility: "{{ syslog_swift_facility }}"
logdir: "swift"
logfile: "swift_latest"
output_tag: true
output_time: true
- name: "haproxy"
enabled: "{{ enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] }}"
facility: "{{ syslog_haproxy_facility }}"
logdir: "haproxy"
logfile: "haproxy_latest"
- name: "glance_tls_proxy"
enabled: "{{ glance_enable_tls_backend | bool and inventory_hostname in groups['glance-api'] }}"
facility: "{{ syslog_glance_tls_proxy_facility }}"
logdir: "glance-tls-proxy"
logfile: "glance-tls-proxy"
- name: "neutron_tls_proxy"
enabled: "{{ neutron_enable_tls_backend | bool and inventory_hostname in groups['neutron-server'] }}"
facility: "{{ syslog_neutron_tls_proxy_facility }}"
logdir: "neutron-tls-proxy"
logfile: "neutron-tls-proxy"
kolla_toolbox_default_volumes:
- "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"

27
ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
View File

@ -22,33 +22,14 @@
</record>
</filter>
<filter syslog.local0.**>
{% for item in syslog_facilities | selectattr('enabled') %}
<filter syslog.{{ item.facility }}.**>
@type record_transformer
<record>
programname swift
</record>
</filter>
<filter syslog.local1.**>
@type record_transformer
<record>
programname haproxy
</record>
</filter>
<filter syslog.{{ syslog_glance_tls_proxy_facility }}.**>
@type record_transformer
<record>
programname glance-tls-proxy
</record>
</filter>
<filter syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
@type record_transformer
<record>
programname neutron-tls-proxy
programname {{ item.logdir }}
</record>
</filter>
{% endfor %}
# Rename internal Fluent message field to match other logs. This removes
# all other fields by default, including the original message field. This is

212
ansible/roles/common/templates/conf/output/00-local.conf.j2
View File

@ -1,81 +1,17 @@
{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
<match syslog.{{ syslog_swift_facility }}.**>
{% for item in syslog_facilities | selectattr('enabled') %}
<match syslog.{{ item.facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/swift/swift_latest
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}
append true
# Disable timestamp in filename for logs
<buffer []>
path /var/log/kolla/swift/swift_latest.*.buffer
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}.*.buffer
</buffer>
</store>
{% if log_direct_to_elasticsearch %}
<store>
@type elasticsearch
host {{ elasticsearch_address }}
port {{ elasticsearch_port }}
scheme {{ fluentd_elasticsearch_scheme }}
{% if fluentd_elasticsearch_path != '' %}
path {{ fluentd_elasticsearch_path }}
{% endif %}
{% if fluentd_elasticsearch_scheme == 'https' %}
ssl_version {{ fluentd_elasticsearch_ssl_version }}
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
{% if fluentd_elasticsearch_cacert | length > 0 %}
ca_file {{ fluentd_elasticsearch_cacert }}
{% endif %}
{% endif %}
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
user {{ fluentd_elasticsearch_user }}
password {{ fluentd_elasticsearch_password }}
{% endif %}
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
suppress_type_name true
</store>
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_api {{ monasca_log_api_internal_endpoint }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_swift_facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
chunk_limit_size 8m
</buffer>
</store>
{% endif %}
</match>
{% endif %}
{% if enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] %}
<match syslog.{{ syslog_haproxy_facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/haproxy/haproxy_latest
append true
# Disable timestamp in filename for logs
<buffer []>
path /var/log/kolla/haproxy/haproxy_latest.*.buffer
</buffer>
# Don't prepend syslog tag or timestamp to log output
<format>
output_tag false
output_time false
output_tag {{ item.output_tag | default(false) | lower }}
output_time {{ item.output_time | default(false) | lower }}
</format>
</store>
{% if log_direct_to_elasticsearch %}
@ -103,7 +39,7 @@
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
suppress_type_name true
</store>
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
@ -118,7 +54,7 @@
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_haproxy_facility }}.*
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ item.facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
@ -127,134 +63,4 @@
</store>
{% endif %}
</match>
{% endif %}
{% if glance_enable_tls_backend | bool %}
<match syslog.{{ syslog_glance_tls_proxy_facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/glance-tls-proxy/glance-tls-proxy
append true
# Disable timestamp in filename for logs
<buffer []>
path /var/log/kolla/glance-tls-proxy/glance-tls-proxy.*.buffer
</buffer>
# Don't prepend syslog tag or timestamp to log output
<format>
output_tag false
output_time false
</format>
</store>
{% if log_direct_to_elasticsearch %}
<store>
@type elasticsearch
host {{ elasticsearch_address }}
port {{ elasticsearch_port }}
scheme {{ fluentd_elasticsearch_scheme }}
{% if fluentd_elasticsearch_path != '' %}
path {{ fluentd_elasticsearch_path }}
{% endif %}
{% if fluentd_elasticsearch_scheme == 'https' %}
ssl_version {{ fluentd_elasticsearch_ssl_version }}
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
{% endif %}
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
user {{ fluentd_elasticsearch_user }}
password {{ fluentd_elasticsearch_password }}
{% endif %}
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
suppress_type_name true
</store>
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_api {{ monasca_log_api_internal_endpoint }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
chunk_limit_size 8m
</buffer>
</store>
{% endif %}
</match>
{% endif %}
{% if neutron_enable_tls_backend | bool %}
<match syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy
append true
# Disable timestamp in filename for logs
<buffer []>
path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy.*.buffer
</buffer>
# Don't prepend syslog tag or timestamp to log output
<format>
output_tag false
output_time false
</format>
</store>
{% if log_direct_to_elasticsearch %}
<store>
@type elasticsearch
host {{ elasticsearch_address }}
port {{ elasticsearch_port }}
scheme {{ fluentd_elasticsearch_scheme }}
{% if fluentd_elasticsearch_path != '' %}
path {{ fluentd_elasticsearch_path }}
{% endif %}
{% if fluentd_elasticsearch_scheme == 'https' %}
ssl_version {{ fluentd_elasticsearch_ssl_version }}
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
{% endif %}
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
user {{ fluentd_elasticsearch_user }}
password {{ fluentd_elasticsearch_password }}
{% endif %}
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
suppress_type_name true
</store>
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_api {{ monasca_log_api_internal_endpoint }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
chunk_limit_size 8m
</buffer>
</store>
{% endif %}
</match>
{% endif %}
{% endfor %}

27
ansible/roles/common/templates/fluentd.json.j2
View File

@ -19,34 +19,13 @@
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% if enable_haproxy | bool and inventory_hostname in groups['loadbalancer'] %}
{% for facility in syslog_facilities | selectattr('enabled') %}
{
"path": "/var/log/kolla/haproxy",
"path": "/var/log/kolla/{{ facility.logdir }}",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{% if glance_enable_tls_backend | bool %}
{
"path": "/var/log/kolla/glance-tls-proxy",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{% if neutron_enable_tls_backend | bool %}
{
"path": "/var/log/kolla/neutron-tls-proxy",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
{
"path": "/var/log/kolla/swift",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{% endfor %}
{
"path": "/var/lib/fluentd/data",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",