Support monitoring Fluentd with Prometheus
This patch adds support for integrating Prometheus with Fluentd. This can be used to extract useful information about the status of Fluentd, such as output buffer capacity and logging rate, and also to extract metrics from logs via custom Fluentd configuration. More information can be found here in [1]. [1] https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus Change-Id: I233d6dd744848ef1f1589a462dbf272ed0f3aaae
This commit is contained in:
parent
06047a36ff
commit
b692ce7af1
@ -434,6 +434,7 @@ prometheus_memcached_exporter_port: "9150"
|
||||
prometheus_rabbitmq_exporter_port: "15692"
|
||||
# Default cadvisor port of 8080 already in use
|
||||
prometheus_cadvisor_port: "18080"
|
||||
prometheus_fluentd_integration_port: "24231"
|
||||
|
||||
# Prometheus alertmanager ports
|
||||
prometheus_alertmanager_port: "9093"
|
||||
@ -1082,6 +1083,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
|
||||
enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
|
||||
enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}"
|
||||
enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
|
||||
enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}"
|
||||
enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
|
||||
enable_prometheus_ceph_mgr_exporter: "no"
|
||||
enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}"
|
||||
|
@ -109,17 +109,29 @@
|
||||
( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
|
||||
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
|
||||
# Inputs
|
||||
fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
|
||||
fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}"
|
||||
default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}"
|
||||
default_input_files:
|
||||
- "conf/input/00-global.conf.j2"
|
||||
- "conf/input/01-syslog.conf.j2"
|
||||
- "conf/input/02-mariadb.conf.j2"
|
||||
- "conf/input/03-rabbitmq.conf.j2"
|
||||
- "conf/input/04-openstack-wsgi.conf.j2"
|
||||
- "conf/input/05-libvirt.conf.j2"
|
||||
- "conf/input/06-zookeeper.conf.j2"
|
||||
- "conf/input/07-kafka.conf.j2"
|
||||
- "conf/input/09-monasca.conf.j2"
|
||||
- name: "conf/input/00-global.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/01-syslog.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/02-mariadb.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/03-rabbitmq.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/04-openstack-wsgi.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/05-libvirt.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/06-zookeeper.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/07-kafka.conf.j2"
|
||||
enabled: true
|
||||
- name: "conf/input/08-prometheus.conf.j2"
|
||||
enabled: "{{ enable_prometheus_fluentd_integration | bool }}"
|
||||
- name: "conf/input/09-monasca.conf.j2"
|
||||
enabled: true
|
||||
customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
|
||||
# Filters
|
||||
fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
|
||||
|
@ -0,0 +1,14 @@
|
||||
<source>
|
||||
@type prometheus
|
||||
bind {{ api_interface_address }}
|
||||
port {{ prometheus_fluentd_integration_port }}
|
||||
metrics_path /metrics
|
||||
</source>
|
||||
|
||||
<source>
|
||||
@type prometheus_output_monitor
|
||||
interval 10
|
||||
<labels>
|
||||
Hostname ${hostname}
|
||||
</labels>
|
||||
</source>
|
@ -74,6 +74,15 @@ scrape_configs:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if enable_prometheus_fluentd_integration | bool %}
|
||||
- job_name: fluentd
|
||||
static_configs:
|
||||
- targets:
|
||||
{% for host in groups['fluentd'] %}
|
||||
- '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if enable_prometheus_ceph_mgr_exporter | bool %}
|
||||
- job_name: ceph_mgr_exporter
|
||||
honor_labels: true
|
||||
|
@ -685,6 +685,7 @@
|
||||
#enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
|
||||
#enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
|
||||
#enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
|
||||
#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}"
|
||||
#enable_prometheus_memcached: "{{ enable_prometheus | bool }}"
|
||||
#enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
|
||||
#enable_prometheus_ceph_mgr_exporter: "no"
|
||||
|
@ -0,0 +1,10 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Adds support for integrating Fluentd metrics into Prometheus. By
|
||||
default this is now enabled when Prometheus is enabled. This behaviour
|
||||
can be overridden via the `enable_prometheus_fluentd_integration` flag. By
|
||||
default the integration provides metrics relating to the processing of logs
|
||||
by Fluentd. These metrics can be useful for monitoring the status of the
|
||||
Fluentd service. Additional metrics can also be extracted from logs via
|
||||
custom Fluentd config.
|
Loading…
Reference in New Issue
Block a user