From 81ab7008e1c4420a3a0c6171e9bdb623ecdc12ff Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Thu, 13 Feb 2020 14:36:35 +0000 Subject: [PATCH] Exposed beat logging levels The elastic beats are currently deployed with a default logging level of info, which can produce large amounts of data, particularly in the case that beats are monitoring one another's logs, and then logging this fact. This commit exposes the configuration, allowing it to be set by the variable log_level. The default level is unchanged. Change-Id: Idbf715fc99450003920a45323123fcba919f2a83 --- elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml | 4 ++++ .../roles/elastic_auditbeat/templates/auditbeat.yml.j2 | 2 +- elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml | 4 ++++ .../roles/elastic_filebeat/templates/filebeat.yml.j2 | 2 +- elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml | 4 ++++ .../roles/elastic_heartbeat/templates/heartbeat.yml.j2 | 2 +- elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml | 4 ++++ .../roles/elastic_journalbeat/templates/journalbeat.yml.j2 | 2 +- elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml | 4 ++++ .../roles/elastic_metricbeat/templates/metricbeat.yml.j2 | 2 +- elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml | 5 +++++ .../roles/elastic_packetbeat/templates/packetbeat.yml.j2 | 2 +- elk_metrics_7x/templates/_macros.j2 | 4 ++-- 13 files changed, 33 insertions(+), 8 deletions(-) diff --git a/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml index 93c4aa96..3388b041 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml @@ -28,3 +28,7 @@ ilm_policy_file_location: "{{ (auditbeat_ilm_policy_file_location | default(defa # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + +# Logging level +# Available log levels are error, warning, info, debug +auditbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 index c3e433ac..8a2480cb 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 @@ -842,7 +842,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('auditbeat') }} +{{ elk_macros.beat_logging('auditbeat', auditbeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml index ff9c51bb..019b4c34 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml @@ -296,3 +296,7 @@ ilm_policy_file_location: "{{ (filebeat_ilm_policy_file_location | default(defau # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + +# Logging level +# Available log levels are error, warning, info, debug +filebeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 index e1caea51..be689d5e 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -1381,7 +1381,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('filebeat') }} +{{ elk_macros.beat_logging('filebeat', filebeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml index 93494a7d..130f6708 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml @@ -28,3 +28,7 @@ ilm_policy_file_location: "{{ (heartbeat_ilm_policy_file_location | default(defa # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + +# Logging level +# Available log levels are error, warning, info, debug +heartbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index f50865b7..19ead7aa 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -1024,7 +1024,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('heartbeat') }} +{{ elk_macros.beat_logging('heartbeat', heartbeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml index 169a5315..f6e0ba4f 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml @@ -28,3 +28,7 @@ ilm_policy_file_location: "{{ (journalbeat_ilm_policy_file_location | default(de # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + +# Logging level +# Available log levels are error, warning, info, debug +journalbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index 97e2cc97..6c7b0d5e 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -780,7 +780,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('journalbeat') }} +{{ elk_macros.beat_logging('journalbeat', journalbeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml index 428961f3..541d0b95 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml @@ -39,3 +39,7 @@ ilm_policy_file_location: "{{ (metricbeat_ilm_policy_file_location | default(def # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + +# Logging level +# Available log levels are error, warning, info, debug +metricbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index cff219bc..071bcc4a 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -1501,7 +1501,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('metricbeat') }} +{{ elk_macros.beat_logging('metricbeat', metricbeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml index a5fb3e2d..d0494e92 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml @@ -29,5 +29,10 @@ ilm_policy_file_location: "{{ (packetbeat_ilm_policy_file_location | default(def # beat processors. Empty dictionary sets `add_host_metadata: ~` processors: {} + # optional list of ports to ignore packetbeat_ignored_ports: [] + +# Logging level +# Available log levels are error, warning, info, debug +packetbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" diff --git a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index 42597021..87654778 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -1216,7 +1216,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #================================ Logging ====================================== -{{ elk_macros.beat_logging('packetbeat') }} +{{ elk_macros.beat_logging('packetbeat', packetbeat_log_level) }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} diff --git a/elk_metrics_7x/templates/_macros.j2 b/elk_metrics_7x/templates/_macros.j2 index c43527a3..dcc1de63 100644 --- a/elk_metrics_7x/templates/_macros.j2 +++ b/elk_metrics_7x/templates/_macros.j2 @@ -360,13 +360,13 @@ setup.kibana: #ssl.curve_types: [] {%- endmacro %} -{% macro beat_logging(beat_name) -%} +{% macro beat_logging(beat_name, log_level='info') -%} # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: info +logging.level: {{ log_level }} # Enable debug output for selected components. To enable all selectors use ["*"] # Other available selectors are "beat", "publish", "service"