Allow beat processors to be defined through configuration data

Processors for each of the beats are currently defined per template and
can be configured directly via template or once deployed. This change
allows processors for all or a subset of beats to be configured through
role data or via an overlay inventory.

Change-Id: I71fc96611082555b43fd0f505219c42c890464ef
This commit is contained in:
Sam Choraria 2020-02-05 13:22:58 +00:00
parent 18a62b704c
commit 81eb58f1e8
13 changed files with 37 additions and 12 deletions
elk_metrics_7x
roles
elastic_auditbeat
defaults
templates
elastic_filebeat
defaults
templates
elastic_heartbeat
defaults
templates
elastic_journalbeat
elastic_metricbeat
defaults
templates
elastic_packetbeat
defaults
templates
templates

@ -25,3 +25,6 @@ ilm_policy: "{{ (auditbeat_ilm_policy | default(default_ilm_policy)) | default({
ilm_policy_template: "{{ auditbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (auditbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (auditbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -266,6 +266,7 @@ auditbeat.modules:
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -375,8 +376,6 @@ auditbeat.modules:
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
#============================= Elastic Cloud ==================================

@ -293,3 +293,6 @@ ilm_policy: "{{ (filebeat_ilm_policy | default(default_ilm_policy)) | default({}
ilm_policy_template: "{{ filebeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (filebeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (filebeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -765,6 +765,7 @@ filebeat.inputs:
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -852,8 +853,8 @@ filebeat.inputs:
#
# The following example enriches each event with host metadata.
#
processors:
- add_host_metadata: ~
#processors:
#- add_host_metadata: ~
# netinfo.enabled: false
#
# The following example enriches each event with process metadata using

@ -25,3 +25,6 @@ ilm_policy: "{{ (heartbeat_ilm_policy | default(default_ilm_policy)) | default({
ilm_policy_template: "{{ heartbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (heartbeat_ilm_policy_filename | default(heartbeat_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (heartbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -407,6 +407,7 @@ heartbeat.scheduler:
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -516,8 +517,6 @@ heartbeat.scheduler:
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
#============================= Elastic Cloud ==================================

@ -25,3 +25,6 @@ ilm_policy: "{{ (journalbeat_ilm_policy | default(default_ilm_policy)) | default
ilm_policy_template: "{{ journalbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (journalbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (journalbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -162,6 +162,7 @@ tags:
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -272,8 +273,6 @@ tags:
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata:
#============================= Elastic Cloud ==================================
# These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/).

@ -36,3 +36,6 @@ ilm_policy: "{{ (metricbeat_ilm_policy | default(default_ilm_policy)) | default(
ilm_policy_template: "{{ metricbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (metricbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (metricbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -880,6 +880,7 @@ metricbeat.modules:
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -989,8 +990,6 @@ metricbeat.modules:
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
#============================= Elastic Cloud ==================================

@ -25,3 +25,6 @@ ilm_policy: "{{ (packetbeat_ilm_policy | default(default_ilm_policy)) | default(
ilm_policy_template: "{{ packetbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (packetbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (packetbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
# beat processors. Empty dictionary sets `add_host_metadata: ~`
processors: {}

@ -599,6 +599,7 @@ packetbeat.ignore_outgoing: false
#max_procs:
#================================ Processors ===================================
{{ elk_macros.beat_processors(processors) }}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
@ -708,8 +709,6 @@ packetbeat.ignore_outgoing: false
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
#============================= Elastic Cloud ==================================

@ -524,3 +524,14 @@ xpack.monitoring.elasticsearch:
#metrics.period: 10s
#state.period: 1m
{%- endmacro %}
{% macro beat_processors(processors) -%}
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata.
processors:
{% if processors is defined and processors is iterable and processors | length > 0 %}
{{ processors | to_yaml }}
{% else %}
- add_host_metadata: ~
{%- endif %}
{%- endmacro %}