From 01352a45f611ebf9301a8c06084b8f6443d2ba9f Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Wed, 20 May 2020 17:07:22 +0100 Subject: [PATCH] Ensure that the beat setup role runs when elk_beat_setup=true Currently it is possible for the elk_beat_setup flag not to trigger the beat_setup role when no kibana nodes are present. This means that there is no way to update the index templates in this situation, so beats will just use a default index setup, and ILM policies will not be applied. This can lead to inconsistent configurations where manual deletion of default-templated indices is required. This can happen during an ELK version upgrade, where new templates should be created but aren't. This commit ensures that the beat setup role is run whenever the elk_beat_setup flag is set to true. As a result, the user can force index templates to be updated even when no kibana nodes are present to ensure consistency. It also documents the previously undocumented elk_beat_setup var in both the Beat role defaults and the beat_setup role itself Change-Id: Ife0d0f12d6b300b84b63c3af14add1732747d44f --- elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_auditbeat/tasks/main.yml | 2 +- elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml | 2 +- elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml | 2 +- elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml | 2 +- elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml | 2 +- elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml | 5 +++++ elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml | 2 +- 13 files changed, 41 insertions(+), 6 deletions(-) diff --git a/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml index 7e7287ba..8d9bb8e6 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml @@ -37,3 +37,8 @@ auditbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" # particularly when auditbeat is deployed to a container host, and largely overlaps # with Packetbeat data auditbeat_ignore_socket_data: false + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_auditbeat/tasks/main.yml index 2c77cd08..13abcb5d 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_auditbeat/tasks/main.yml @@ -98,7 +98,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "auditbeat" diff --git a/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml b/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml index a7794883..d6c0b4ee 100644 --- a/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml @@ -32,3 +32,8 @@ elastic_beat_kibana_host: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" # This is by default set to elastic_beat_kibana_host, but if using a non-standard # interface, the no_proxy enivronment may need to be changed. elastic_beat_no_proxy: "{{ elastic_beat_kibana_host }}" + +# Override flag to force set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml index a8aced0b..1427dac5 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml @@ -306,3 +306,8 @@ filebeat_iptables_enabled: false # The following allows one to specify log files in non-standard locations, # defaulting to log collection in syslog filebeat_iptables_log_paths: ["var/log/syslog"] + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml index b442898c..2c601a6f 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml @@ -101,7 +101,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "filebeat" diff --git a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml index 47227176..2fe60e70 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml @@ -36,3 +36,8 @@ heartbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" # List of interfaces with which heartbeat tests connectivity. # Defaults to ansible_host on all hosts in the inventory to which heartbeat is deployed # heartbeat_icmp_hosts: "{{ groups['heartbeat'] | default(groups['heartbeat_deployment_targets']) | map('extract', hostvars, ['ansible_host']) | list }}" + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml index db454447..6c3f7c4a 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml @@ -96,7 +96,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "heartbeat" diff --git a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml index e2db9984..4f914e87 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml @@ -43,3 +43,8 @@ journalbeat_queue_flush_min_events: 2048 # Max time before queue is flushed (even if queue_min_events is not reached) journalbeat_queue_flush_timeout: 1s + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml index e991930f..dc4a534b 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml @@ -106,7 +106,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "journalbeat" diff --git a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml index 541d0b95..ede1d4c0 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml @@ -43,3 +43,8 @@ processors: {} # Logging level # Available log levels are error, warning, info, debug metricbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml index d32d2293..d8c1de13 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml @@ -316,7 +316,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "metricbeat" diff --git a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml index d0494e92..faa8d6e2 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml @@ -36,3 +36,8 @@ packetbeat_ignored_ports: [] # Logging level # Available log levels are error, warning, info, debug packetbeat_log_level: "{{ elastic_beat_log_level | default('info') }}" + +# Override flag to force Beats to set up index templates. If this is not +# set, templates are only pushed when the user is either upgrading the +# beat version or deploying for the first time in the presence of kibana nodes +elk_beat_setup: false diff --git a/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml index e1dee70c..0be9d95d 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml @@ -81,7 +81,7 @@ include_role: name: elastic_beat_setup when: - - (groups['kibana'] | length) > 0 + - (groups['kibana'] | length) > 0 or (elk_beat_setup | bool) vars: elastic_beat_name: "packetbeat"