diff --git a/elk_metrics_7x/createElasticIndexes.yml b/elk_metrics_7x/createElasticIndexes.yml index 12a9bc58..4af8faac 100644 --- a/elk_metrics_7x/createElasticIndexes.yml +++ b/elk_metrics_7x/createElasticIndexes.yml @@ -19,7 +19,7 @@ - role: elastic_data_hosts vars: - _elastic_refresh_interval: "{{ (elasticsearch_number_of_replicas | int) * 5 }}" + _elastic_refresh_interval: "{{ (elasticsearch_beat_settings.number_of_replicas | int) * 5 }}" elastic_refresh_interval: "{{ (_elastic_refresh_interval > 0) | ternary(30, _elastic_refresh_interval) }}" vars_files: @@ -83,7 +83,7 @@ indices.queries.cache.size: "5%" - name: "_all/_settings" index_options: - index.number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + index.number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" index.translog.durability: "async" index.refresh_interval: "{{ ((elastic_refresh_interval | int) > 30) | ternary(30, elastic_refresh_interval) }}s" @@ -129,7 +129,7 @@ | list }} settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" index: mapping: total_fields: @@ -151,7 +151,7 @@ template: ".monitoring*" order: 1 settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" number_of_shards: 1 - name: Create custom skydive index template @@ -170,7 +170,7 @@ template: "skydive*" order: 1 settings: - number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_replicas: "{{ elasticsearch_beat_settings.number_of_replicas | int }}" number_of_shards: 1 diff --git a/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 b/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 index b0dfaf3f..764ab3d5 100644 --- a/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 +++ b/elk_metrics_7x/roles/elastic_apm_server/templates/apm-server.yml.j2 @@ -134,7 +134,7 @@ apm-server: {{ elk_macros.setup_dashboards('apm') }} #=============================== Template ====================================== -{{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 index c8954ad5..b8affa06 100644 --- a/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 @@ -811,7 +811,7 @@ processors: {{ elk_macros.setup_dashboards('auditbeat') }} #=============================== Template ====================================== -{{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml index b8f8897d..5c31b2b3 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/defaults/main.yml @@ -30,3 +30,7 @@ elastic_data_interface: "{{ ansible_default_ipv4['alias'] }}" # to override the default of ansible_host:logstash_beat_input_port # for each node, or to provide different coordinators when necessary # elasticsearch_coordination_node_socket_addresses: [] + +## Elasticsearch index template settings +# Set the default max number of fields used in a query +# elastic_max_docvalue_fields_search: "100" diff --git a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml index 572fbe1f..aad576af 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -205,12 +205,20 @@ elastic_thread_pool_size: "{{ ((ansible_processor_count | int) >= 24) | ternary( # Set a data node facts. The data nodes, in the case of elasticsearch are also # ingest nodes. elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" + +# Input data for the beat config templates +elasticsearch_beat_settings: + number_of_replicas: "{{ elasticsearch_number_of_replicas }}" + max_docvalue_fields_search: "{{ elastic_max_docvalue_fields_search | default('100') }}" + + elasticsearch_data_hosts: |- {% set data_hosts = elasticsearch_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} {% set _ = data_hosts.insert(0, '127.0.0.1:' ~ elastic_port) %} {% endif %} {{ data_hosts }} + logstash_data_hosts: |- {% set data_hosts = logstash_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} diff --git a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 index d3413f7c..a3da522c 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -1349,7 +1349,7 @@ processors: {{ elk_macros.setup_dashboards('filebeat') }} #============================== Template ====================================== -{{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index fa574cc1..c714e21a 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -994,7 +994,7 @@ processors: {{ elk_macros.setup_dashboards('heartbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index 1d1807f5..864bd7b2 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -749,7 +749,7 @@ processors: {{ elk_macros.setup_dashboards('journalbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index e9ab2bd4..52e959e9 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -1460,8 +1460,8 @@ processors: {{ elk_macros.setup_dashboards('metricbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} - +{{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} + #============================== Setup ILM ===================================== # Configure Index Lifecycle Management Index Lifecycle Management creates a diff --git a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index dc947f4c..8ae35f18 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -1185,7 +1185,7 @@ processors: {{ elk_macros.setup_dashboards('packetbeat') }} #============================== Template ===================================== -{{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +{{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== diff --git a/elk_metrics_7x/templates/_macros.j2 b/elk_metrics_7x/templates/_macros.j2 index 9147af4a..2f52352c 100644 --- a/elk_metrics_7x/templates/_macros.j2 +++ b/elk_metrics_7x/templates/_macros.j2 @@ -249,7 +249,7 @@ setup.dashboards.enabled: false #setup.dashboards.retry.maximum: 0 {%- endmacro %} -{% macro setup_template(beat_name, host, data_nodes, elasticsearch_replicas) -%} +{% macro setup_template(beat_name, host, data_nodes, elasticsearch_beat_settings={}) -%} # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. # These settings can be adjusted to load your own template or overwrite existing ones. @@ -296,7 +296,11 @@ setup.template.settings: number_of_routing_shards: {{ (shards | int) * 2 }} # The default number of replicas will be based on the number of data nodes # within the environment with a limit of 2 replicas. - number_of_replicas: {{ elasticsearch_replicas | int }} + number_of_replicas: {{ elasticsearch_beat_settings.number_of_replicas | int }} + # Maximum doc values allowed for default kibana search +{% if 'max_docvalue_fields_search' in elasticsearch_beat_settings %} + max_docvalue_fields_search: {{ elasticsearch_beat_settings.max_docvalue_fields_search | int }} +{% endif %} # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html