diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml index fee27dc1..f2e7905a 100644 --- a/elk_metrics_6x/installElastic.yml +++ b/elk_metrics_6x/installElastic.yml @@ -67,6 +67,8 @@ dest: /etc/elasticsearch/elasticsearch.yml - src: templates/jvm.options.j2 dest: /etc/elasticsearch/jvm.options + - src: templates/es-log4j2.properties.j2 + dest: /etc/elasticsearch/log4j2.properties tags: - config diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 96d21664..3e00a426 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -36,7 +36,7 @@ enabled: true state: restarted - - name: Ensure kibana is installed. + - name: Ensure kibana is installed apt: name: kibana state: present diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 5f16c6b2..9edb925f 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -52,7 +52,7 @@ install_recommends: yes update_cache: yes - - name: Ensure Logstash is installed. + - name: Ensure Logstash is installed apt: name: logstash state: present diff --git a/elk_metrics_6x/readme.rst b/elk_metrics_6x/readme.rst index 1ee32af9..58144c15 100644 --- a/elk_metrics_6x/readme.rst +++ b/elk_metrics_6x/readme.rst @@ -70,6 +70,7 @@ lines. haproxy_backend_port: 9200 # This is set using the "elastic_port" variable haproxy_balance_type: tcp + Optional | run the haproxy-install playbook ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elk_metrics_6x/templates/_include_setup_template.yml.j2 b/elk_metrics_6x/templates/_include_setup_template.yml.j2 new file mode 100644 index 00000000..84820180 --- /dev/null +++ b/elk_metrics_6x/templates/_include_setup_template.yml.j2 @@ -0,0 +1,18 @@ +{% set shards = ((groups["elastic-logstash"] | length) * 3) | int %} + +# Elasticsearch template settings +setup.template.settings: + + # A dictionary of settings to place into the settings.index dictionary + # of the Elasticsearch template. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html + index: + number_of_shards: {{ shards }} + codec: best_compression + # This provides for an index split of up to 2 times the number of available shards + number_of_routing_shards: {{ (shards | int) * 2 * (groups["elastic-logstash"] | length) }} + + # A dictionary of settings for the _source field. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + #_source: + #enabled: false diff --git a/elk_metrics_6x/templates/apm-server.yml.j2 b/elk_metrics_6x/templates/apm-server.yml.j2 index 78751537..9b17a0db 100644 --- a/elk_metrics_6x/templates/apm-server.yml.j2 +++ b/elk_metrics_6x/templates/apm-server.yml.j2 @@ -275,22 +275,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/auditbeat.yml.j2 b/elk_metrics_6x/templates/auditbeat.yml.j2 index 6e1de8e8..d85c5032 100644 --- a/elk_metrics_6x/templates/auditbeat.yml.j2 +++ b/elk_metrics_6x/templates/auditbeat.yml.j2 @@ -716,22 +716,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/elasticsearch.yml.j2 b/elk_metrics_6x/templates/elasticsearch.yml.j2 index 517cdb01..10af56cd 100644 --- a/elk_metrics_6x/templates/elasticsearch.yml.j2 +++ b/elk_metrics_6x/templates/elasticsearch.yml.j2 @@ -43,17 +43,24 @@ http.port: {{ elastic_port }} # {% set IP_ARR=[] %} {% for host in groups['elastic-logstash'] %} -{% if IP_ARR.insert(loop.index,hostvars[host]['ansible_host']) %} +{% if IP_ARR.extend([hostvars[host]['ansible_host'] | string]) %} {% endif %} {% endfor %} -discovery.zen.ping.unicast.hosts: [{{ IP_ARR | join(', ') }}] -node.master: {{ (inventory_hostname == groups['elastic-logstash'][0]) | ternary(true, false) }} -node.data: true +{% set available_nodes = (groups['elastic-logstash'] | length) %} +{# the master node count takes half the available nodes or sets it's self as 1 #} +{% set _master_node_count = (available_nodes > 1) | ternary(((available_nodes // 2) | int), 1) %} +{# if the master node count is even, add one to it otherwise use the provided value #} +{% set master_node_count = ((_master_node_count | int) % 2 != 0) | ternary(_master_node_count, (_master_node_count + 1)) %} + +discovery.zen.ping.unicast.hosts: {{ IP_ARR }} # # Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): -# -# discovery.zen.minimum_master_nodes: 3 -# +discovery.zen.minimum_master_nodes: {{ master_node_count | int }} +# The first set of nodes in the master_node_count are marked as such +node.master: {{ (inventory_hostname in groups['elastic-logstash'][:master_node_count | int]) | ternary(true, false) }} +# Every node in the master list and every other node after will be a data node +node.data: {{ (inventory_hostname in (groups['elastic-logstash'][:master_node_count| int] + groups['elastic-logstash'][master_node_count | int::2])) | ternary(true, false) }} + # For more information, see the documentation at: # # @@ -61,7 +68,7 @@ node.data: true # # Block initial recovery after a full cluster restart until N nodes are started: # -# gateway.recover_after_nodes: 3 +gateway.recover_after_nodes: {{ master_node_count }} # # For more information, see the documentation at: # diff --git a/elk_metrics_6x/templates/es-log4j2.properties.j2 b/elk_metrics_6x/templates/es-log4j2.properties.j2 new file mode 100644 index 00000000..7091d480 --- /dev/null +++ b/elk_metrics_6x/templates/es-log4j2.properties.j2 @@ -0,0 +1,85 @@ +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified +appender.rolling.strategy.action.condition.nested_condition.age = {{ elastic_logstash_retention }}D + + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 128MB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 2 + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log +appender.index_search_slowlog_rolling.layout.type = PatternLayout +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval = 1 +appender.index_search_slowlog_rolling.policies.time.modulate = true + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval = 1 +appender.index_indexing_slowlog_rolling.policies.time.modulate = true + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2 index b6c8e64e..1721ab2d 100644 --- a/elk_metrics_6x/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/templates/filebeat.yml.j2 @@ -1764,22 +1764,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/heartbeat.yml.j2 b/elk_metrics_6x/templates/heartbeat.yml.j2 index 7e211fd3..f3840670 100644 --- a/elk_metrics_6x/templates/heartbeat.yml.j2 +++ b/elk_metrics_6x/templates/heartbeat.yml.j2 @@ -825,22 +825,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/journalbeat.yml.j2 b/elk_metrics_6x/templates/journalbeat.yml.j2 index 0445a313..00e5e84a 100644 --- a/elk_metrics_6x/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/templates/journalbeat.yml.j2 @@ -631,22 +631,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2 index 8d3b0c9b..fd48861b 100644 --- a/elk_metrics_6x/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/templates/metricbeat.yml.j2 @@ -104,7 +104,7 @@ metricbeat.modules: # Include the cumulative CPU tick values with the process metrics. Defaults # to false. - process.include_cpu_ticks: {{ inventory_hostname == physical_host }} + #process.include_cpu_ticks: false # Configure reverse DNS lookup on remote IP addresses in the socket metricset. socket.reverse_lookup.enabled: true @@ -1085,22 +1085,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %} diff --git a/elk_metrics_6x/templates/packetbeat.yml.j2 b/elk_metrics_6x/templates/packetbeat.yml.j2 index 090fff2e..b2ffc80f 100644 --- a/elk_metrics_6x/templates/packetbeat.yml.j2 +++ b/elk_metrics_6x/templates/packetbeat.yml.j2 @@ -1104,22 +1104,7 @@ setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - index: - # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. - number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} - codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false +{% include 'templates/_include_setup_template.yml.j2' %} #============================== Kibana ===================================== {% include 'templates/_include_kibana_setup.yml.j2' %}