Correct cluster properties

When running a multi-node ELK stack (5+ nodes) the elasticsearch server needs to be able to mark and identify nodes that will be data and master nodes. This change ensures that the first set of available master nodes is mark accordingly as both data and master and that all other nodes are marked as cluster members with alternating nodes being marked as data nodes. With this change the cluster will be able to grow as well as handle large data sets more efficiently. This change ensures that sharding is correctly started when the cluster is deployed and that the beat services are using the recommended sharding practices. Change-Id: Id555132199f268b21aaa939a84760c744238dd2b Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
2018-05-02 15:31:54 -05:00 · 2018-05-02 15:31:54 -05:00 · 6ebca3d78f
commit 6ebca3d78f
parent 3cd55cc3c1
14 changed files with 131 additions and 123 deletions
--- a/elk_metrics_6x/installElastic.yml
+++ b/elk_metrics_6x/installElastic.yml
@ -67,6 +67,8 @@
          dest: /etc/elasticsearch/elasticsearch.yml
        - src: templates/jvm.options.j2
          dest: /etc/elasticsearch/jvm.options
+        - src: templates/es-log4j2.properties.j2
+          dest: /etc/elasticsearch/log4j2.properties
      tags:
        - config

--- a/elk_metrics_6x/installKibana.yml
+++ b/elk_metrics_6x/installKibana.yml
@ -36,7 +36,7 @@
        enabled: true
        state: restarted

-    - name: Ensure kibana  is installed.
+    - name: Ensure kibana  is installed
      apt:
        name: kibana
        state: present
--- a/elk_metrics_6x/installLogstash.yml
+++ b/elk_metrics_6x/installLogstash.yml
@ -52,7 +52,7 @@
        install_recommends: yes
        update_cache: yes

-    - name: Ensure Logstash is installed.
+    - name: Ensure Logstash is installed
      apt:
        name: logstash
        state: present
--- a/elk_metrics_6x/readme.rst
+++ b/elk_metrics_6x/readme.rst
@ -70,6 +70,7 @@ lines.
          haproxy_backend_port: 9200  # This is set using the "elastic_port" variable
          haproxy_balance_type: tcp

+
 Optional | run the haproxy-install playbook
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

--- a/elk_metrics_6x/templates/_include_setup_template.yml.j2
+++ b/elk_metrics_6x/templates/_include_setup_template.yml.j2
@ -0,0 +1,18 @@
+{% set shards = ((groups["elastic-logstash"] | length) * 3) | int %}
+
+# Elasticsearch template settings
+setup.template.settings:
+
+  # A dictionary of settings to place into the settings.index dictionary
+  # of the Elasticsearch template. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+  index:
+    number_of_shards: {{ shards }}
+    codec: best_compression
+    # This provides for an index split of up to 2 times the number of available shards
+    number_of_routing_shards: {{ (shards | int) * 2 * (groups["elastic-logstash"] | length) }}
+
+  # A dictionary of settings for the _source field. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
+  #_source:
+    #enabled: false
--- a/elk_metrics_6x/templates/apm-server.yml.j2
+++ b/elk_metrics_6x/templates/apm-server.yml.j2
@ -275,22 +275,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/auditbeat.yml.j2
+++ b/elk_metrics_6x/templates/auditbeat.yml.j2
@ -716,22 +716,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/elasticsearch.yml.j2
+++ b/elk_metrics_6x/templates/elasticsearch.yml.j2
@ -43,17 +43,24 @@ http.port: {{ elastic_port }}
 #
 {% set IP_ARR=[] %}
 {% for host in groups['elastic-logstash'] %}
-{%   if IP_ARR.insert(loop.index,hostvars[host]['ansible_host']) %}
+{%   if IP_ARR.extend([hostvars[host]['ansible_host'] | string]) %}
 {%   endif %}
 {% endfor %}
-discovery.zen.ping.unicast.hosts: [{{ IP_ARR | join(', ') }}]
-node.master: {{ (inventory_hostname == groups['elastic-logstash'][0]) | ternary(true, false) }}
-node.data: true
+{% set available_nodes = (groups['elastic-logstash'] | length) %}
+{# the master node count takes half the available nodes or sets it's self as 1 #}
+{% set _master_node_count = (available_nodes > 1) | ternary(((available_nodes // 2) | int), 1) %}
+{# if the master node count is even, add one to it otherwise use the provided value #}
+{% set master_node_count = ((_master_node_count | int) % 2 != 0) | ternary(_master_node_count, (_master_node_count + 1)) %}
+
+discovery.zen.ping.unicast.hosts: {{ IP_ARR }}
 #
 # Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
-#
-# discovery.zen.minimum_master_nodes: 3
-#
+discovery.zen.minimum_master_nodes: {{ master_node_count | int }}
+# The first set of nodes in the master_node_count are marked as such
+node.master: {{ (inventory_hostname in groups['elastic-logstash'][:master_node_count | int]) | ternary(true, false) }}
+# Every node in the master list and every other node after will be a data node
+node.data: {{ (inventory_hostname in (groups['elastic-logstash'][:master_node_count| int] + groups['elastic-logstash'][master_node_count | int::2])) | ternary(true, false) }}
+
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
 #
@ -61,7 +68,7 @@ node.data: true
 #
 # Block initial recovery after a full cluster restart until N nodes are started:
 #
-# gateway.recover_after_nodes: 3
+gateway.recover_after_nodes: {{ master_node_count }}
 #
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
--- a/elk_metrics_6x/templates/es-log4j2.properties.j2
+++ b/elk_metrics_6x/templates/es-log4j2.properties.j2
@ -0,0 +1,85 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 128MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.fileIndex = nomax
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
+appender.rolling.strategy.action.condition.nested_condition.age = {{ elastic_logstash_retention }}D
+
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 128MB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 2
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false
--- a/elk_metrics_6x/templates/filebeat.yml.j2
+++ b/elk_metrics_6x/templates/filebeat.yml.j2
@ -1764,22 +1764,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/heartbeat.yml.j2
+++ b/elk_metrics_6x/templates/heartbeat.yml.j2
@ -825,22 +825,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/journalbeat.yml.j2
+++ b/elk_metrics_6x/templates/journalbeat.yml.j2
@ -631,22 +631,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/metricbeat.yml.j2
+++ b/elk_metrics_6x/templates/metricbeat.yml.j2
@ -104,7 +104,7 @@ metricbeat.modules:

  # Include the cumulative CPU tick values with the process metrics. Defaults
  # to false.
-  process.include_cpu_ticks: {{ inventory_hostname == physical_host }}
+  #process.include_cpu_ticks: false

  # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
  socket.reverse_lookup.enabled: true
@ -1085,22 +1085,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}
--- a/elk_metrics_6x/templates/packetbeat.yml.j2
+++ b/elk_metrics_6x/templates/packetbeat.yml.j2
@ -1104,22 +1104,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true

-# Elasticsearch template settings
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
+{% include 'templates/_include_setup_template.yml.j2' %}

 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}