Correct cluster properties

When running a multi-node ELK stack (5+ nodes) the elasticsearch server
needs to be able to mark and identify nodes that will be data and master
nodes. This change ensures that the first set of available master nodes
is mark accordingly as both data and master and that all other nodes are
marked as cluster members with alternating nodes being marked as data
nodes. With this change the cluster will be able to grow as well as
handle large data sets more efficiently.

This change ensures that sharding is correctly started when the cluster
is deployed and that the beat services are using the recommended
sharding practices.

Change-Id: Id555132199f268b21aaa939a84760c744238dd2b
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installElastic.yml

@@ -67,6 +67,8 @@
           dest: /etc/elasticsearch/elasticsearch.yml
         - src: templates/jvm.options.j2
           dest: /etc/elasticsearch/jvm.options
+        - src: templates/es-log4j2.properties.j2
+          dest: /etc/elasticsearch/log4j2.properties
       tags:
         - config
 

elk_metrics_6x/installKibana.yml

@@ -36,7 +36,7 @@
         enabled: true
         state: restarted
 
-    - name: Ensure kibana  is installed.
+    - name: Ensure kibana  is installed
       apt:
         name: kibana
         state: present

elk_metrics_6x/installLogstash.yml

@@ -52,7 +52,7 @@
         install_recommends: yes
         update_cache: yes
 
-    - name: Ensure Logstash is installed.
+    - name: Ensure Logstash is installed
       apt:
         name: logstash
         state: present

elk_metrics_6x/readme.rst

@@ -70,6 +70,7 @@ lines.
           haproxy_backend_port: 9200  # This is set using the "elastic_port" variable
           haproxy_balance_type: tcp
 
+
 Optional | run the haproxy-install playbook
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

elk_metrics_6x/templates/_include_setup_template.yml.j2

@@ -0,0 +1,18 @@
+{% set shards = ((groups["elastic-logstash"] | length) * 3) | int %}
+
+# Elasticsearch template settings
+setup.template.settings:
+
+  # A dictionary of settings to place into the settings.index dictionary
+  # of the Elasticsearch template. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+  index:
+    number_of_shards: {{ shards }}
+    codec: best_compression
+    # This provides for an index split of up to 2 times the number of available shards
+    number_of_routing_shards: {{ (shards | int) * 2 * (groups["elastic-logstash"] | length) }}
+
+  # A dictionary of settings for the _source field. For more details, please check
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
+  #_source:
+    #enabled: false

elk_metrics_6x/templates/apm-server.yml.j2

@@ -275,22 +275,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -716,22 +716,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/elasticsearch.yml.j2

@@ -43,17 +43,24 @@ http.port: {{ elastic_port }}
 #
 {% set IP_ARR=[] %}
 {% for host in groups['elastic-logstash'] %}
-{%   if IP_ARR.insert(loop.index,hostvars[host]['ansible_host']) %}
+{%   if IP_ARR.extend([hostvars[host]['ansible_host'] | string]) %}
 {%   endif %}
 {% endfor %}
-discovery.zen.ping.unicast.hosts: [{{ IP_ARR | join(', ') }}]
+{% set available_nodes = (groups['elastic-logstash'] | length) %}
-node.master: {{ (inventory_hostname == groups['elastic-logstash'][0]) | ternary(true, false) }}
+{# the master node count takes half the available nodes or sets it's self as 1 #}
-node.data: true
+{% set _master_node_count = (available_nodes > 1) | ternary(((available_nodes // 2) | int), 1) %}
+{# if the master node count is even, add one to it otherwise use the provided value #}
+{% set master_node_count = ((_master_node_count | int) % 2 != 0) | ternary(_master_node_count, (_master_node_count + 1)) %}
+
+discovery.zen.ping.unicast.hosts: {{ IP_ARR }}
 #
 # Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
-#
+discovery.zen.minimum_master_nodes: {{ master_node_count | int }}
-# discovery.zen.minimum_master_nodes: 3
+# The first set of nodes in the master_node_count are marked as such
-#
+node.master: {{ (inventory_hostname in groups['elastic-logstash'][:master_node_count | int]) | ternary(true, false) }}
+# Every node in the master list and every other node after will be a data node
+node.data: {{ (inventory_hostname in (groups['elastic-logstash'][:master_node_count| int] + groups['elastic-logstash'][master_node_count | int::2])) | ternary(true, false) }}
+
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
 #
@@ -61,7 +68,7 @@ node.data: true
 #
 # Block initial recovery after a full cluster restart until N nodes are started:
 #
-# gateway.recover_after_nodes: 3
+gateway.recover_after_nodes: {{ master_node_count }}
 #
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>

elk_metrics_6x/templates/es-log4j2.properties.j2

@@ -0,0 +1,85 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 128MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.fileIndex = nomax
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
+appender.rolling.strategy.action.condition.nested_condition.age = {{ elastic_logstash_retention }}D
+
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 128MB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 2
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false

elk_metrics_6x/templates/filebeat.yml.j2

@@ -1764,22 +1764,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -825,22 +825,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -631,22 +631,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -104,7 +104,7 @@ metricbeat.modules:
 
   # Include the cumulative CPU tick values with the process metrics. Defaults
   # to false.
-  process.include_cpu_ticks: {{ inventory_hostname == physical_host }}
+  #process.include_cpu_ticks: false
 
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
   socket.reverse_lookup.enabled: true
@@ -1085,22 +1085,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -1104,22 +1104,7 @@ setup.template.fields: "${path.config}/fields.yml"
 # Overwrite existing template
 setup.template.overwrite: true
 
-# Elasticsearch template settings
+{% include 'templates/_include_setup_template.yml.j2' %}
-setup.template.settings:
-
-  # A dictionary of settings to place into the settings.index dictionary
-  # of the Elasticsearch template. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
-  index:
-    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
-    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
-    codec: best_compression
-    #number_of_routing_shards: 30
-
-  # A dictionary of settings for the _source field. For more details, please check
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
-  #_source:
-    #enabled: false
 
 #============================== Kibana =====================================
 {% include 'templates/_include_kibana_setup.yml.j2' %}