Add curator and dynamic shard counts

Curator has been added to automatically maintain the cluster with
sensible defaults when it pertains to data retention.

The index counts have been modified such that they're determined by the
size of the initial cluster. While these shard counts can be modified
post deployment by reindexing the data, it's not something being done at
this time.

Depends-On: https://review.openstack.org/c/565807
Change-Id: I249d715ae5241ab57c4117b14377e4d07cb6e984
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installCurator.yml

@@ -0,0 +1,101 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install Curator
+  hosts: "elastic-logstash"
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  pre_tasks:
+    - name: Ensure curator is installed
+      pip:
+        name: elasticsearch-curator
+        state: "{{ elk_package_state | default('present') }}"
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
+
+  tasks:
+    - name: create the system group
+      group:
+        name: "curator"
+        state: "present"
+        system: "yes"
+
+    - name: Create the curator system user
+      user:
+        name: "curator"
+        group: "curator"
+        comment: "curator user"
+        shell: "/bin/false"
+        createhome: "yes"
+        home: "/var/lib/curator"
+
+    - name: Create curator data path
+      file:
+        path: "{{ item }}"
+        state: directory
+        owner: "curator"
+        group: "curator"
+        mode: "0755"
+        recurse: true
+      with_items:
+        - "/var/lib/curator"
+        - "/var/log/curator"
+        - "/etc/curator"
+
+    - name: Drop curator conf file
+      template:
+        src: templates/curator.yml.j2
+        dest: /var/lib/curator/curator.yml
+
+    - name: Drop curator action file
+      template:
+        src: templates/curator-actions.yml.j2
+        dest: /var/lib/curator/actions.yml
+
+  post_tasks:
+    - name: Run the systemd service role
+      include_role:
+        name: systemd_service
+        private: true
+      vars:
+        systemd_service_enabled: true
+        systemd_user_name: curator
+        systemd_group_name: curator
+        systemd_services:
+          - service_name: "curator"
+            execstarts:
+              - /usr/local/bin/curator
+                --config /var/lib/curator/curator.yml
+                /var/lib/curator/actions.yml
+            timer:
+              state: "started"
+              options:
+                OnBootSec: 30min
+                OnUnitActiveSec: 48h
+                Persistent: true
+
+    - name: Enable and restart curator.timer
+      systemd:
+        name: "curator.timer"
+        enabled: true
+        state: restarted

elk_metrics_6x/installJournalbeat.yml

@@ -49,7 +49,7 @@
         state: "present"
         system: "yes"
 
-    - name: Create the nova system user
+    - name: Create the journalbeat user
       user:
         name: "journalbeat"
         group: "journalbeat"

elk_metrics_6x/templates/apm-server.yml.j2

@@ -282,7 +282,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -723,7 +723,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/curator-actions.yml.j2

@@ -0,0 +1,168 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices older than 60 days (based on index name), for logstash-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_logstash_retention }}
+  2:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for apm-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: apm-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_apm_retention }}
+  3:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for auditbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: auditbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_auditbeat_retention }}
+  4:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for filebeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: filebeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_filebeat_retention }}
+  5:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for heartbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: heartbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_heartbeat_retention }}
+  6:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for journalbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: journalbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_journalbeat_retention }}
+  7:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for metricbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: metricbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_metricbeat_retention }}
+  8:
+    action: delete_indices
+    description: >-
+      Delete indices older than 5 days (based on index name), for packetbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: packetbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_packetbeat_retention }}

elk_metrics_6x/templates/curator.yml.j2

@@ -0,0 +1,32 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+client:
+  hosts:
+    - {{ ansible_host }}
+  port: 9200
+  url_prefix: ""
+  use_ssl: false
+  ssl_no_validate: true
+  http_auth: ""
+  timeout: 120
+  master_only: true
+
+logging:
+  loglevel: INFO
+  logfile: /var/log/curator/curator
+  logformat: default
+  blacklist:
+    - elasticsearch
+    - urllib3

elk_metrics_6x/templates/filebeat.yml.j2

@@ -1771,7 +1771,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -832,7 +832,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -638,7 +638,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -1092,7 +1092,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -1111,7 +1111,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/vars/variables.yml

@@ -4,6 +4,16 @@ elastic_hap_port: 9201
 cluster_name: openstack_elk
 node_name: ${HOSTNAME}
 
+# elastic curator vars
+# all retention options are in days
+elastic_logstash_retention: 28
+elastic_apm_retention: 14
+elastic_auditbeat_retention: 14
+elastic_filebeat_retention: 14
+elastic_heartbeat_retention: 7
+elastic_journalbeat_retention: 14
+elastic_metricbeat_retention: 14
+elastic_packetbeat_retention: 7
 
 # kibana vars
 kibana_interface: 0.0.0.0