From 0c41b0fd70ddd1c76dae6240b2e8c9f44fe2e656 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 2 May 2018 01:11:41 -0500 Subject: [PATCH] Add curator and dynamic shard counts Curator has been added to automatically maintain the cluster with sensible defaults when it pertains to data retention. The index counts have been modified such that they're determined by the size of the initial cluster. While these shard counts can be modified post deployment by reindexing the data, it's not something being done at this time. Depends-On: https://review.openstack.org/c/565807 Change-Id: I249d715ae5241ab57c4117b14377e4d07cb6e984 Signed-off-by: Kevin Carter --- elk_metrics_6x/installCurator.yml | 101 +++++++++++ elk_metrics_6x/installJournalbeat.yml | 2 +- elk_metrics_6x/templates/apm-server.yml.j2 | 3 +- elk_metrics_6x/templates/auditbeat.yml.j2 | 3 +- .../templates/curator-actions.yml.j2 | 168 ++++++++++++++++++ elk_metrics_6x/templates/curator.yml.j2 | 32 ++++ elk_metrics_6x/templates/filebeat.yml.j2 | 3 +- elk_metrics_6x/templates/heartbeat.yml.j2 | 3 +- elk_metrics_6x/templates/journalbeat.yml.j2 | 3 +- elk_metrics_6x/templates/metricbeat.yml.j2 | 3 +- elk_metrics_6x/templates/packetbeat.yml.j2 | 3 +- elk_metrics_6x/vars/variables.yml | 10 ++ 12 files changed, 326 insertions(+), 8 deletions(-) create mode 100644 elk_metrics_6x/installCurator.yml create mode 100644 elk_metrics_6x/templates/curator-actions.yml.j2 create mode 100644 elk_metrics_6x/templates/curator.yml.j2 diff --git a/elk_metrics_6x/installCurator.yml b/elk_metrics_6x/installCurator.yml new file mode 100644 index 00000000..c6ed3fce --- /dev/null +++ b/elk_metrics_6x/installCurator.yml @@ -0,0 +1,101 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Curator + hosts: "elastic-logstash" + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + pre_tasks: + - name: Ensure curator is installed + pip: + name: elasticsearch-curator + state: "{{ elk_package_state | default('present') }}" + + - name: exit playbook after uninstall + meta: end_play + when: + - elk_package_state | default('present') == 'absent' + + tasks: + - name: create the system group + group: + name: "curator" + state: "present" + system: "yes" + + - name: Create the curator system user + user: + name: "curator" + group: "curator" + comment: "curator user" + shell: "/bin/false" + createhome: "yes" + home: "/var/lib/curator" + + - name: Create curator data path + file: + path: "{{ item }}" + state: directory + owner: "curator" + group: "curator" + mode: "0755" + recurse: true + with_items: + - "/var/lib/curator" + - "/var/log/curator" + - "/etc/curator" + + - name: Drop curator conf file + template: + src: templates/curator.yml.j2 + dest: /var/lib/curator/curator.yml + + - name: Drop curator action file + template: + src: templates/curator-actions.yml.j2 + dest: /var/lib/curator/actions.yml + + post_tasks: + - name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_enabled: true + systemd_user_name: curator + systemd_group_name: curator + systemd_services: + - service_name: "curator" + execstarts: + - /usr/local/bin/curator + --config /var/lib/curator/curator.yml + /var/lib/curator/actions.yml + timer: + state: "started" + options: + OnBootSec: 30min + OnUnitActiveSec: 48h + Persistent: true + + - name: Enable and restart curator.timer + systemd: + name: "curator.timer" + enabled: true + state: restarted diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index e47947e8..6f895831 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -49,7 +49,7 @@ state: "present" system: "yes" - - name: Create the nova system user + - name: Create the journalbeat user user: name: "journalbeat" group: "journalbeat" diff --git a/elk_metrics_6x/templates/apm-server.yml.j2 b/elk_metrics_6x/templates/apm-server.yml.j2 index 1e32624e..78751537 100644 --- a/elk_metrics_6x/templates/apm-server.yml.j2 +++ b/elk_metrics_6x/templates/apm-server.yml.j2 @@ -282,7 +282,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/auditbeat.yml.j2 b/elk_metrics_6x/templates/auditbeat.yml.j2 index 7b560200..6e1de8e8 100644 --- a/elk_metrics_6x/templates/auditbeat.yml.j2 +++ b/elk_metrics_6x/templates/auditbeat.yml.j2 @@ -723,7 +723,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/curator-actions.yml.j2 b/elk_metrics_6x/templates/curator-actions.yml.j2 new file mode 100644 index 00000000..e8791b14 --- /dev/null +++ b/elk_metrics_6x/templates/curator-actions.yml.j2 @@ -0,0 +1,168 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +actions: + 1: + action: delete_indices + description: >- + Delete indices older than 60 days (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_logstash_retention }} + 2: + action: delete_indices + description: >- + Delete indices older than 10 days (based on index name), for apm- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: apm- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_apm_retention }} + 3: + action: delete_indices + description: >- + Delete indices older than 15 days (based on index name), for auditbeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: auditbeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_auditbeat_retention }} + 4: + action: delete_indices + description: >- + Delete indices older than 15 days (based on index name), for filebeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: filebeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_filebeat_retention }} + 5: + action: delete_indices + description: >- + Delete indices older than 10 days (based on index name), for heartbeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: heartbeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_heartbeat_retention }} + 6: + action: delete_indices + description: >- + Delete indices older than 15 days (based on index name), for journalbeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: journalbeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_journalbeat_retention }} + 7: + action: delete_indices + description: >- + Delete indices older than 10 days (based on index name), for metricbeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: metricbeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_metricbeat_retention }} + 8: + action: delete_indices + description: >- + Delete indices older than 5 days (based on index name), for packetbeat- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: packetbeat- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ elastic_packetbeat_retention }} diff --git a/elk_metrics_6x/templates/curator.yml.j2 b/elk_metrics_6x/templates/curator.yml.j2 new file mode 100644 index 00000000..50a66299 --- /dev/null +++ b/elk_metrics_6x/templates/curator.yml.j2 @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +client: + hosts: + - {{ ansible_host }} + port: 9200 + url_prefix: "" + use_ssl: false + ssl_no_validate: true + http_auth: "" + timeout: 120 + master_only: true + +logging: + loglevel: INFO + logfile: /var/log/curator/curator + logformat: default + blacklist: + - elasticsearch + - urllib3 diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2 index 8b7696b9..b6c8e64e 100644 --- a/elk_metrics_6x/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/templates/filebeat.yml.j2 @@ -1771,7 +1771,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/heartbeat.yml.j2 b/elk_metrics_6x/templates/heartbeat.yml.j2 index 0973b06d..7e211fd3 100644 --- a/elk_metrics_6x/templates/heartbeat.yml.j2 +++ b/elk_metrics_6x/templates/heartbeat.yml.j2 @@ -832,7 +832,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/journalbeat.yml.j2 b/elk_metrics_6x/templates/journalbeat.yml.j2 index 4ca71f34..0445a313 100644 --- a/elk_metrics_6x/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/templates/journalbeat.yml.j2 @@ -638,7 +638,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2 index 3b4377f1..8d3b0c9b 100644 --- a/elk_metrics_6x/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/templates/metricbeat.yml.j2 @@ -1092,7 +1092,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/templates/packetbeat.yml.j2 b/elk_metrics_6x/templates/packetbeat.yml.j2 index 7b1ce5f6..090fff2e 100644 --- a/elk_metrics_6x/templates/packetbeat.yml.j2 +++ b/elk_metrics_6x/templates/packetbeat.yml.j2 @@ -1111,7 +1111,8 @@ setup.template.settings: # of the Elasticsearch template. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html index: - number_of_shards: 3 + # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number. + number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }} codec: best_compression #number_of_routing_shards: 30 diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index bef7d976..de4f75cf 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -4,6 +4,16 @@ elastic_hap_port: 9201 cluster_name: openstack_elk node_name: ${HOSTNAME} +# elastic curator vars +# all retention options are in days +elastic_logstash_retention: 28 +elastic_apm_retention: 14 +elastic_auditbeat_retention: 14 +elastic_filebeat_retention: 14 +elastic_heartbeat_retention: 7 +elastic_journalbeat_retention: 14 +elastic_metricbeat_retention: 14 +elastic_packetbeat_retention: 7 # kibana vars kibana_interface: 0.0.0.0