modify fact gathering to use local facts

This change makes the retention gathering operation faster by storing
the retention values as "local facts". The local facts are then
referenced in templates loading from the local fact file instead of
running repetitive queries which are slow making very large deployments
cumbersome. To make the retention policy fact gathering process smarter
it will now automatically refresh if undefined or should the
elasticsearch cluster size change. This will ensure we're improving
speed of execution while also catering to the needs of deployers to
grow, or shrink, elasticsearch cluster sizes.

Documentation has been added regarding the new option and why it may be
of use to depoyers.

Change-Id: I3936ee94461ac39fb8bc78dc2c873e6067552461
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/README.rst

@@ -570,6 +570,22 @@ state variable, `elk_package_state`, to latest.
     ansible-playbook site.yml $USER_VARS -e 'elk_package_state="latest"'
 
 
+Forcing the Elasticsearch cluster retention policy to refresh
+-------------------------------------------------------------
+
+To force the cluster retention policy to refresh set `elastic_retention_refresh`, to
+"yes". When setting `elastic_retention_refresh` to "yes" the retention policy will forcibly
+be refresh across all hosts. This option should only be used when the Elasticsearch storage
+array is modified on an existing cluster. Should the Elasticseach cluster size change
+(nodes added or removed) the retention policy will automatically be refreshed on playbook
+execution.
+
+.. code-block:: bash
+
+    cd /opt/openstack-ansible-ops/elk_metrics_6x
+    ansible-playbook site.yml $USER_VARS -e 'elastic_retention_refresh="yes"'
+
+
 Trouble shooting
 ----------------
 

elk_metrics_6x/roles/elastic_curator/tasks/main.yml

@@ -30,6 +30,13 @@
   when:
       ansible_service_mgr != 'systemd'
 
+- name: Refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
+  tags:
+    - always
+
 - name: Ensure virtualenv is installed
   package:
     name: "{{ curator_distro_packages }}"

elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2

@@ -15,10 +15,10 @@
 
 {% set action_items = [] -%}
 {# Delete index loop #}
-{% for key in elastic_beat_retention_policy_keys -%}
+{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%}
 {%   set delete_indices = {} -%}
 {# Total retention size in days #}
-{%   set _index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%}
+{%   set _index_retention = ansible_local['elastic']['retention']['elastic_' + key + '_retention'] -%}
 {%   set index_retention = ((_index_retention | int) > 0) | ternary(_index_retention, 1) | int %}
 {%   set _ = delete_indices.update(
    {

elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2

@@ -15,10 +15,10 @@
 
 {% set action_items = [] -%}
 {# Delete index loop #}
-{% for key in elastic_beat_retention_policy_keys -%}
+{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%}
 {%   set delete_indices = {} -%}
 {# Total retention size in gigabytes #}
-{%   set _index_size = ((hostvars[inventory_hostname]['elastic_' + key + '_size'] | int) // 1024) -%}
+{%   set _index_size = ((ansible_local['elastic']['retention']['elastic_' + key + '_size'] | int) // 1024) -%}
 {%   set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %}
 {%   set _ = delete_indices.update(
    {

elk_metrics_6x/roles/elastic_retention/defaults/main.yml

@@ -94,3 +94,6 @@ elastic_beat_retention_policy_hosts:
   packetbeat:
     weight: 1
     hosts: "{{ groups['hosts'] | default([]) }}"
+
+# Refresh the elasticsearch retention policy local facts.
+elastic_retention_refresh: false

elk_metrics_6x/roles/elastic_retention/tasks/main.yml

@@ -13,40 +13,90 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Query es storage
-  uri:
-    url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs"
-    method: GET
-  register: elk_data
-  environment:
-    no_proxy: "{{ coordination_nodes[0].split(':')[0] }}"
-  until:
-    - elk_data is success and elk_data['json'] is defined
-  retries: 5
-  delay: 30
-  run_once: true
+- name: Ensure local facts directory exists
+  file:
+    dest: "/etc/ansible/facts.d"
+    state: directory
+    group: "root"
+    owner: "root"
+    mode:  "0755"
+    recurse: no
 
-- name: Set retention keys fact
-  set_fact:
-    es_storage_json: "{{ elk_data['json'] }}"
+- name: Initialize local facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/elastic.fact"
+    section: "retention"
+    option: cacheable
+    value: true
 
-- name: Load retention algo variables
-  include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml"
+- name: Refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
   tags:
     - always
 
-- name: Set retention facts (mb size)
-  set_fact: "elastic_{{ item.key }}_size={{ item.value }}"
-  when:
-    - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_size"] is undefined
-  with_dict: "{{ es_storage_per_index }}"
+- name: Retention storage block
+  block:
+    - name: Query es storage
+      uri:
+        url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs"
+        method: GET
+      register: elk_data
+      environment:
+        no_proxy: "{{ coordination_nodes[0].split(':')[0] }}"
+      until:
+        - elk_data is success and elk_data['json'] is defined
+      retries: 5
+      delay: 30
+      run_once: true
 
-- name: Set retention facts (days)
-  set_fact: "elastic_{{ item.key }}_retention={{ item.value }}"
-  when:
-    - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_retention"] is undefined
-  with_dict: "{{ es_days_per_index }}"
+    - name: Set retention keys fact
+      set_fact:
+        es_storage_json: "{{ elk_data['json'] }}"
 
-- name: Set retention keys fact
-  set_fact:
-    elastic_beat_retention_policy_keys: "{{ elastic_beat_retention_policy_hosts.keys() | list }}"
+    - name: Load retention algo variables
+      include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml"
+      tags:
+        - always
+
+    - name: Set storage fact
+      ini_file:
+        dest: "/etc/ansible/facts.d/elastic.fact"
+        section: "retention"
+        option: "cluster_nodes"
+        value: "{{ groups['elastic-logstash'] | length }}"
+
+    - name: Set retention policy fact
+      ini_file:
+        dest: "/etc/ansible/facts.d/elastic.fact"
+        section: "retention"
+        option: "elastic_beat_retention_policy_keys"
+        value: "{{ elastic_beat_retention_policy_hosts.keys() | list }}"
+
+    - name: Set size fact
+      ini_file:
+        dest: "/etc/ansible/facts.d/elastic.fact"
+        section: "retention"
+        option: "elastic_{{ item.key }}_size"
+        value: "{{ item.value }}"
+      with_dict: "{{ es_storage_per_index }}"
+
+    - name: Set retention fact
+      ini_file:
+        dest: "/etc/ansible/facts.d/elastic.fact"
+        section: "retention"
+        option: "elastic_{{ item.key }}_retention"
+        value: "{{ item.value }}"
+      with_dict: "{{ es_days_per_index }}"
+
+    - name: Refresh local facts
+      setup:
+        filter: ansible_local
+        gather_subset: "!all"
+      tags:
+        - always
+  when:
+      - (ansible_local['elastic']['retention']['cluster_nodes'] is undefined) or
+        ((groups['elastic-logstash'] | length) != (ansible_local['elastic']['retention']['cluster_nodes'] | int)) or
+        (elastic_retention_refresh | bool)

elk_metrics_6x/roles/elastic_rollup/tasks/main.yml

@@ -33,6 +33,7 @@
       OK with the old rollup job being purged, add the following option
       `elastic_allow_rollup_purge=yes` to the command line and rerun the
       playbook.
+  run_once: true
   when:
     - check_rollup['json']['jobs'] | length > 0
     - not elastic_allow_rollup_purge | bool
@@ -44,7 +45,7 @@
         min_days_until_rollup: |-
           {% set index_retention = [] %}
           {% for item in ansible_play_hosts %}
-          {%   set _ = index_retention.append(hostvars[item]['elastic_' + index_name + '_retention'] | int) %}
+          {%   set _ = index_retention.append(ansible_local['elastic']['retention']['elastic_' + index_name + '_retention'] | int) %}
           {% endfor %}
           {{ index_retention | min }}
       run_once: true