Update rollback plan and configs

* Added options for the rollback plan so that if a rollback is executed
  all beat packages will be removed.

* additional updates to streamline elk and fix container bindmounts,
  the  use of group information for metric and heartbeat information.

* Readme information has been fixed

Change-Id: Icd070259db5b19d289d10033b1f055125f56e18c
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/common_task_install_elk_repo.yml

@@ -16,7 +16,7 @@
 - name: add Elastic search public GPG key (same for Metricsbeat)
   apt_key:
     url: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
-    state: "present"
+    state: "{{ elk_package_state | default('present') }}"
 
 - name: enable apt-transport-https
   apt:
@@ -27,4 +27,4 @@
 - name: add metricsbeat repo to apt sources list
   apt_repository:
     repo: 'deb https://artifacts.elastic.co/packages/6.x/apt stable main'
-    state: present
+    state: "{{ elk_package_state | default('present') }}"

elk_metrics_6x/installAPMserver.yml

@@ -14,10 +14,17 @@
     - name: Ensure apm-server is installed
       apt:
         name: "{{ item }}"
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
       with_items:
         - apm-server
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   post_tasks:
     - name: Drop apm-server conf file

elk_metrics_6x/installAuditbeat.yml

@@ -14,11 +14,18 @@
     - name: Ensure Auditbeat is installed
       apt:
         name: "{{ item }}"
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
       with_items:
         - audispd-plugins
         - auditbeat
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   post_tasks:
     - name: Drop auditbeat conf file

elk_metrics_6x/installElastic.yml

@@ -15,14 +15,19 @@
       tags:
         - sysctl
 
+    - name: Ensure mount directories exists
+      file:
+        path: "/openstack/{{ inventory_hostname }}/elasticsearch"
+        state: "directory"
+      delegate_to: "{{ physical_host }}"
+
     - name: elasticsearch datapath bind mount
       lxc_container:
         name: "{{ inventory_hostname }}"
         container_command: |
           [[ ! -d "/var/lib/elasticsearch" ]] && mkdir -p "/var/lib/elasticsearch"
-          [[ ! -d "/var/lib/elasticsearch-olddata" ]] && mkdir -p "/var/lib/elasticsearch-olddata"
         container_config:
-          - "lxc.mount.entry=/openstack/{{ inventory_hostname }} var/lib/elasticsearch none bind 0 0"
+          - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/elasticsearch var/lib/elasticsearch none bind 0 0"
       delegate_to: "{{ physical_host }}"
       when:
         - physical_host != inventory_hostname
@@ -48,7 +53,7 @@
         src: templates/elasticsearch.yml.j2
         dest: /etc/elasticsearch/elasticsearch.yml
       tags:
-          - config
+        - config
 
     - name: Ensure elasticsearch ownership
       file:
@@ -56,6 +61,8 @@
         owner: elasticsearch
         group: elasticsearch
         recurse: true
+      tags:
+        - config
 
     - name: Enable and restart elastic
       systemd:
@@ -63,4 +70,4 @@
         enabled: true
         state: restarted
       tags:
-          - config
+        - config

elk_metrics_6x/installFilebeat.yml

@@ -14,10 +14,17 @@
     - name: Ensure Filebeat is installed
       apt:
         name: "{{ item }}"
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
       with_items:
         - filebeat
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   tasks:
     - name: Check for apache

elk_metrics_6x/installHeartbeat.yml

@@ -14,8 +14,15 @@
     - name: Ensure heartbeat is installed
       apt:
         name: "heartbeat-elastic"
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   post_tasks:
     - name: Drop heartbeat conf file

elk_metrics_6x/installLogstash.yml

@@ -17,13 +17,19 @@
       tags:
         - sysctl
 
+    - name: Ensure mount directories exists
+      file:
+        path: "/openstack/{{ inventory_hostname }}/logstash"
+        state: "directory"
+      delegate_to: "{{ physical_host }}"
+
     - name: logstash datapath bind mount
       lxc_container:
         name: "{{ inventory_hostname }}"
         container_command: |
           [[ ! -d "/var/lib/logstash" ]] && mkdir -p "/var/lib/logstash"
         container_config:
-          - "lxc.mount.entry=/openstack/{{ inventory_hostname }} var/lib/logstash none bind 0 0"
+          - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/logstash var/lib/logstash none bind 0 0"
       delegate_to: "{{ physical_host }}"
       when:
         - physical_host != inventory_hostname

elk_metrics_6x/installMetricbeat.yml

@@ -14,8 +14,15 @@
     - name: Ensure Metricsbeat is installed
       apt:
         name: metricbeat
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   tasks:
     - name: Check for apache

elk_metrics_6x/installPacketbeat.yml

@@ -14,11 +14,18 @@
     - name: Ensure packetbeat is installed
       apt:
         name: "{{ item }}"
-        state: present
+        state: "{{ elk_package_state | default('present') }}"
         update_cache: true
       with_items:
         - tcpdump
         - packetbeat
+      tags:
+        - package_install
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
 
   post_tasks:
     - name: Drop packetbeat conf file

elk_metrics_6x/readme.rst

@@ -154,4 +154,5 @@ If everything goes bad, you can clean up with the following command
 
 .. code-block:: bash
 
-     openstack-ansible lxc-containers-destroy.yml --limit=kibana:elastic-logstash_all
+     openstack-ansible /opt/openstack-ansible-ops/elk_metrics_6x/site.yml -e "elk_package_state=absent" --tags package_install
+     openstack-ansible /opt/openstack-ansible/playbooks/lxc-containers-destroy.yml --limit=kibana:elastic-logstash_all

elk_metrics_6x/site.yml

@@ -16,7 +16,10 @@
 - import_playbook: installElastic.yml
 - import_playbook: installLogstash.yml
 - import_playbook: installKibana.yml
+- import_playbook: installAPMserver.yml
+
 - import_playbook: installMetricbeat.yml
 - import_playbook: installPacketbeat.yml
 - import_playbook: installAuditbeat.yml
 - import_playbook: installHeartbeat.yml
+- import_playbook: installFilebeat.yml

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -64,28 +64,21 @@ heartbeat.monitors:
   # sub-dictionary. Default is false.
   #fields_under_root: false
 
-{% set tcp_hosts = [] %}
-{% set http_hosts = [] %}
-{% set haproxy_host = hostvars[groups['haproxy_all'][0]] %}
-{% for item in haproxy_host['haproxy_default_services'] + haproxy_extra_services | default([]) %}
-{%   set item_service = item['service'] %}
-{%   for backend in item_service['haproxy_backend_nodes'] + item_service['haproxy_backup_nodes'] | default([]) %}
-{%     set backend_host = hostvars[backend]['ansible_host'] %}
-{%     set port = item_service['haproxy_check_port'] | default(item_service['haproxy_port']) %}
-{%     if not '{{' in backend_host and not '{{' in (port | string) %}
-{%       if item_service['haproxy_balance_type'] | default('tcp') == 'tcp' %}
-{%         set _ = tcp_hosts.extend([backend_host + ":" + (port | string)]) %}
-{%       elif item_service['haproxy_balance_type'] | default('tcp') == 'http' %}
-{%         set _ = http_hosts.extend(["http://" + backend_host + ":" + (port | string) + "/"]) %}
-{%       endif %}
-{%     endif %}
-{%   endfor %}
-{% endfor %}
+{% for item in heartbeat_services %}
+{%   if inventory_hostname in groups['utility_all'] | default([]) %}
+{%     if item.type == 'tcp' %}
+{%       set hosts = [] %}
+{%       for port in item.ports | default([])  %}
+{%         for backend in item.group | default([])  %}
+{%           set backend_host = hostvars[backend]['ansible_host'] %}
+{%           set _ = hosts.extend([backend_host + ":" + (port | string)]) %}
+{%         endfor %}
+{%       endfor %}
 - type: tcp # monitor type `tcp`. Connect via TCP and optionally verify endpoint
             # by sending/receiving a custom payload
 
   # Monitor name used for job name and document type
-  #name: tcp
+  name: "{{ item.name }}"
 
   # Enable/Disable monitor
   #enabled: true
@@ -109,7 +102,7 @@ heartbeat.monitors:
   #     Using `tls`/`ssl`, an SSL connection is established. If no ssl is configured,
   #     system defaults will be used (not supported on windows).
   #     If `port` is missing in url, the ports setting is required.
-  hosts: [{{ tcp_hosts | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' ) }}]
+  hosts: [{{ hosts | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' ) }}]
 
   # Configure IP protocol types to ping on if hostnames are configured.
   # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`.
@@ -153,11 +146,18 @@ heartbeat.monitors:
 
     # Required TLS protocols
     #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"]
-
+{%     elif item.type == 'http' %}
+{%       set hosts = [] %}
+{%       for port in item.ports | default([])  %}
+{%         for backend in item.group | default([])  %}
+{%           set backend_host = hostvars[backend]['ansible_host'] %}
+{%           set _ = hosts.extend(["http://" + backend_host + ":" + (port | string) + item.path]) %}
+{%         endfor %}
+{%       endfor %}
 - type: http # monitor type `http`. Connect via HTTP an optionally verify response
 
   # Monitor name used for job name and document type
-  #name: http
+  name: "{{ item.name }}"
 
   # Enable/Disable monitor
   #enabled: true
@@ -166,13 +166,13 @@ heartbeat.monitors:
   schedule: '@every 30s' # every 30 seconds from start of beat
 
   # Configure URLs to ping
-  urls: [{{ http_hosts | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' ) }}]
+  urls: [{{ hosts | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' ) }}]
 
   # Configure IP protocol types to ping on if hostnames are configured.
   # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`.
   ipv4: true
   ipv6: true
-  mode: any
+  mode: "any"
 
   # Configure file json file to be watched for changes to the monitor:
   #watch.poll_file:
@@ -204,10 +204,11 @@ heartbeat.monitors:
   # Request settings:
   check.request:
     # Configure HTTP method to use. Only 'HEAD', 'GET' and 'POST' methods are allowed.
-    method: "HEAD"
+    method: "{{ item.method }}"
 
     # Dictionary of additional HTTP headers to send:
-    #headers:
+    headers:
+      User-agent: osa-heartbeat-healthcheck
 
     # Optional request body content
     #body:
@@ -223,6 +224,9 @@ heartbeat.monitors:
 
     # Required response contents.
     #body:
+{%     endif %}
+{%   endif %}
+{% endfor %}
 
 heartbeat.scheduler:
   # Limit number of concurrent tasks executed by heartbeat. The task limit if

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -229,27 +229,7 @@ metricbeat.modules:
 #  #response.enabled: false
 #  #dedot.enabled: false
 #
-{% if inventory_hostname in groups['utility_all'] | default([]) %}
-{% set haproxy_host = hostvars[groups['haproxy_all'][0]] %}
-{%   for item in haproxy_host['haproxy_default_services'] + haproxy_extra_services | default([]) %}
-{%     set item_service = item['service'] %}
-{%     for backend in item_service['haproxy_backend_nodes'] + item_service['haproxy_backup_nodes'] | default([]) %}
-{%       set backend_host = hostvars[backend]['ansible_host'] %}
-{%       set port = item_service['haproxy_check_port'] | default(item_service['haproxy_port']) %}
-{%       if not '{{' in backend_host and not '{{' in (port | string) %}
-- module: http
-  metricsets: ["server"]
-  host: "{{ backend_host }}"
-  port: {{ port | int }}
-  enabled: true
-  method: "{{ (item_service['haproxy_backend_options'] | default(['check', 'HEAD', '/']))[0].split()[1] | default('GET') }}"
-  path: "{{ (item_service['haproxy_backend_options'] | default(['check', 'HEAD', '/']))[0].split()[2] | default('/') }}"
 
-{%       endif %}
-{%     endfor %}
-{%   endfor %}
-{% endif %}
-#
 ##------------------------------- Jolokia Module ------------------------------
 #- module: jolokia
 #  metricsets: ["jmx"]
@@ -443,12 +423,12 @@ metricbeat.modules:
 - module: rabbitmq
   metricsets: ["node", "queue"]
   period: 10s
-  hosts: ["localhost:5672", "localhost:5671"]
-#
+  hosts: ["localhost:5672", "localhost:5671", "localhost:15672", "localhost:15671"]
   username: {{ rabbitmq_monitoring_userid | default('monitoring') }}
   password: {{ rabbitmq_monitoring_password  }}
 #
 {% endif %}
+
 ##-------------------------------- Redis Module -------------------------------
 #- module: redis
 #  metricsets: ["info", "keyspace"]

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -167,19 +167,20 @@ packetbeat.protocols:
 
 - type: http
   # Enable HTTP monitoring. Default: true
-{% set ns = namespace(enabled=(inventory_hostname in groups['shared-infra_hosts'] | default([]))) %}
-{% if not ns.enabled | bool %}
-{%   for _item in groups['shared-infra_hosts'] | default([]) %}
-{%     if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
-{%       set ns.enabled = true %}
+{% set used_ports = [53, 443, 2049, 3306, 5432, 5672, 6379, 9042, 9090, 11211, 27017] %}
+{% set ports = [] %}
+{% for item in heartbeat_services %}
+{%   for port in item.ports %}
+{%     if (item.type == 'http') and (not port in used_ports) %}
+{%       set _ = ports.extend([port]) %}
 {%     endif %}
 {%   endfor %}
-{% endif %}
-  enabled: {{ ns.enabled }}
+{% endfor %}
+  enabled: true
 
   # Configure the ports where to listen for HTTP traffic. You can disable
   # the HTTP protocol by commenting out the list of ports.
-  ports: [80, 81, 5000, 6385, 8000, 8002, 8004, 8041, 8042, 8080, 8180, 8181, 8185, 8386, 8774, 8775, 8776, 8779, 8780, 9191, 9201, 9292, 9311, 9511, 9696, 9876, 9890, 15672, 35357]
+  ports: {{ ports | unique }}
 
   # Uncomment the following to hide certain parameters in URL or forms attached
   # to HTTP requests. The names of the parameters are case insensitive.
@@ -482,7 +483,7 @@ packetbeat.protocols:
 
   # If this option is enabled, the client and server certificates and
   # certificate chains are sent to Elasticsearch. The default is true.
-  #send_certificates: true
+  send_certificates: true
 
   # If this option is enabled, the raw certificates will be stored
   # in PEM format under the `raw` key. The default is false.

elk_metrics_6x/vars/variables.yml

@@ -18,6 +18,240 @@ kibana_server_name: "{{ ansible_hostname }}"
 logstash_beat_input_port: 5044
 
 
+# Beat options
+heartbeat_services:
+  - group: "{{ groups['galera_all'] }}"
+    name: galera HTTP
+    ports:
+      - 9200
+    type: tcp
+  - group: "{{ groups['galera_all'] }}"
+    name: galera TCP
+    ports:
+      - 3306
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['repo_all'] }}"
+    name: repo git
+    ports:
+      - 9418
+    type: tcp
+  - group: "{{ groups['repo_all'] }}"
+    name: repo server
+    ports:
+      - 8181
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['repo_all'] }}"
+    name: repo cache
+    ports:
+      - 3142
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['repo_all'] }}"
+    name: repo acng
+    ports:
+      - 80
+    type: http
+    method: HEAD
+    path: "/acng-report.html"
+  - group: "{{ groups['glance_api'] }}"
+    name: glance api
+    ports:
+      - 9292
+    type: http
+    method: HEAD
+    path: "/healthcheck"
+  - group: "{{ groups['glance_api'] }}"
+    name: glance registry
+    ports:
+      - 9191
+    type: http
+    method: HEAD
+    path: "/healthcheck"
+  - group: "{{ groups['gnocchi_all'] }}"
+    name: gnocchi api
+    ports:
+      - 8041
+    type: http
+    method: HEAD
+    path: "/healthcheck"
+  - group: "{{ groups['heat_api_cfn'] }}"
+    name: heat cfn api
+    ports:
+      - 8000
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['heat_api'] }}"
+    name: heat api
+    ports:
+      - 8004
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['keystone_all'] }}"
+    name: keystone api
+    ports:
+      - 5000
+      - 35357
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['neutron_server'] }}"
+    name: neutron server
+    ports:
+      - 9696
+    type: http
+    method: GET
+    path: "/"
+  - group: "{{ groups['nova_api_metadata'] }}"
+    name: nova api metadata
+    ports:
+      - 8775
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['nova_api_os_compute'] }}"
+    name: nova api compute
+    ports:
+      - 8774
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['nova_api_placement'] }}"
+    name: nova api placement
+    ports:
+      - 8780
+    type: http
+    method: GET
+    path: "/"
+  - group: "{{ groups['nova_console'] }}"
+    name: nova console
+    ports:
+      - 6080
+      - 6082
+      - 6083
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['cinder_api'] }}"
+    name: cinder api
+    ports:
+      - 8776
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['horizon_all'] }}"
+    name: horizon
+    ports:
+      - 80
+      - 443
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['sahara_api'] }}"
+    name: sahara api
+    ports:
+      - 8386
+    type: http
+    method: HEAD
+    path: "/healthcheck"
+  - group: "{{ groups['swift_proxy'] }}"
+    name: swift proxy
+    ports:
+      - 8080
+    type: http
+    method: HEAD
+    path: "/healthcheck"
+  - group: "{{ groups['aodh_api'] }}"
+    name: aodh api
+    ports:
+      - 8042
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['ironic_api'] }}"
+    name: ironic api
+    ports:
+      - 6385
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['rabbitmq_all'] }}"
+    name: rabbitmq management
+    ports:
+      - 15672
+      - 15671
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['rabbitmq_all'] }}"
+    name: rabbitmq access
+    ports:
+      - 5672
+      - 5671
+    type: tcp
+  - group: "{{ groups['magnum_all'] }}"
+    name: magnum api
+    ports:
+      - 9511
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['trove_api'] }}"
+    name: trove api
+    ports:
+      - 8779
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['barbican_api'] }}"
+    name: barbican api
+    ports:
+      - 9311
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['designate_api'] }}"
+    name: designate api
+    ports:
+      - 9001
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['octavia_all'] }}"
+    name: octavia api
+    ports:
+      - 9876
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['tacker_all'] }}"
+    name: tracker api
+    ports:
+      - 9890
+    type: http
+    method: HEAD
+    path: "/"
+  - group: "{{ groups['neutron_server'] }}"
+    name: opendaylight
+    ports:
+      - 8180
+      - 8185
+    type: tcp
+  - group: "{{ groups['neutron_server'] }}"
+    name: ceph-rgw
+    ports:
+      - 7980
+    type: http
+    method: HEAD
+    path: "/"
+
+
 # apm
 apm_token: SuperSecrete
 
@@ -25,10 +259,10 @@ apm_token: SuperSecrete
 # Grafana
 grafana_dashboards:
   - dashboard_id: 5566
-    revision_id: 0
+    revision_id: 5
     datasource: "metricbeat-Elasticsearch"
   - dashboard_id: 5569
-    revision_id: 0
+    revision_id: 3
     datasource: "filebeat-Elasticsearch"
 
 grafana_datasources:

grafana/readme.rst

@@ -62,7 +62,7 @@ Create the containers
 
 .. code-block:: bash
 
-   cd /opt/openstack-ansible-playbooks
+   cd /opt/openstack-ansible/playbooks
    openstack-ansible lxc-containers-create.yml -e 'container_group=grafana'
 
 install grafana