Update elk_6x for 6.5.x

This change updates the roles / playbooks to begin using Elasticsearch
relesae 6.5.x. Core to this change is the conversion of the journalbeat
role from custom compiled go, to simple package install which was made
possible by the folks at elastic within this release. Because of the
conversion the "beats-community" playbook has been removed given its now
empty.

A change to the bootstrap script was made allowing it to parse an OS id
with a "-" in it, like "opensuse-tumbleweed".

Change-Id: Ic9b80234d6a6ce876bff885f3223874602d55dd6
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh

@@ -19,6 +19,8 @@ export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
 OPTS+=('ANSIBLE_EMBED_HOME')
 
 source /etc/os-release
+export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
+
 if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
   if [  ${ID} = "ubuntu" ]; then
     apt-get update

elk_metrics_6x/installKibana.yml

@@ -39,11 +39,6 @@
             attributes:
               title: "*"
               timeFieldName: "@timestamp"
-        - name: "journalbeat-*"
-          index_options:
-            attributes:
-              title: "journalbeat-*"
-              timeFieldName: "@timestamp"
       register: kibana_indexes
       until: kibana_indexes is success
       retries: 6

elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml

@@ -32,4 +32,4 @@ galaxy_info:
     - elastic-stack
 dependencies:
   - role: elastic_data_hosts
-  - role: go_install_1.10
+  - role: elastic_repositories

elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml

@@ -13,32 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Notice
+- name: Load templates
-  debug:
+  shell: >-
-    msg: >-
+    journalbeat setup
-      The setup task for templates and dashboards is broken due to missing
+    {{ item }}
-      configuration. Once the following issue
+    -E 'output.logstash.enabled=false'
-      [ https://github.com/mheese/journalbeat/issues/136 ] is
+    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-      resolved, this should be enabled.
+    -E 'setup.template.enabled=true'
-
+    -E 'setup.template.overwrite=true'
-# - name: Load templates
+    -e -v
-#   shell: >-
+  with_items:
-#     /usr/local/bin/journalbeat -setup
+    - "--template"
-#     {{ item }}
+    - "--pipelines"
-#     -E 'output.logstash.enabled=false'
+    - "--machine-learning"
-#     -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
+    - "--dashboards"
-#     -E 'setup.template.enabled=true'
+  register: templates
-#     -E 'setup.template.overwrite=true'
+  until: templates is success
-#     -e -v
+  retries: 5
-#   with_items:
+  delay: 5
-#     - "--template"
+  run_once: true
-#     - "--pipelines"
+  tags:
-#     - "--machine-learning"
+    - setup
-#     - "--dashboards"
-#   register: templates
-#   until: templates is success
-#   retries: 5
-#   delay: 5
-#   run_once: true
-#   tags:
-#     - setup

elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml

@@ -25,6 +25,14 @@
   tags:
     - always
 
+- name: Uninstall legacy journalbeat
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/systemd/system/journalbeat.service
+    - /usr/local/bin/journalbeat
+
 - name: Ensure beat is installed
   package:
     name: "{{ journalbeat_distro_packages }}"
@@ -39,42 +47,16 @@
   tags:
     - package_install
 
-- name: create the system group
+- name: Ensure beat is installed (aarch64)
-  group:
+  apt:
-    name: "journalbeat"
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/journalbeat-6.5.0-arm64.deb'
-    state: "present"
+  when:
-    system: "yes"
+    - ansible_pkg_mgr == 'apt'
-
+    - ansible_architecture == 'aarch64'
-- name: Create the journalbeat user
-  user:
-    name: "journalbeat"
-    group: "journalbeat"
-    comment: "journalbeat user"
-    shell: "/bin/false"
-    createhome: "yes"
-    home: "/usr/share/journalbeat"
-
-- name: Create journalbeat data path
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "journalbeat"
-    group: "journalbeat"
-    mode: "0755"
-  with_items:
-    - "/usr/share/journalbeat"
-    - "/var/lib/journalbeat"
-    - "/etc/journalbeat"
-
-- name: Install journalbeat
-  shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat
-  args:
-    creates: /usr/local/bin/journalbeat
-  environment:
-    PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin"
-    GOPATH: /usr/local
   notify:
-    - Enable and restart journalbeat
+    - Enable and restart heartbeat
+  tags:
+    - package_install
 
 - name: Create journalbeat systemd service config dir
   file:
@@ -95,7 +77,7 @@
   notify:
     - Enable and restart journalbeat
 
-- name: Drop journalbeat conf files
+- name: Drop journalbeat configs
   template:
     src: "{{ item.src }}"
     dest: "{{ item.dest }}"
@@ -106,30 +88,6 @@
   notify:
     - Enable and restart journalbeat
 
-- name: Run the systemd service role
-  include_role:
-    name: systemd_service
-    private: true
-  vars:
-    systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
-    systemd_service_restart_changed: false
-    systemd_services:
-      - service_name: "journalbeat"
-        execstarts:
-          - /usr/local/bin/journalbeat
-            -c /etc/journalbeat/journalbeat.yml
-            -path.home /usr/share/journalbeat
-            -path.config /etc/journalbeat
-            -path.data /var/lib/journalbeat
-            -path.logs /var/log/beats
-        config_overrides:
-          Service:
-            EnvironmentFile: "-/etc/default/go1.10.1"
-          Unit:
-            Documentation: https://github.com/mheese/journalbeat/blob/master/README.md
-            Wants: network-online.target
-            After: network-online.target
-
 - include_tasks: journalbeat_setup.yml
   tags:
     - setup

elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - systemd-devel

elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - systemd-devel

elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - libsystemd-dev

elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml

@@ -52,7 +52,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/site-beats-community.yml

@@ -1,14 +0,0 @@
----
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- import_playbook: installJournalbeat.yml

elk_metrics_6x/site-beats-core.yml

@@ -35,3 +35,5 @@
 - import_playbook: fieldRefresh.yml
   vars:
     index_pattern: filebeat-*
+
+- import_playbook: installJournalbeat.yml

elk_metrics_6x/site.yml

@@ -13,5 +13,4 @@
 
 - import_playbook: site-elka.yml
 - import_playbook: site-beats-core.yml
-- import_playbook: site-beats-community.yml
 - import_playbook: createElasticIndexes.yml