From 68664a9dc1e572460773154fa2bd32b58e10c6ae Mon Sep 17 00:00:00 2001 From: Georgina Shippey Date: Tue, 9 Jul 2019 12:06:25 +0100 Subject: [PATCH] Config updates for elk 7.x Updated ELK config files to elk 7.x reference samples, bringing over existing customisation from elk_metrics_6x. Removed deprecated use of --pipeline in elastic_beat_setup/tasks/main.yml, --pipeline is no longer a valid cli argument. Updated logstash-pipelines and removed the dynamic insertion of the date into index names. This function is now done with the new ILM feature in elasticsearch rather than logstash. Installation of each beat creates an ILM policy for that beat and this patch does not change the default policy. It is possible that the default policy will exhaust the available storage and future work needs to be done to address this. The non-beat elements of the logstash pipeline (syslog, collectd and others) are not yet updated to be compatible with ILM. Change-Id: I735b64c2b7b93e23562f35266134a176a00af1b7 --- elk_metrics_7x/README.rst | 22 +- elk_metrics_7x/ansible-role-requirements.yml | 6 +- elk_metrics_7x/createElasticIndexes.yml | 22 +- elk_metrics_7x/installCurator.yml | 30 - .../elastic_beat_setup/defaults/main.yml | 2 +- .../roles/elastic_beat_setup/tasks/main.yml | 10 +- .../roles/elastic_curator/handlers/main.yml | 25 - .../roles/elastic_curator/meta/main.yml | 34 - .../elastic_curator/tasks/curator_systemd.yml | 46 -- .../elastic_curator/tasks/curator_upstart.yml | 32 - .../roles/elastic_curator/tasks/main.yml | 103 --- .../templates/curator-actions-age.yml.j2 | 65 -- .../templates/curator-actions-size.yml.j2 | 63 -- .../elastic_curator/templates/curator.yml.j2 | 32 - .../roles/elastic_curator/vars/redhat.yml | 17 - .../roles/elastic_curator/vars/suse.yml | 17 - .../elastic_curator/vars/ubuntu-14.04.yml | 17 - .../roles/elastic_curator/vars/ubuntu.yml | 18 - .../templates/filebeat.yml.j2 | 699 +++++++++++++++--- .../templates/heartbeat.yml.j2 | 603 +++++++++++++-- .../templates/journalbeat.yml.j2 | 143 ++-- .../elastic_kibana/templates/kibana.yml.j2 | 113 ++- .../roles/elastic_logstash/defaults/main.yml | 2 +- .../templates/logstash.yml.j2 | 17 +- .../templates/metricbeat.yml.j2 | 699 ++++++++++++------ .../templates/packetbeat.yml.j2 | 207 +++--- .../elastic_repositories/vars/ubuntu.yml | 2 +- .../roles/elastic_retention/defaults/main.yml | 118 --- .../roles/elastic_retention/meta/main.yml | 34 - .../roles/elastic_retention/tasks/main.yml | 104 --- .../calculate_index_retention_default.yml | 58 -- .../roles/elastic_rollup/defaults/main.yml | 1 + .../roles/elastic_rollup/meta/main.yml | 2 - .../roles/elastic_rollup/tasks/main.yml | 15 - .../templates/elasticsearch.yml.j2 | 92 ++- elk_metrics_7x/site-elka.yml | 2 - elk_metrics_7x/templates/_macros.j2 | 84 ++- elk_metrics_7x/templates/jvm.options.j2 | 7 - .../templates/logstash-pipelines.yml.j2 | 6 +- .../tests/ansible-role-requirements.yml | 16 +- elk_metrics_7x/tests/functional.yml | 34 +- elk_metrics_7x/tests/manual-test.rc | 8 +- elk_metrics_7x/tests/post-run.yml | 2 +- elk_metrics_7x/tests/run-cleanup.sh | 2 +- elk_metrics_7x/tests/run-setup.yml | 4 +- elk_metrics_7x/tests/run-tests.sh | 12 +- elk_metrics_7x/tests/testAPI.yml | 8 +- elk_metrics_7x/tests/testLayout.yml | 2 +- elk_metrics_7x/vars/variables.yml | 35 + zuul.d/jobs.yaml | 14 + zuul.d/project.yaml | 1 + 51 files changed, 2118 insertions(+), 1589 deletions(-) delete mode 100644 elk_metrics_7x/installCurator.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/handlers/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/meta/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/tasks/curator_systemd.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/tasks/curator_upstart.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/tasks/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/templates/curator-actions-age.yml.j2 delete mode 100644 elk_metrics_7x/roles/elastic_curator/templates/curator-actions-size.yml.j2 delete mode 100644 elk_metrics_7x/roles/elastic_curator/templates/curator.yml.j2 delete mode 100644 elk_metrics_7x/roles/elastic_curator/vars/redhat.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/vars/suse.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/vars/ubuntu-14.04.yml delete mode 100644 elk_metrics_7x/roles/elastic_curator/vars/ubuntu.yml delete mode 100644 elk_metrics_7x/roles/elastic_retention/defaults/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_retention/meta/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_retention/tasks/main.yml delete mode 100644 elk_metrics_7x/roles/elastic_retention/vars/calculate_index_retention_default.yml diff --git a/elk_metrics_7x/README.rst b/elk_metrics_7x/README.rst index cc0e40f1..d21cd162 100644 --- a/elk_metrics_7x/README.rst +++ b/elk_metrics_7x/README.rst @@ -246,7 +246,7 @@ Copy the env.d file into place .. code-block:: bash - cd /opt/openstack-ansible-ops/elk_metrics_6x + cd /opt/openstack-ansible-ops/elk_metrics_7x cp env.d/elk.yml /etc/openstack_deploy/env.d/ Copy the conf.d file into place @@ -312,7 +312,7 @@ deploy logstash, deploy Kibana, and then deploy all of the service beats. .. code-block:: bash - cd /opt/openstack-ansible-ops/elk_metrics_6x + cd /opt/openstack-ansible-ops/elk_metrics_7x ansible-playbook site.yml $USER_VARS @@ -332,7 +332,7 @@ deploy logstash, deploy Kibana, and then deploy all of the service beats. .. code-block:: bash - ln -s /opt/openstack-ansible/inventory/group_vars /opt/openstack-ansible-ops/elk_metrics_6x/group_vars + ln -s /opt/openstack-ansible/inventory/group_vars /opt/openstack-ansible-ops/elk_metrics_7x/group_vars The individual playbooks found within this repository can be independently run @@ -434,7 +434,7 @@ configuration file using the key/value pairs as options. - server1.local:9092 - server2.local:9092 - server3.local:9092 - client_id: "elk_metrics_6x" + client_id: "elk_metrics_7x" compression_type: "gzip" security_protocol: "SSL" id: "UniqueOutputID" @@ -472,7 +472,7 @@ See the grafana directory for more information on how to deploy grafana. Once When deploying grafana, source the variable file from ELK in order to automatically connect grafana to the Elasticsearch datastore and import dashboards. Including the variable file is as simple as adding -``-e @../elk_metrics_6x/vars/variables.yml`` to the grafana playbook +``-e @../elk_metrics_7x/vars/variables.yml`` to the grafana playbook run. Included dashboards. @@ -485,7 +485,7 @@ Example command using the embedded Ansible from within the grafana directory. .. code-block:: bash ansible-playbook ${USER_VARS} installGrafana.yml \ - -e @../elk_metrics_6x/vars/variables.yml \ + -e @../elk_metrics_7x/vars/variables.yml \ -e 'galera_root_user="root"' \ -e 'galera_address={{ internal_lb_vip_address }}' @@ -566,7 +566,7 @@ state variable, `elk_package_state`, to latest. .. code-block:: bash - cd /opt/openstack-ansible-ops/elk_metrics_6x + cd /opt/openstack-ansible-ops/elk_metrics_7x ansible-playbook site.yml $USER_VARS -e 'elk_package_state="latest"' @@ -582,7 +582,7 @@ execution. .. code-block:: bash - cd /opt/openstack-ansible-ops/elk_metrics_6x + cd /opt/openstack-ansible-ops/elk_metrics_7x ansible-playbook site.yml $USER_VARS -e 'elastic_retention_refresh="yes"' @@ -593,7 +593,7 @@ If everything goes bad, you can clean up with the following command .. code-block:: bash - openstack-ansible /opt/openstack-ansible-ops/elk_metrics_6x/site.yml -e 'elk_package_state="absent"' --tags package_install + openstack-ansible /opt/openstack-ansible-ops/elk_metrics_7x/site.yml -e 'elk_package_state="absent"' --tags package_install openstack-ansible /opt/openstack-ansible/playbooks/lxc-containers-destroy.yml --limit elk_all @@ -616,14 +616,14 @@ deployed to the environment as if this was a production installation. After the test build is completed the cluster will test it's layout and ensure processes are functioning normally. Logs for the cluster can be found at -`/tmp/elk-metrics-6x-logs`. +`/tmp/elk-metrics-7x-logs`. To rerun the playbooks after a test build, source the `tests/manual-test.rc` file and follow the onscreen instructions. To clean-up a test environment and start from a bare server slate the `run-cleanup.sh` script can be used. This script is distructive and will purge -all `elk_metrics_6x` related services within the local test environment. +all `elk_metrics_7x` related services within the local test environment. .. code-block:: bash diff --git a/elk_metrics_7x/ansible-role-requirements.yml b/elk_metrics_7x/ansible-role-requirements.yml index 4a5ad5d4..068b7712 100644 --- a/elk_metrics_7x/ansible-role-requirements.yml +++ b/elk_metrics_7x/ansible-role-requirements.yml @@ -1,13 +1,13 @@ --- - name: systemd_service scm: git - src: https://git.openstack.org/openstack/ansible-role-systemd_service + src: https://opendev.org/openstack/ansible-role-systemd_service version: master - name: systemd_mount scm: git - src: https://git.openstack.org/openstack/ansible-role-systemd_mount + src: https://opendev.org/openstack/ansible-role-systemd_mount version: master - name: config_template scm: git - src: https://git.openstack.org/openstack/ansible-config_template + src: https://opendev.org/openstack/ansible-config_template version: master diff --git a/elk_metrics_7x/createElasticIndexes.yml b/elk_metrics_7x/createElasticIndexes.yml index dd04128d..12a9bc58 100644 --- a/elk_metrics_7x/createElasticIndexes.yml +++ b/elk_metrics_7x/createElasticIndexes.yml @@ -15,6 +15,9 @@ hosts: "elastic-logstash[0]" become: true + roles: + - role: elastic_data_hosts + vars: _elastic_refresh_interval: "{{ (elasticsearch_number_of_replicas | int) * 5 }}" elastic_refresh_interval: "{{ (_elastic_refresh_interval > 0) | ternary(30, _elastic_refresh_interval) }}" @@ -24,9 +27,6 @@ environment: "{{ deployment_environment_variables | default({}) }}" - roles: - - role: elastic_retention - post_tasks: - name: Create beat indexes uri: @@ -41,7 +41,7 @@ delay: 30 with_items: |- {% set beat_indexes = [] %} - {% for key, value in elastic_beat_retention_policy_hosts.items() %} + {% for key, value in elastic_beats.items() %} {% if ((value.hosts | length) > 0) and (value.make_index | default(false) | bool) %} {% set _index = { @@ -124,7 +124,7 @@ index_option: index_patterns: >- {{ - (elastic_beat_retention_policy_hosts.keys() | list) + (elastic_beats.keys() | list) | map('regex_replace', '(.*)', '\1-' ~ '*') | list }} @@ -152,7 +152,7 @@ order: 1 settings: number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" - number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}" + number_of_shards: 1 - name: Create custom skydive index template uri: @@ -171,7 +171,7 @@ order: 1 settings: number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" - number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}" + number_of_shards: 1 - name: Create/Setup known indexes in Kibana @@ -183,10 +183,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" roles: - - role: elastic_retention + - role: elastic_data_hosts post_tasks: - - name: Create kibana indexe patterns + - name: Create kibana index patterns uri: url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}" method: POST @@ -198,7 +198,7 @@ kbn-xsrf: "{{ inventory_hostname | to_uuid }}" with_items: |- {% set beat_indexes = [] %} - {% for key, value in elastic_beat_retention_policy_hosts.items() %} + {% for key, value in elastic_beats.items() %} {% if (value.hosts | length) > 0 %} {% set _index = { @@ -219,7 +219,7 @@ {% set _ = beat_indexes.append(_index) %} {% endif %} {% endfor %} - {% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'title': '*'}}}) %} + {% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'timeFieldName': '@timestamp', 'title': '*'}}}) %} {{ beat_indexes }} register: kibana_indexes until: kibana_indexes is success diff --git a/elk_metrics_7x/installCurator.yml b/elk_metrics_7x/installCurator.yml deleted file mode 100644 index 85746723..00000000 --- a/elk_metrics_7x/installCurator.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Curator - hosts: "elastic-logstash" - become: true - gather_facts: true - vars: - haproxy_ssl: false - - vars_files: - - vars/variables.yml - - environment: "{{ deployment_environment_variables | default({}) }}" - - roles: - - role: elastic_curator - - tags: - - beat-install diff --git a/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml b/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml index 998b689f..8d29f37a 100644 --- a/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_beat_setup/defaults/main.yml @@ -15,7 +15,7 @@ # Each setup flag is run one at a time. elastic_setup_flags: - - "--template" + - "--index-management" - "--pipelines" # - "--dashboards" diff --git a/elk_metrics_7x/roles/elastic_beat_setup/tasks/main.yml b/elk_metrics_7x/roles/elastic_beat_setup/tasks/main.yml index 2a359510..75ac7ee3 100644 --- a/elk_metrics_7x/roles/elastic_beat_setup/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_beat_setup/tasks/main.yml @@ -41,7 +41,11 @@ sed -i 's@"id": "{{ elastic_beat_name }}\-\*",@"id": "{{ elastic_beat_name }}",@g' /usr/share/{{ elastic_beat_name }}/kibana/6/index-pattern/*.json {% endif %} {{ elastic_beat_name }} setup - {{ item }} + {% if elastic_beat_name == "heartbeat" and item == "--index-management" -%} + --template + {%- else -%} + {{ item }} + {%- endif %} {{ elastic_beat_setup_options }} -e -v with_items: "{{ elastic_setup_flags }}" @@ -53,10 +57,10 @@ delay: 5 run_once: true when: - - ((ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] is undefined) or + - (((ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] is undefined) or (not (ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] | bool))) or ((elk_package_state | default('present')) == "latest") or - (elk_beat_setup | default(false) | bool) + (elk_beat_setup | default(false) | bool)) and not (elastic_beat_name == "heartbeat" and item == "--pipelines") tags: - setup diff --git a/elk_metrics_7x/roles/elastic_curator/handlers/main.yml b/elk_metrics_7x/roles/elastic_curator/handlers/main.yml deleted file mode 100644 index ef153bdc..00000000 --- a/elk_metrics_7x/roles/elastic_curator/handlers/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Enable and restart curator.timer - systemd: - name: "curator.timer" - enabled: true - state: restarted - when: - - (elk_package_state | default('present')) != 'absent' - - ansible_service_mgr == 'systemd' - tags: - - config diff --git a/elk_metrics_7x/roles/elastic_curator/meta/main.yml b/elk_metrics_7x/roles/elastic_curator/meta/main.yml deleted file mode 100644 index c1e7ffbe..00000000 --- a/elk_metrics_7x/roles/elastic_curator/meta/main.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: OpenStack - description: Elastic v6.x curator role - company: Rackspace - license: Apache2 - min_ansible_version: 2.5 - platforms: - - name: Ubuntu - versions: - - trusty - - xenial - - bionic - categories: - - cloud - - development - - elasticsearch - - elastic-stack -dependencies: - - role: elastic_retention diff --git a/elk_metrics_7x/roles/elastic_curator/tasks/curator_systemd.yml b/elk_metrics_7x/roles/elastic_curator/tasks/curator_systemd.yml deleted file mode 100644 index 3c2b17ec..00000000 --- a/elk_metrics_7x/roles/elastic_curator/tasks/curator_systemd.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Run the systemd service role - include_role: - name: systemd_service - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_user_name: curator - systemd_group_name: curator - systemd_services: - - service_name: "curator" - execstarts: - - /opt/elasticsearch-curator/bin/curator - --config /var/lib/curator/curator.yml - /var/lib/curator/actions-age.yml - timer: - state: "started" - options: - OnBootSec: 30min - OnUnitActiveSec: 12h - Persistent: true - - service_name: "curator-size" - execstarts: - - /opt/elasticsearch-curator/bin/curator - --config /var/lib/curator/curator.yml - /var/lib/curator/actions-size.yml - timer: - state: "started" - options: - OnBootSec: 30min - OnUnitActiveSec: 1h - Persistent: true diff --git a/elk_metrics_7x/roles/elastic_curator/tasks/curator_upstart.yml b/elk_metrics_7x/roles/elastic_curator/tasks/curator_upstart.yml deleted file mode 100644 index 3a3d1d49..00000000 --- a/elk_metrics_7x/roles/elastic_curator/tasks/curator_upstart.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Create cron job for curator (age) - cron: - name: "Run curator" - minute: "0" - hour: "1" - user: "curator" - job: "/opt/elasticsearch-curator/bin/curator --config /var/lib/curator/curator.yml /var/lib/curator/actions-age.yml" - cron_file: "elasticsearch-curator" - -- name: Create cron job for curator (size) - cron: - name: "Run curator" - minute: "0" - hour: "*/5" - user: "curator" - job: "/opt/elasticsearch-curator/bin/curator --config /var/lib/curator/curator.yml /var/lib/curator/actions-size.yml" - cron_file: "elasticsearch-curator" diff --git a/elk_metrics_7x/roles/elastic_curator/tasks/main.yml b/elk_metrics_7x/roles/elastic_curator/tasks/main.yml deleted file mode 100644 index 1c014d3e..00000000 --- a/elk_metrics_7x/roles/elastic_curator/tasks/main.yml +++ /dev/null @@ -1,103 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Gather variables for each operating system - include_vars: "{{ item }}" - with_first_found: - - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" - - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" - - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" - - "{{ ansible_distribution | lower }}.yml" - - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" - - "{{ ansible_os_family | lower }}.yml" - tags: - - always - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Ensure virtualenv is installed - package: - name: "{{ curator_distro_packages }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}" - tags: - - package_install - -- name: Create the virtualenv (if it does not exist) - command: "virtualenv --never-download --no-site-packages /opt/elasticsearch-curator" - args: - creates: "/opt/elasticsearch-curator/bin/activate" - -- name: Ensure curator is installed - pip: - name: "elasticsearch-curator<6" - state: "{{ elk_package_state | default('present') }}" - extra_args: --isolated - virtualenv: /opt/elasticsearch-curator - register: _pip_task - until: _pip_task is success - retries: 3 - delay: 2 - tags: - - package_install - -- name: create the system group - group: - name: "curator" - state: "present" - system: "yes" - -- name: Create the curator system user - user: - name: "curator" - group: "curator" - comment: "curator user" - shell: "/bin/false" - createhome: "yes" - home: "/var/lib/curator" - -- name: Create curator data path - file: - path: "{{ item }}" - state: directory - owner: "curator" - group: "curator" - mode: "0755" - recurse: true - with_items: - - "/var/lib/curator" - - "/var/log/curator" - - "/etc/curator" - -- name: Drop curator conf file(s) - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: "curator.yml.j2" - dest: /var/lib/curator/curator.yml - - src: "curator-actions-age.yml.j2" - dest: /var/lib/curator/actions-age.yml - - src: "curator-actions-size.yml.j2" - dest: /var/lib/curator/actions-size.yml - notify: - - Enable and restart curator.timer - -- include_tasks: "curator_{{ ansible_service_mgr }}.yml" diff --git a/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-age.yml.j2 b/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-age.yml.j2 deleted file mode 100644 index d3e5e95b..00000000 --- a/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-age.yml.j2 +++ /dev/null @@ -1,65 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{% set action_items = [] -%} -{# Delete index loop #} -{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} -{% set delete_indices = {} -%} -{# Total retention size in days #} -{% set _index_retention = ansible_local['elastic']['retention']['elastic_' + key + '_retention'] -%} -{% set index_retention = ((_index_retention | int) > 0) | ternary(_index_retention, 1) | int %} -{% set _ = delete_indices.update( - { - 'action': 'delete_indices', - 'description': 'Prune indices for ' + key + ' after ' ~ index_retention ~ ' days', - 'options': { - 'ignore_empty_list': true, - 'disable_action': false - } - } - ) --%} -{% set filters = [] -%} -{% set _ = filters.append( - { - 'filtertype': 'pattern', - 'kind': 'prefix', - 'value': key - } - ) --%} -{% set _ = filters.append( - { - 'filtertype': 'age', - 'source': 'name', - 'direction': 'older', - 'timestring': '%Y.%m.%d', - 'unit': 'days', - 'unit_count': index_retention - } - ) --%} -{% set _ = delete_indices.update({'filters': filters}) -%} -{% set _ = action_items.append(delete_indices) -%} -{% endfor -%} - -{% set actions = {} -%} -{% for action_item in action_items -%} -{% set _ = actions.update({loop.index: action_item}) -%} -{% endfor -%} - -{# Render all actions #} -{% set curator_actions = {'actions': actions} -%} -{{ curator_actions | to_nice_yaml(indent=2) }} diff --git a/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-size.yml.j2 b/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-size.yml.j2 deleted file mode 100644 index 86cb6527..00000000 --- a/elk_metrics_7x/roles/elastic_curator/templates/curator-actions-size.yml.j2 +++ /dev/null @@ -1,63 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{% set action_items = [] -%} -{# Delete index loop #} -{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} -{% set delete_indices = {} -%} -{# Total retention size in gigabytes #} -{% set _index_size = ((ansible_local['elastic']['retention']['elastic_' + key + '_size'] | int) // 1024) -%} -{% set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %} -{% set _ = delete_indices.update( - { - 'action': 'delete_indices', - 'description': 'Prune indices for ' + key + ' after index is > ' ~ index_size ~ 'gb', - 'options': { - 'ignore_empty_list': true, - 'disable_action': false - } - } - ) --%} -{% set filters = [] -%} -{% set _ = filters.append( - { - 'filtertype': 'pattern', - 'kind': 'prefix', - 'value': key - } - ) --%} -{% set _ = filters.append( - { - 'filtertype': 'space', - 'disk_space': index_size, - 'use_age': true, - 'source': 'creation_date' - } - ) --%} -{% set _ = delete_indices.update({'filters': filters}) -%} -{% set _ = action_items.append(delete_indices) -%} -{% endfor -%} - -{% set actions = {} -%} -{% for action_item in action_items -%} -{% set _ = actions.update({loop.index: action_item}) -%} -{% endfor -%} - -{# Render all actions #} -{% set curator_actions = {'actions': actions} -%} -{{ curator_actions | to_nice_yaml(indent=2) }} diff --git a/elk_metrics_7x/roles/elastic_curator/templates/curator.yml.j2 b/elk_metrics_7x/roles/elastic_curator/templates/curator.yml.j2 deleted file mode 100644 index ff0754ee..00000000 --- a/elk_metrics_7x/roles/elastic_curator/templates/curator.yml.j2 +++ /dev/null @@ -1,32 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -client: - hosts: - - {{ ansible_host }} - port: {{ elastic_port }} - url_prefix: "" - use_ssl: false - ssl_no_validate: true - http_auth: "" - timeout: 120 - master_only: true - -logging: - loglevel: INFO - logfile: /var/log/curator/curator - logformat: default - blacklist: - - elasticsearch - - urllib3 diff --git a/elk_metrics_7x/roles/elastic_curator/vars/redhat.yml b/elk_metrics_7x/roles/elastic_curator/vars/redhat.yml deleted file mode 100644 index a3eb703e..00000000 --- a/elk_metrics_7x/roles/elastic_curator/vars/redhat.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -curator_distro_packages: - - python-virtualenv diff --git a/elk_metrics_7x/roles/elastic_curator/vars/suse.yml b/elk_metrics_7x/roles/elastic_curator/vars/suse.yml deleted file mode 100644 index a3eb703e..00000000 --- a/elk_metrics_7x/roles/elastic_curator/vars/suse.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -curator_distro_packages: - - python-virtualenv diff --git a/elk_metrics_7x/roles/elastic_curator/vars/ubuntu-14.04.yml b/elk_metrics_7x/roles/elastic_curator/vars/ubuntu-14.04.yml deleted file mode 100644 index a3eb703e..00000000 --- a/elk_metrics_7x/roles/elastic_curator/vars/ubuntu-14.04.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -curator_distro_packages: - - python-virtualenv diff --git a/elk_metrics_7x/roles/elastic_curator/vars/ubuntu.yml b/elk_metrics_7x/roles/elastic_curator/vars/ubuntu.yml deleted file mode 100644 index 363a2210..00000000 --- a/elk_metrics_7x/roles/elastic_curator/vars/ubuntu.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -curator_distro_packages: - - python-virtualenv - - virtualenv diff --git a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 index 49ab5525..85e29526 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -9,10 +9,10 @@ # https://www.elastic.co/guide/en/beats/filebeat/index.html -#========================== Modules configuration ============================ +#========================== Modules configuration ============================= filebeat.modules: -#------------------------------- System Module ------------------------------- +#-------------------------------- System Module -------------------------------- - module: system # Syslog syslog: @@ -23,11 +23,11 @@ filebeat.modules: #var.paths: # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. - var.convert_timezone: false + #var.convert_timezone: false - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: # Authorization logs auth: @@ -40,35 +40,112 @@ filebeat.modules: # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. var.convert_timezone: false - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#------------------------------- Apache2 Module ------------------------------ -- module: apache2 +#-------------------------------- Apache Module -------------------------------- +- module: apache + # Access logs access: enabled: "{{ filebeat_httpd_enabled | default(true) }}" + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. var.paths: - /openstack/log/*horizon*/horizon/*access.log + + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + + # Error logs error: enabled: "{{ filebeat_httpd_enabled | default(true) }}" + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. var.paths: - /openstack/log/*horizon*/horizon/horizon-error.log -#------------------------------- Auditd Module ------------------------------- -- module: auditd - log: - enabled: "{{ filebeat_auditd_enabled | default(true) }}" + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + +#-------------------------------- Auditd Module -------------------------------- +#- module: auditd +# log: +# enabled: "{{ filebeat_auditd_enabled | default(true) }}" # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#------------------------------- Icinga Module ------------------------------- +#---------------------------- Elasticsearch Module ---------------------------- +#- module: elasticsearch + # Server log + #server: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + #gc: + #enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + #audit: + #enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + #slowlog: + #enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + #deprecation: + #enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + +#------------------------------- Haproxy Module ------------------------------- +#- module: haproxy + # All logs + #log: + #enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + +#-------------------------------- Icinga Module -------------------------------- #- module: icinga # Main logs #main: @@ -78,9 +155,9 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: # Debug logs #debug: @@ -90,9 +167,9 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: # Startup logs #startup: @@ -102,11 +179,11 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#--------------------------------- IIS Module -------------------------------- +#--------------------------------- IIS Module --------------------------------- #- module: iis # Access logs #access: @@ -116,9 +193,9 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: # Error logs #error: @@ -128,11 +205,11 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#-------------------------------- Kafka Module ------------------------------- +#-------------------------------- Kafka Module -------------------------------- #- module: kafka # All logs #log: @@ -146,25 +223,37 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false -#------------------------------ logstash Module ------------------------------ -- module: logstash +#-------------------------------- Kibana Module -------------------------------- +#- module: kibana + # All logs + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + +#------------------------------- Logstash Module ------------------------------- +#- module: logstash # logs - log: - enabled: true + #log: + #enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. # var.paths: # Slow logs - slowlog: - enabled: true + #slowlog: + #enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: -#------------------------------- mongodb Module ------------------------------ +#------------------------------- Mongodb Module ------------------------------- #- module: mongodb # Logs #log: @@ -178,30 +267,70 @@ filebeat.modules: # can be added under this section. #input: -#-------------------------------- MySQL Module ------------------------------- -- module: mysql +#-------------------------------- MySQL Module -------------------------------- +#- module: mysql + # Error logs error: enabled: "{{ filebeat_galera_enabled | default(true) }}" + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. var.paths: - /openstack/log/*galera*/mysql_logs/galera_server_error.log - /var/log/mysql_logs/galera_server_error.log - slowlog: - enabled: false -#-------------------------------- Nginx Module ------------------------------- + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + + # Slow logs + #slowlog: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + +#-------------------------------- Nginx Module -------------------------------- - module: nginx + # Access logs access: enabled: "{{ filebeat_nginx_enabled | default(true) }}" + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. var.paths: - /openstack/log/*repo_container*/nginx/*access.log - /openstack/log/*keystone*/nginx/*access.log + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + # Error logs error: enabled: "{{ filebeat_nginx_enabled | default(true) }}" + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. var.paths: - /openstack/log/*repo_container*/nginx/*error.log - /openstack/log/*keystone*/nginx/*error.log -#------------------------------- Osquery Module ------------------------------ + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + +#------------------------------- Osquery Module ------------------------------- - module: osquery result: enabled: "{{ filebeat_osquery_enabled | default(true) }}" @@ -215,7 +344,7 @@ filebeat.modules: # of the document. The default is true. var.use_namespace: true -#----------------------------- PostgreSQL Module ----------------------------- +#------------------------------ PostgreSQL Module ------------------------------ #- module: postgresql # Logs #log: @@ -225,11 +354,11 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#-------------------------------- Redis Module ------------------------------- +#-------------------------------- Redis Module -------------------------------- #- module: redis # Main logs #log: @@ -249,7 +378,15 @@ filebeat.modules: # Optional, the password to use when connecting to Redis. #var.password: -#------------------------------- Traefik Module ------------------------------ +#----------------------------- Google Santa Module ----------------------------- +#- module: santa + #log: + #enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the the default path. + #var.paths: + +#------------------------------- Traefik Module ------------------------------- #- module: traefik # Access logs #access: @@ -259,27 +396,27 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Prospector configuration (advanced). Any prospector configuration option + # Input configuration (advanced). Any input configuration option # can be added under this section. - #prospector: + #input: -#=========================== Filebeat prospectors ============================= +#=========================== Filebeat inputs ============================= -# List of prospectors to fetch data. -filebeat.prospectors: -# Each - is a prospector. Most options can be set at the prospector level, so -# you can use different prospectors for various configurations. -# Below are the prospector specific configurations. +# List of inputs to fetch data. +filebeat.inputs: +# Each - is an input. Most options can be set at the input level, so +# you can use different inputs for various configurations. +# Below are the input specific configurations. # Type of the files. Based on this the way the file is read is decided. -# The different types cannot be mixed in one prospector +# The different types cannot be mixed in one input # # Possible options are: # * log: Reads every line of the log file (default) # * stdin: Reads the standard in -#------------------------------ Log prospector -------------------------------- +#------------------------------ Log input -------------------------------- {% for p in filebeat_prospectors %} - type: {{ p['type'] }} enabled: {{ p['enabled'] }} @@ -298,20 +435,22 @@ filebeat.prospectors: {% endfor %} {% endfor %} -#----------------------------- Stdin prospector ------------------------------- +#----------------------------- Stdin input ------------------------------- # Configuration to use stdin input #- type: stdin -#------------------------- Redis slowlog prospector --------------------------- -# Experimental: Config options for the redis slow log prospector +#------------------------- Redis slowlog input --------------------------- +# Experimental: Config options for the redis slow log input #- type: redis - #hosts: ["localhost:6379"] - #username: - #password: #enabled: false + + # List of hosts to pool to retrieve the slow log information. + #hosts: ["localhost:6379"] + + # How often the input checks for redis slow log. #scan_frequency: 10s - # Timeout after which time the prospector should return an error + # Timeout after which time the input should return an error #timeout: 1s # Network type to be used for redis connection. Default: tcp @@ -323,17 +462,140 @@ filebeat.prospectors: # Redis AUTH password. Empty by default. #password: foobared -#------------------------------ Udp prospector -------------------------------- -# Experimental: Config options for the udp prospector +#------------------------------ Udp input -------------------------------- +# Experimental: Config options for the udp input #- type: udp + #enabled: false # Maximum size of the message received over UDP - #max_message_size: 10240 + #max_message_size: 10KiB + +#------------------------------ TCP input -------------------------------- +# Experimental: Config options for the TCP input +#- type: tcp + #enabled: false + + # The host and port to receive the new event + #host: "localhost:9000" + + # Character used to split new message + #line_delimiter: "\n" + + # Maximum size in bytes of the message received over TCP + #max_message_size: 20MiB + + # The number of seconds of inactivity before a remote connection is closed. + #timeout: 300s + + # Use SSL settings for TCP. + #ssl.enabled: true + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for client verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL server authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Server Certificate Key, + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of client authentication are supported. Valid options + # are `none`, `optional`, and `required`. When `certificate_authorities` is set it will + # default to `required` otherwise it will be set to `none`. + #ssl.client_authentication: "required" + +#------------------------------ Syslog input -------------------------------- +# Experimental: Config options for the Syslog input +# Accept RFC3164 formatted syslog event via UDP. +#- type: syslog + #enabled: false + #protocol.udp: + # The host and port to receive the new event + #host: "localhost:9000" + + # Maximum size of the message received over UDP + #max_message_size: 10KiB + +# Accept RFC3164 formatted syslog event via TCP. +#- type: syslog + #enabled: false + + #protocol.tcp: + # The host and port to receive the new event + #host: "localhost:9000" + + # Character used to split new message + #line_delimiter: "\n" + + # Maximum size in bytes of the message received over TCP + #max_message_size: 20MiB + + # The number of seconds of inactivity before a remote connection is closed. + #timeout: 300s + + # Use SSL settings for TCP. + #ssl.enabled: true + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for client verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL server authentication. + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Server Certificate Key, + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections. + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites. + #ssl.curve_types: [] + + # Configure what types of client authentication are supported. Valid options + # are `none`, `optional`, and `required`. When `certificate_authorities` is set it will + # default to `required` otherwise it will be set to `none`. + #ssl.client_authentication: "required" + +#------------------------------ Docker input -------------------------------- +# Experimental: Docker input reads and parses `json-file` logs from Docker +#- type: docker + #enabled: false + + # Combine partial lines flagged by `json-file` format + #combine_partials: true + + # Use this to read from all containers, replace * with a container id to read from one: + #containers: + # stream: all # can be all, stdout or stderr + # ids: + # - '*' #========================== Filebeat autodiscover ============================== # Autodiscover allows you to detect changes in the system and spawn new modules -# or prospectors as they happen. +# or inputs as they happen. #filebeat.autodiscover: # List of enabled autodiscover providers @@ -349,14 +611,33 @@ filebeat.prospectors: #========================= Filebeat global options ============================ -# Name of the registry file. If a relative path is used, it is considered relative to the +# Registry data path. If a relative path is used, it is considered relative to the # data path. -#filebeat.registry_file: ${path.data}/registry +#filebeat.registry.path: ${path.data}/registry -# These config files must have the full filebeat config part inside, but only -# the prospector part is processed. All global options like spool_size are ignored. -# The config_dir MUST point to a different directory then where the main filebeat config file is in. -#filebeat.config_dir: +# The permissions mask to apply on registry data, and meta files. The default +# value is 0600. Must be a valid Unix-style file permissions mask expressed in +# octal notation. This option is not supported on Windows. +#filebeat.registry.file_permissions: 0600 + +# The timeout value that controls when registry entries are written to disk +# (flushed). When an unwritten update exceeds this value, it triggers a write +# to disk. When flush is set to 0s, the registry is written to disk after each +# batch of events has been published successfully. The default value is 0s. +#filebeat.registry.flush: 0s + + +# Starting with Filebeat 7.0, the registry uses a new directory format to store +# Filebeat state. After you upgrade, Filebeat will automatically migrate a 6.x +# registry file to use the new directory format. If you changed +# filebeat.registry.path while upgrading, set filebeat.registry.migrate_file to +# point to the old registry file. +#filebeat.registry.migrate_file: ${path.data}/registry + +# By default Ingest pipelines are not updated if a pipeline with the same ID +# already exists. If this option is enabled Filebeat overwrites pipelines +# everytime a new Elasticsearch connection is established. +#filebeat.overwrite_pipelines: false # How long filebeat waits on shutdown for the publisher to finish. # Default is 0, not waiting. @@ -364,9 +645,9 @@ filebeat.prospectors: # Enable filebeat config reloading #filebeat.config: - #prospectors: + #inputs: #enabled: false - #path: prospectors.d/*.yml + #path: inputs.d/*.yml #reload.enabled: true #reload.period: 10s #modules: @@ -385,8 +666,7 @@ filebeat.prospectors: # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. -tags: - - filebeat +#tags: ["filebeat",] # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested @@ -411,7 +691,8 @@ tags: # Hints the minimum number of events stored in the queue, # before providing a batch of events to the outputs. - # A value of 0 (the default) ensures events are immediately available + # The default value is set to 2048. + # A value of 0 ensures events are immediately available # to be sent to the outputs. #flush.min_events: 2048 @@ -419,6 +700,66 @@ tags: # if the number of events stored in the queue is < min_flush_events. #flush.timeout: 1s + # The spool queue will store events in a local spool file, before + # forwarding the events to the outputs. + # + # Beta: spooling to disk is currently a beta feature. Use with care. + # + # The spool file is a circular buffer, which blocks once the file/buffer is full. + # Events are put into a write buffer and flushed once the write buffer + # is full or the flush_timeout is triggered. + # Once ACKed by the output, events are removed immediately from the queue, + # making space for new events to be persisted. + #spool: + # The file namespace configures the file path and the file creation settings. + # Once the file exists, the `size`, `page_size` and `prealloc` settings + # will have no more effect. + #file: + # Location of spool file. The default value is ${path.data}/spool.dat. + #path: "${path.data}/spool.dat" + + # Configure file permissions if file is created. The default value is 0600. + #permissions: 0600 + + # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. + #size: 100MiB + + # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. + #page_size: 4KiB + + # If prealloc is set, the required space for the file is reserved using + # truncate. The default value is true. + #prealloc: true + + # Spool writer settings + # Events are serialized into a write buffer. The write buffer is flushed if: + # - The buffer limit has been reached. + # - The configured limit of buffered events is reached. + # - The flush timeout is triggered. + #write: + # Sets the write buffer size. + #buffer_size: 1MiB + + # Maximum duration after which events are flushed if the write buffer + # is not full yet. The default value is 1s. + #flush.timeout: 1s + + # Number of maximum buffered events. The write buffer is flushed once the + # limit is reached. + #flush.events: 16384 + + # Configure the on-disk event encoding. The encoding can be changed + # between restarts. + # Valid encodings are: json, ubjson, and cbor. + #codec: cbor + #read: + # Reader flush timeout, waiting for more events to become available, so + # to fill a complete batch as required by the outputs. + # If flush_timeout is 0, all available events are forwarded to the + # outputs immediately. + # The default value is 0s. + #flush.timeout: 0s + # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. #max_procs: @@ -432,8 +773,8 @@ tags: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, and -# add_cloud_metadata. +# The supported processors are drop_fields, drop_event, include_fields, +# decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that # contain CPU load percentages, but remove the fields that contain CPU ticks @@ -453,6 +794,22 @@ tags: # equals: # http.code: 200 # +# The following example renames the field a to b: +# +#processors: +#- rename: +# fields: +# - from: "a" +# to: "b" +# +# The following example tokenizes the string into fields: +# +#processors: +#- dissect: +# tokenizer: "%{key1} - %{key2}" +# field: "message" +# target_prefix: "dissect" +# # The following example enriches each event with metadata from the cloud # provider about the host machine. It works on EC2, GCE, DigitalOcean, # Tencent Cloud, and Alibaba Cloud. @@ -477,7 +834,9 @@ tags: # match_pids: ["process.pid", "process.ppid"] # match_source: true # match_source_index: 4 +# match_short_id: false # cleanup_timeout: 60 +# labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" @@ -490,8 +849,31 @@ tags: # #processors: #- add_docker_metadata: ~ +# +# The following example enriches each event with host metadata. +# processors: - add_host_metadata: ~ +# netinfo.enabled: false +# +# The following example enriches each event with process metadata using +# process IDs included in the event. +# +#processors: +#- add_process_metadata: +# match_pids: ["system.process.ppid"] +# target: system.process.parent +# +# The following example decodes fields containing JSON strings +# and replaces the strings with valid JSON objects. +# +#processors: +#- decode_json_fields: +# fields: ["field1", "field2", ...] +# process_array: false +# max_depth: 1 +# target: "" +# overwrite_keys: false #============================= Elastic Cloud ================================== @@ -524,12 +906,15 @@ processors: # Set gzip compression level. #compression_level: 0 + # Configure escaping HTML symbols in strings. + #escape_html: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" - # Dictionary of HTTP parameters to pass within the url with index operations. + # Dictionary of HTTP parameters to pass within the URL with index operations. #parameters: #param1: value1 #param2: value2 @@ -540,19 +925,19 @@ processors: # Optional index name. The default is "filebeat" plus date # and generates [filebeat-]YYYY.MM.DD keys. # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. - #index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}" + #index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. #pipeline: "" - # Optional HTTP Path + # Optional HTTP path #path: "/elasticsearch" # Custom HTTP headers to add to each request #headers: # X-My-Header: Contents of the header - # Proxy server url + # Proxy server URL #proxy_url: http://proxy:3128 # The number of times a particular Elasticsearch index operation is attempted. If @@ -564,39 +949,49 @@ processors: # The default is 50. #bulk_max_size: 50 - # Configure http request timeout before failing an request to Elasticsearch. + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 # Use SSL settings for HTTPS. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are + # and certificates will be accepted. In this mode, SSL-based connections are # susceptible to man-in-the-middle attacks. Use only for testing. Default is # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - # SSL configuration. By default is off. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" - # Client Certificate Key + # Client certificate key #ssl.key: "/etc/pki/client/cert.key" - # Optional passphrase for decrypting the Certificate Key. + # Optional passphrase for decrypting the certificate key. #ssl.key_passphrase: '' # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -606,13 +1001,12 @@ processors: #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count) }} - #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] @@ -621,7 +1015,7 @@ processors: # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats - # The Kafka event key setting. Use format string to create unique event key. + # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' @@ -642,28 +1036,38 @@ processors: #username: '' #password: '' - # Kafka version filebeat is assumed to run against. Defaults to the oldest - # supported stable version (currently version 0.8.2.0) - #version: 0.8.2 + # Kafka version filebeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' - # Metadata update configuration. Metadata do contain leader information - # deciding which broker to use when publishing. + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 - # Waiting time between retries during leader elections. Default is 250ms. + # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m + # Strategy for fetching the topics metadata from the broker. Default is true. + #full: true + # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. + # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. @@ -692,6 +1096,10 @@ processors: # default is gzip. #compression: gzip + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + # The maximum permitted size of JSON-encoded messages. Bigger messages will be # dropped. The default value is 1000000 (bytes). This value should be equal to # or less than the broker's message.max.bytes. @@ -707,7 +1115,7 @@ processors: # purposes. The default is "beats". #client_id: beats - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. @@ -720,7 +1128,7 @@ processors: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -736,7 +1144,7 @@ processors: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -748,20 +1156,24 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Redis servers to connect to. If load balancing is enabled, the + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] - # The Redis port to use if hosts does not contain a port number. The default - # is 6379. - #port: 6379 - # The name of the Redis list or channel the events are published to. The # default is filebeat. #key: filebeat - # The password to authenticate with. The default is no authentication. + # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. @@ -795,6 +1207,17 @@ processors: # until all events are published. The default is 3. #max_retries: 3 + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + # The maximum number of events to bulk in a single Redis request or pipeline. # The default is 2048. #bulk_max_size: 2048 @@ -849,6 +1272,14 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/filebeat" @@ -876,8 +1307,13 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false #================================= Paths ====================================== @@ -905,20 +1341,41 @@ processors: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs +#================================ Keystore ========================================== +# Location of the Keystore containing the keys and their sensitive values. +#keystore.path: "${path.config}/beats.keystore" + #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('filebeat') }} -#=============================== Template ====================================== +#============================== Template ====================================== {{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} -#============================== Kibana ===================================== +#============================== Setup ILM ===================================== + +# Configure Index Lifecycle Management Index Lifecycle Management creates a +# write alias and adds additional settings to the template. +# The elasticsearch.output.index setting will be replaced with the write alias +# if ILM is enabled. + +# Enabled ILM support. Valid values are true, false, and auto. The beat will +# detect availabilty of Index Lifecycle Management in Elasticsearch and enable +# or disable ILM support. +#setup.ilm.enabled: auto + +# Configure the ILM write alias name. +#setup.ilm.rollover_alias: "filebeat" + +# Configure rollover index pattern. +#setup.ilm.pattern: "{now/d}-000001" + + +#============================== Kibana ====================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %} - #================================ Logging ====================================== {{ elk_macros.beat_logging('filebeat') }} - #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} @@ -936,3 +1393,13 @@ processors: # Port on which the HTTP endpoint will bind. Default is 5066. #http.port: 5066 + +#============================= Process Security ================================ + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false diff --git a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index f2def5b8..d18c5b8c 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -15,6 +15,17 @@ {% set _ = icmp_hosts.extend([hostvars[host_item]['ansible_host']]) %} {% endif %} {% endfor %} + +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + # Configure monitors heartbeat.monitors: - type: icmp # monitor type `icmp` (requires root) uses ICMP Echo Request to ping @@ -27,7 +38,7 @@ heartbeat.monitors: enabled: true # Configure task schedule using cron-like syntax - schedule: '@every 30s' # every 30 seconds from start of beat + schedule: '*/30 * * * * * *' # exactly every 30 seconds like 10:00:00, 10:00:30, ... # List of hosts to ping hosts: {{ (icmp_hosts | default([])) | to_json }} @@ -37,14 +48,6 @@ heartbeat.monitors: ipv6: true mode: any - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # Total running time per ping test. timeout: {{ icmp_hosts | length }}s @@ -100,13 +103,27 @@ heartbeat.monitors: # by sending/receiving a custom payload # Monitor name used for job name and document type + name: {{ item.name }} + + # Enable/Disable monitor + enabled: true + + # Configure task schedule + schedule: '@every 30s' # every 30 seconds from start of beat + + # configure hosts to ping. + # Entries can be: + # - plain host name or IP like `localhost`: + # Requires ports configs to be checked. If ssl is configured, + # a SSL/TLS based connection will be established. Otherwise plain tcp connection + # will be established name: "{{ item.name }}" # Enable/Disable monitor enabled: true # Configure task schedule - schedule: '@every 45s' # every 30 seconds from start of beat + schedule: '@every 45s' # every 5 seconds from start of beat # configure hosts to ping. # Entries can be: @@ -132,14 +149,6 @@ heartbeat.monitors: ipv6: true mode: any - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # List of ports to ping if host does not contain a port number # ports: [80, 9200, 5044] @@ -178,6 +187,15 @@ heartbeat.monitors: {% endfor %} {% endfor %} {% if hosts | length > 0 %} + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + - type: http # monitor type `http`. Connect via HTTP an optionally verify response # Monitor name used for job name and document type @@ -187,7 +205,7 @@ heartbeat.monitors: enabled: true # Configure task schedule - schedule: '@every 60s' # every 30 seconds from start of beat + schedule: '@every 60s' # every 5 seconds from start of beat # Configure URLs to ping urls: {{ (hosts | default([])) | to_json }} @@ -196,7 +214,7 @@ heartbeat.monitors: # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. ipv4: true ipv6: true - mode: "any" + mode: any # Configure file json file to be watched for changes to the monitor: #watch.poll_file: @@ -206,7 +224,7 @@ heartbeat.monitors: # Interval between file file changed checks. #interval: 5s - # Optional HTTP proxy url. If not set HTTP_PROXY environment variable will be used. + # Optional HTTP proxy url. #proxy_url: '' # Total test connection and data exchange timeout @@ -233,7 +251,6 @@ heartbeat.monitors: # Dictionary of additional HTTP headers to send: headers: User-agent: osa-heartbeat-healthcheck - # Optional request body content #body: @@ -255,6 +272,24 @@ heartbeat.monitors: {% endif %} {% endfor %} + # Parses the body as JSON, then checks against the given condition expression + #json: + #- description: Explanation of what the check does + # condition: + # equals: + # myField: expectedValue + + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + + heartbeat.scheduler: # Limit number of concurrent tasks executed by heartbeat. The task limit if # disabled if set to 0. The default is 0. @@ -347,7 +382,7 @@ heartbeat.scheduler: # Sets the write buffer size. #buffer_size: 1MiB - # Maximum duration after which events are flushed, if the write buffer + # Maximum duration after which events are flushed if the write buffer # is not full yet. The default value is 1s. #flush.timeout: 1s @@ -361,7 +396,7 @@ heartbeat.scheduler: #codec: cbor #read: # Reader flush timeout, waiting for more events to become available, so - # to fill a complete batch, as required by the outputs. + # to fill a complete batch as required by the outputs. # If flush_timeout is 0, all available events are forwarded to the # outputs immediately. # The default value is 0s. @@ -515,12 +550,15 @@ processors: # Set gzip compression level. #compression_level: 0 + # Configure escaping HTML symbols in strings. + #escape_html: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" - # Dictionary of HTTP parameters to pass within the url with index operations. + # Dictionary of HTTP parameters to pass within the URL with index operations. #parameters: #param1: value1 #param2: value2 @@ -531,19 +569,19 @@ processors: # Optional index name. The default is "heartbeat" plus date # and generates [heartbeat-]YYYY.MM.DD keys. # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. - #index: "heartbeat-%{[beat.version]}-%{+yyyy.MM.dd}" + #index: "heartbeat-%{[agent.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. #pipeline: "" - # Optional HTTP Path + # Optional HTTP path #path: "/elasticsearch" # Custom HTTP headers to add to each request #headers: # X-My-Header: Contents of the header - # Proxy server url + # Proxy server URL #proxy_url: http://proxy:3128 # The number of times a particular Elasticsearch index operation is attempted. If @@ -555,40 +593,50 @@ processors: # The default is 50. #bulk_max_size: 50 - # Configure http request timeout before failing an request to Elasticsearch. + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 # Use SSL settings for HTTPS. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are + # and certificates will be accepted. In this mode, SSL-based connections are # susceptible to man-in-the-middle attacks. Use only for testing. Default is # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - # SSL configuration. By default is off. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" - # Client Certificate Key + # Client certificate key #ssl.key: "/etc/pki/client/cert.key" - # Optional passphrase for decrypting the Certificate Key. + # Optional passphrase for decrypting the certificate key. #ssl.key_passphrase: '' # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites - #ssl.curve_types: [] + # Configure curve types for ECDHE-based cipher suites + # #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. @@ -603,7 +651,7 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] @@ -612,7 +660,7 @@ processors: # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats - # The Kafka event key setting. Use format string to create unique event key. + # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' @@ -633,28 +681,38 @@ processors: #username: '' #password: '' - # Kafka version heartbeat is assumed to run against. Defaults to the oldest - # supported stable version (currently version 0.8.2.0) - #version: 0.8.2 + # Kafka version heartbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' - # Metadata update configuration. Metadata do contain leader information - # deciding which broker to use when publishing. + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 - # Waiting time between retries during leader elections. Default is 250ms. + # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m + # Strategy for fetching the topics metadata from the broker. Default is true. + #full: true + # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. + # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. @@ -683,6 +741,10 @@ processors: # default is gzip. #compression: gzip + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + # The maximum permitted size of JSON-encoded messages. Bigger messages will be # dropped. The default value is 1000000 (bytes). This value should be equal to # or less than the broker's message.max.bytes. @@ -698,7 +760,7 @@ processors: # purposes. The default is "beats". #client_id: beats - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. @@ -711,7 +773,7 @@ processors: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -727,7 +789,7 @@ processors: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -739,20 +801,24 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Redis servers to connect to. If load balancing is enabled, the + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] - # The Redis port to use if hosts does not contain a port number. The default - # is 6379. - #port: 6379 - # The name of the Redis list or channel the events are published to. The # default is heartbeat. #key: heartbeat - # The password to authenticate with. The default is no authentication. + # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. @@ -786,6 +852,17 @@ processors: # until all events are published. The default is 3. #max_retries: 3 + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + # The maximum number of events to bulk in a single Redis request or pipeline. # The default is 2048. #bulk_max_size: 2048 @@ -842,11 +919,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false # Path to the directory where to save the generated files. The option is # mandatory. @@ -877,11 +954,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false #================================= Paths ====================================== @@ -916,9 +993,27 @@ processors: #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('heartbeat') }} -#=============================== Template ====================================== +#============================== Template ===================================== {{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +#============================== Setup ILM ===================================== + +# Configure Index Lifecycle Management Index Lifecycle Management creates a +# write alias and adds additional settings to the template. +# The elasticsearch.output.index setting will be replaced with the write alias +# if ILM is enabled. + +# Enabled ILM support. Valid values are true, false, and auto. The beat will +# detect availabilty of Index Lifecycle Management in Elasticsearch and enable +# or disable ILM support. +#setup.ilm.enabled: auto + +# Configure the ILM write alias name. +#setup.ilm.rollover_alias: "heartbeat" + +# Configure rollover index pattern. +#setup.ilm.pattern: "{now/d}-000001" + #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} @@ -949,3 +1044,389 @@ processors: # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false +################### Heartbeat Configuration Example ######################### + +# This file is a full configuration example documenting all non-deprecated +# options in comments. For a shorter configuration example, that contains +# only some common options, please see heartbeat.yml in the same directory. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/heartbeat/index.html + +############################# Heartbeat ###################################### +{% set icmp_hosts = [] %} +{% for host_item in groups['all'] %} +{% if hostvars[host_item]['ansible_host'] is defined %} +{% set _ = icmp_hosts.extend([hostvars[host_item]['ansible_host']]) %} +{% endif %} +{% endfor %} + +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + +# Configure monitors +heartbeat.monitors: +- type: icmp # monitor type `icmp` (requires root) uses ICMP Echo Request to ping + # configured hosts + + # Monitor name used for job name and document type. + name: icmp + + # Enable/Disable monitor + enabled: true + + # Configure task schedule using cron-like syntax + schedule: '*/5 * * * * * *' # exactly every 5 seconds like 10:00:00, 10:00:05, ... + + # List of hosts to ping + hosts: {{ (icmp_hosts | default([])) | to_json }} + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # Total running time per ping test. + timeout: {{ icmp_hosts | length }}s + + # Waiting duration until another ICMP Echo Request is emitted. + wait: 1s + + # The tags of the monitors are included in their own field with each + # transaction published. Tags make it easy to group servers by different + # logical properties. + #tags: ["service-X", "web-tier"] + + # Optional fields that you can specify to add additional information to the + # monitor output. Fields can be scalar values, arrays, dictionaries, or any nested + # combination of these. + #fields: + # env: staging + + # If this option is set to true, the custom fields are stored as top-level + # fields in the output document instead of being grouped under a fields + # sub-dictionary. Default is false. + #fields_under_root: false + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +# heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + #path: /path/to/my/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + #reload.enabled: true + # How often to check for changes + #reload.period: 1s + +{% for item in heartbeat_services %} +{% if item.type == 'tcp' %} +{% set hosts = [] %} +{% for port in item.ports | default([]) %} +{% for backend in item.group | default([]) %} +{% set backend_host = hostvars[backend]['ansible_host'] %} +{% set _ = hosts.extend([backend_host + ":" + (port | string)]) %} +{% endfor %} +{% endfor %} +{% if hosts | length > 0 %} +- type: tcp # monitor type `tcp`. Connect via TCP and optionally verify endpoint + # by sending/receiving a custom payload + + # Monitor name used for job name and document type + name: "{{ item.name }}" + + # Enable/Disable monitor + enabled: true + + # Configure task schedule + schedule: '@every 5s' # every 5 seconds from start of beat + + # configure hosts to ping. + # Entries can be: + # - plain host name or IP like `localhost`: + # Requires ports configs to be checked. If ssl is configured, + # a SSL/TLS based connection will be established. Otherwise plain tcp connection + # will be established + # - hostname + port like `localhost:12345`: + # Connect to port on given host. If ssl is configured, + # a SSL/TLS based connection will be established. Otherwise plain tcp connection + # will be established + # - full url syntax. `scheme://:[port]`. The `` can be one of + # `tcp`, `plain`, `ssl` and `tls`. If `tcp`, `plain` is configured, a plain + # tcp connection will be established, even if ssl is configured. + # Using `tls`/`ssl`, an SSL connection is established. If no ssl is configured, + # system defaults will be used (not supported on windows). + # If `port` is missing in url, the ports setting is required. + hosts: {{ (hosts | default([])) | to_json }} + + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # List of ports to ping if host does not contain a port number + # ports: [80, 9200, 5044] + + # Total test connection and data exchange timeout + #timeout: 16s + + # Optional payload string to send to remote and expected answer. If none is + # configured, the endpoint is expected to be up if connection attempt was + # successful. If only `send_string` is configured, any response will be + # accepted as ok. If only `receive_string` is configured, no payload will be + # send, but client expects to receive expected payload on connect. + #check: + #send: '' + #receive: '' + + # SOCKS5 proxy url + # proxy_url: '' + + # Resolve hostnames locally instead on SOCKS5 server: + #proxy_use_local_resolver: false + + # TLS/SSL connection settings: + #ssl: + # Certificate Authorities + #certificate_authorities: [''] + + # Required TLS protocols + #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] +{% endif %} +{% elif item.type == 'http' %} +{% set hosts = [] %} +{% for port in item.ports | default([]) %} +{% for backend in item.group | default([]) %} +{% set backend_host = hostvars[backend]['ansible_host'] %} +{% set _ = hosts.extend(["http://" + backend_host + ":" + (port | string) + item.path]) %} +{% endfor %} +{% endfor %} +{% if hosts | length > 0 %} + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + +- type: http # monitor type `http`. Connect via HTTP an optionally verify response + + # Monitor name used for job name and document type + name: "{{ item.name }}" + + # Enable/Disable monitor + enabled: true + + # Configure task schedule + schedule: '@every 5s' # every 5 seconds from start of beat + + # Configure URLs to ping + urls: {{ (hosts | default([])) | to_json }} + + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + + # Optional HTTP proxy url. + #proxy_url: '' + + # Total test connection and data exchange timeout + #timeout: 16s + + # Optional Authentication Credentials + #username: '' + #password: '' + + # TLS/SSL connection settings for use with HTTPS endpoint. If not configured + # system defaults will be used. + #ssl: + # Certificate Authorities + #certificate_authorities: [''] + + # Required TLS protocols + #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] + + # Request settings: + check.request: + # Configure HTTP method to use. Only 'HEAD', 'GET' and 'POST' methods are allowed. + method: "{{ item.method }}" + + # Dictionary of additional HTTP headers to send: + headers: + User-agent: osa-heartbeat-healthcheck + # Optional request body content + #body: + + # Expected response settings +{% if item.check_response is defined %} + check.response: {{ item.check_response }} + #check.response: + # Expected status code. If not configured or set to 0 any status code not + # being 404 is accepted. + #status: 0 + + # Required response headers. + #headers: + + # Required response contents. + #body: +{% endif %} +{% endif %} +{% endif %} +{% endfor %} + + # Parses the body as JSON, then checks against the given condition expression + #json: + #- description: Explanation of what the check does + # condition: + # equals: + # myField: expectedValue + + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + + +heartbeat.scheduler: + # Limit number of concurrent tasks executed by heartbeat. The task limit if + # disabled if set to 0. The default is 0. + #username: "beats_system" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the URL with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. The default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + #metrics.period: 10s + #state.period: 1m + +#================================ HTTP Endpoint ====================================== +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +#http.port: 5066 + +#============================= Process Security ================================ + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false diff --git a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index c00ac6fc..20013905 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -20,12 +20,12 @@ journalbeat.inputs: - paths: ["/var/log/journal"] # The number of seconds to wait before trying to read again from journals. - #backoff: 1s + backoff: 10s # The maximum number of seconds to wait before attempting to read again from journals. - #max_backoff: 60s + max_backoff: 60s # Position to start reading from journal. Valid values: head, tail, cursor - seek: cursor + seek: head # Fallback position if no cursor data is available. #cursor_seek_fallback: head @@ -46,17 +46,11 @@ journalbeat: # data path. registry_file: registry - # The number of seconds to wait before trying to read again from journals. - backoff: 10s - # The maximum number of seconds to wait before attempting to read again from journals. - max_backoff: 60s - - # Position to start reading from all journal. Possible values: head, tail, cursor - seek: head - - # Exact matching for field values of events. - # Matching for nginx entries: "systemd.unit=nginx" - #matches: [] +#==================== Elasticsearch template setting ========================== +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false #================================ General ====================================== @@ -143,7 +137,7 @@ tags: # Sets the write buffer size. #buffer_size: 1MiB - # Maximum duration after which events are flushed, if the write buffer + # Maximum duration after which events are flushed if the write buffer # is not full yet. The default value is 1s. #flush.timeout: 1s @@ -157,7 +151,7 @@ tags: #codec: cbor #read: # Reader flush timeout, waiting for more events to become available, so - # to fill a complete batch, as required by the outputs. + # to fill a complete batch as required by the outputs. # If flush_timeout is 0, all available events are forwarded to the # outputs immediately. # The default value is 0s. @@ -277,9 +271,9 @@ tags: # max_depth: 1 # target: "" # overwrite_keys: false -processors: -- add_host_metadata: ~ +processors: +- add_host_metadata: #============================= Elastic Cloud ================================== # These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -295,8 +289,7 @@ processors: #================================ Outputs ====================================== -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. +# Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- #output.elasticsearch: @@ -309,23 +302,18 @@ processors: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 #hosts: ["localhost:9200"] - # Enabled ilm (beta) to use index lifecycle management instead daily indices. - #ilm.enabled: false - #ilm.rollover_alias: "journalbeat" - #ilm.pattern: "{now/d}-000001" - # Set gzip compression level. #compression_level: 0 - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" #password: "changeme" - # Dictionary of HTTP parameters to pass within the url with index operations. + # Dictionary of HTTP parameters to pass within the URL with index operations. #parameters: #param1: value1 #param2: value2 @@ -336,19 +324,19 @@ processors: # Optional index name. The default is "journalbeat" plus date # and generates [journalbeat-]YYYY.MM.DD keys. # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. - #index: "journalbeat-%{[beat.version]}-%{+yyyy.MM.dd}" + #index: "journalbeat-%{[agent.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. #pipeline: "" - # Optional HTTP Path + # Optional HTTP path #path: "/elasticsearch" # Custom HTTP headers to add to each request #headers: # X-My-Header: Contents of the header - # Proxy server url + # Proxy server URL #proxy_url: http://proxy:3128 # The number of times a particular Elasticsearch index operation is attempted. If @@ -371,45 +359,45 @@ processors: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s - # Configure http request timeout before failing a request to Elasticsearch. + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 # Use SSL settings for HTTPS. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are + # and certificates will be accepted. In this mode, SSL-based connections are # susceptible to man-in-the-middle attacks. Use only for testing. Default is # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - # SSL configuration. By default is off. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" - # Client Certificate Key + # Client certificate key #ssl.key: "/etc/pki/client/cert.key" - # Optional passphrase for decrypting the Certificate Key. + # Optional passphrase for decrypting the certificate key. #ssl.key_passphrase: '' # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never + #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count, 'journalbeat') }} @@ -418,7 +406,7 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] @@ -427,7 +415,7 @@ processors: # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats - # The Kafka event key setting. Use format string to create unique event key. + # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' @@ -453,30 +441,33 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false - # Metadata update configuration. Metadata do contain leader information - # deciding which broker to use when publishing. + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 - # Waiting time between retries during leader elections. Default is 250ms. + # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m + # Strategy for fetching the topics metadata from the broker. Default is true. + #full: true + # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. + # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. @@ -524,7 +515,7 @@ processors: # purposes. The default is "beats". #client_id: beats - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. @@ -537,7 +528,7 @@ processors: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -553,7 +544,7 @@ processors: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -570,23 +561,19 @@ processors: # Pretty print json event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false - # The list of Redis servers to connect to. If load balancing is enabled, the + # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] - # The Redis port to use if hosts does not contain a port number. The default - # is 6379. - #port: 6379 - # The name of the Redis list or channel the events are published to. The # default is journalbeat. #key: journalbeat - # The password to authenticate with. The default is no authentication. + # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. @@ -687,11 +674,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false # Path to the directory where to save the generated files. The option is # mandatory. @@ -722,11 +709,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false #================================= Paths ====================================== @@ -761,9 +748,28 @@ processors: #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('journalbeat') }} -#=============================== Template ====================================== +#============================== Template ===================================== {{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} +#============================== Setup ILM ===================================== + +# Configure Index Lifecycle Management Index Lifecycle Management creates a +# write alias and adds additional settings to the template. +# The elasticsearch.output.index setting will be replaced with the write alias +# if ILM is enabled. + +# Enabled ILM support. Valid values are true, false, and auto. The beat will +# detect availabilty of Index Lifecycle Management in Elasticsearch and enable +# or disable ILM support. +#setup.ilm.enabled: auto + +# Configure the ILM write alias name. +#setup.ilm.rollover_alias: "journalbeat" + +# Configure rollover index pattern. +#setup.ilm.pattern: "{now/d}-000001" + + #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} @@ -772,7 +778,7 @@ processors: #================================ Logging ====================================== {{ elk_macros.beat_logging('journalbeat') }} -#============================== Xpack Monitoring =============================== +#============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} #================================ HTTP Endpoint ====================================== @@ -794,3 +800,8 @@ processors: # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false diff --git a/elk_metrics_7x/roles/elastic_kibana/templates/kibana.yml.j2 b/elk_metrics_7x/roles/elastic_kibana/templates/kibana.yml.j2 index e6e09ca1..cad996ac 100644 --- a/elk_metrics_7x/roles/elastic_kibana/templates/kibana.yml.j2 +++ b/elk_metrics_7x/roles/elastic_kibana/templates/kibana.yml.j2 @@ -1,92 +1,125 @@ # Kibana is served by a back end server. This setting specifies the port to use. - server.port: {{ kibana_port }} +server.port: {{ kibana_port }} -# This setting specifies the IP address of the back end server. - server.host: {{ kibana_interface }} +# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. +# The default is 'localhost', which usually means remote machines will not be able to connect. +# To allow connections from remote users, set this parameter to a non-loopback address. +server.host: {{ kibana_interface }} -# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This setting -# cannot end in a slash. -# server.basePath: "" +# Enables you to specify a path to mount Kibana at if you are running behind a proxy. +# Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath +# from requests it receives, and to prevent a deprecation warning at startup. +# This setting cannot end in a slash. +#server.basePath: "" + +# Specifies whether Kibana should rewrite requests that are prefixed with +# `server.basePath` or require that they are rewritten by your reverse proxy. +# This setting was effectively always `false` before Kibana 6.3 and will +# default to `true` starting in Kibana 7.0. +#server.rewriteBasePath: false # The maximum payload size in bytes for incoming server requests. -# server.maxPayloadBytes: 1048576 +#server.maxPayloadBytes: 1048576 -# The URL of the Elasticsearch instance to use for all your queries. - elasticsearch.url: "http://127.0.0.1:{{ elastic_port }}" +# The Kibana server's name. This is used for display purposes. +#server.name: "your-hostname" + +# The URLs of the Elasticsearch instances to use for all your queries. +elasticsearch.hosts: "http://127.0.0.1:{{ elastic_port }}" # When this setting's value is true Kibana uses the hostname specified in the server.host # setting. When the value of this setting is false, Kibana uses the hostname of the host # that connects to this Kibana instance. -# elasticsearch.preserveHost: true +#elasticsearch.preserveHost: true # Kibana uses an index in Elasticsearch to store saved searches, visualizations and # dashboards. Kibana creates a new index if the index doesn't already exist. -# kibana.index: ".kibana" +#kibana.index: ".kibana" # The default application to load. -# kibana.defaultAppId: "discover" +#kibana.defaultAppId: "discover" # If your Elasticsearch is protected with basic authentication, these settings provide # the username and password that the Kibana server uses to perform maintenance on the Kibana # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which # is proxied through the Kibana server. -# elasticsearch.username: "user" -# elasticsearch.password: "pass" +#elasticsearch.username: "user" +#elasticsearch.password: "pass" -# Paths to the PEM-format SSL certificate and SSL key files, respectively. These -# files enable SSL for outgoing requests from the Kibana server to the browser. -# server.ssl.cert: /path/to/your/server.crt -# server.ssl.key: /path/to/your/server.key +# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. +# These settings enable SSL for outgoing requests from the Kibana server to the browser. +#server.ssl.enabled: false +#server.ssl.certificate: /path/to/your/server.crt +#server.ssl.key: /path/to/your/server.key # Optional settings that provide the paths to the PEM-format SSL certificate and key files. # These files validate that your Elasticsearch backend uses the same key files. -# elasticsearch.ssl.cert: /path/to/your/client.crt -# elasticsearch.ssl.key: /path/to/your/client.key +#elasticsearch.ssl.certificate: /path/to/your/client.crt +#elasticsearch.ssl.key: /path/to/your/client.key # Optional setting that enables you to specify a path to the PEM file for the certificate # authority for your Elasticsearch instance. -# elasticsearch.ssl.ca: /path/to/your/CA.pem +#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ] -# To disregard the validity of SSL certificates, change this setting's value to false. -# elasticsearch.ssl.verify: true +# To disregard the validity of SSL certificates, change this setting's value to 'none'. +#elasticsearch.ssl.verificationMode: full # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of # the elasticsearch.requestTimeout setting. -# elasticsearch.pingTimeout: 1500 +#elasticsearch.pingTimeout: 1500 # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value # must be a positive integer. - elasticsearch.requestTimeout: {{ kibana_elastic_request_timeout }} +elasticsearch.requestTimeout: {{ kibana_elastic_request_timeout }} + +# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side +# headers, set this value to [] (an empty list). +#elasticsearch.requestHeadersWhitelist: [ authorization ] + +# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten +# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. +#elasticsearch.customHeaders: {} # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. -# elasticsearch.shardTimeout: 0 +#elasticsearch.shardTimeout: 30000 # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. -# elasticsearch.startupTimeout: 5000 +#elasticsearch.startupTimeout: 5000 + +# Logs queries sent to Elasticsearch. Requires logging.verbose set to true. +#elasticsearch.logQueries: false # Specifies the path where Kibana creates the process ID file. -# pid.file: /var/run/kibana.pid +#pid.file: /var/run/kibana.pid # Enables you specify a file where Kibana stores log output. - logging.dest: stdout +logging.dest: stdout # Set the value of this setting to true to suppress all logging output. -# logging.silent: false +#logging.silent: false # Set the value of this setting to true to suppress all logging output other than error messages. -# logging.quiet: false +#logging.quiet: false # Set the value of this setting to true to log all events, including system usage information # and all requests. -# logging.verbose: false +#logging.verbose: false + +# Set the interval in milliseconds to sample system and process performance +# metrics. Minimum is 100ms. Defaults to 5000. +#ops.interval: 5000 + +# Specifies locale to be used for all localizable strings, dates and number formats. +#i18n.locale: "en" # ---------------------------------- X-Pack ------------------------------------ # X-Pack Monitoring -# https://www.elastic.co/guide/en/kibana/6.3/monitoring-settings-kb.html - xpack.monitoring.enabled: true - xpack.xpack_main.telemetry.enabled: false - xpack.monitoring.kibana.collection.enabled: true - xpack.monitoring.kibana.collection.interval: 30000 - xpack.monitoring.min_interval_seconds: 30 - xpack.monitoring.ui.enabled: true - xpack.monitoring.ui.container.elasticsearch.enabled: true +# https://www.elastic.co/guide/en/kibana/7.0/monitoring-settings-kb.html +xpack.monitoring.enabled: true +xpack.xpack_main.telemetry.enabled: false +xpack.monitoring.kibana.collection.enabled: true +xpack.monitoring.kibana.collection.interval: 30000 +xpack.monitoring.min_interval_seconds: 30 +xpack.monitoring.ui.enabled: true +xpack.monitoring.ui.container.elasticsearch.enabled: true + diff --git a/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml b/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml index cbd9ca49..7da7af75 100644 --- a/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_logstash/defaults/main.yml @@ -52,7 +52,7 @@ logstash_deploy_filters: true # - server1.local:9092 # - server2.local:9092 # - server3.local:9092 -# client_id: "elk_metrics_6x" +# client_id: "elk_metrics_7x" # compression_type: "gzip" # security_protocol: "SSL" diff --git a/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 b/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 index f4d0559e..fd4ebc1c 100644 --- a/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 +++ b/elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2 @@ -38,7 +38,6 @@ path.data: /var/lib/logstash # # This defaults to the number of the host's CPU cores. # - {% set _d_processors = ((ansible_processor_count | int) * 3) %} {% set _processors = ((_d_processors | int) > 0) | ternary(_d_processors, 2) %} {% set _t_processors = (_processors | int) + (ansible_processor_count | int) %} @@ -225,14 +224,15 @@ path.logs: /var/log/logstash # Where to find custom plugins # path.plugins: [] # -# ---------------------------------- X-Pack ------------------------------------ +# ------------ X-Pack Settings (not applicable for OSS build)-------------- +# # X-Pack Monitoring # https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html xpack.monitoring.enabled: true #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: password -xpack.monitoring.elasticsearch.url: ["127.0.0.1:9200"] -#xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] +xpack.monitoring.elasticsearch.hosts: ["http://127.0.0.1:9200"] +#xpack.monitoring.elasticsearch.ssl.certificate_authority: [ "/path/to/ca.crt" ] #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file #xpack.monitoring.elasticsearch.ssl.truststore.password: password #xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file @@ -242,18 +242,19 @@ xpack.monitoring.elasticsearch.sniffing: {{ elastic_sniffing_enabled | default(f xpack.monitoring.collection.interval: 30s xpack.monitoring.collection.pipeline.details.enabled: true # -# ------------ X-Pack Settings (not applicable for OSS build)-------------- # X-Pack Management # https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html #xpack.management.enabled: false #xpack.management.pipeline.id: ["main", "apache_logs"] #xpack.management.elasticsearch.username: logstash_admin_user #xpack.management.elasticsearch.password: password -#xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] -#xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] +#xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"] +#xpack.management.elasticsearch.ssl.certificate_authority: [ "/path/to/ca.crt" ] #xpack.management.elasticsearch.ssl.truststore.path: /path/to/file #xpack.management.elasticsearch.ssl.truststore.password: password #xpack.management.elasticsearch.ssl.keystore.path: /path/to/file #xpack.management.elasticsearch.ssl.keystore.password: password -#xpack.management.elasticsearch.sniffing: {{ elastic_sniffing_enabled | default(false) }} +#xpack.management.elasticsearch.ssl.verification_mode: certificate +#xpack.management.elasticsearch.sniffing: false #xpack.management.logstash.poll_interval: 5s + diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index 1f9d9eff..f337cb70 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -44,9 +44,7 @@ metricbeat.max_start_delay: 10s # metricsets: ["leader", "self", "store"] # period: 30s # hosts: ["${host}:2379"] - #========================== Modules configuration ============================ - {% set metric_sets = ['network', 'process', 'process_summary', 'uptime'] %} {% if physical_host is defined and physical_host != inventory_hostname %} {% set host_mount_devices = (hostvars[physical_host]['ansible_mounts'] | map(attribute='device') | list) %} @@ -58,23 +56,24 @@ metricbeat.max_start_delay: 10s {% set _ = metric_sets.extend(['cpu', 'load', 'memory', 'core', 'diskio', 'raid', 'socket', 'filesystem', 'fsstat']) %} {% endif %} metricbeat.modules: + #------------------------------- System Module ------------------------------- -# metricsets: -# - cpu # CPU usage -# - filesystem # File system usage for each mountpoint -# - fsstat # File system summary metrics -# - load # CPU load averages -# - memory # Memory usage -# - network # Network IO -# - process # Per process metrics -# - process_summary # Process summary -# - uptime # System Uptime -# - core # Per CPU core usage -# - diskio # Disk IO -# - raid # Raid -# - socket # Sockets and connection info (linux only) - module: system metricsets: {{ metric_sets }} + #- cpu # CPU usage + #- load # CPU load averages + #- memory # Memory usage + #- network # Network IO + #- process # Per process metrics + #- process_summary # Process summary + #- uptime # System Uptime + #- socket_summary # Socket summary + #- core # Per CPU core usage + #- diskio # Disk IO + #- filesystem # File system usage for each mountpoint + #- fsstat # File system summary metrics + #- raid # Raid + #- socket # Sockets and connection info (linux only) enabled: true period: 60s processes: ['.*'] @@ -86,6 +85,9 @@ metricbeat.modules: # A list of filesystem types to ignore. The filesystem metricset will not # collect data from filesystems matching any of the specified types, and # fsstats will not include data from these filesystems in its summary stats. + # If not set, types associated to virtual filesystems are automatically + # added when this information is available in the system (e.g. the list of + # `nodev` types in `/proc/filesystem`). #filesystem.ignore_types: [] # These options allow you to filter out all processes that are not @@ -93,7 +95,7 @@ metricbeat.modules: # If both the `by_cpu` and `by_memory` options are used, the union of the two sets # is included. process.include_top_n: - # + # Set to false to disable this feature and include all processes enabled: true @@ -119,60 +121,88 @@ metricbeat.modules: # to false. #process.include_cpu_ticks: false + # Raid mount point to monitor + #raid.mount_point: '/' + # Configure reverse DNS lookup on remote IP addresses in the socket metricset. socket.reverse_lookup.enabled: true socket.reverse_lookup.success_ttl: 60s socket.reverse_lookup.failure_ttl: 60s -##------------------------------ Aerospike Module ----------------------------- + # Diskio configurations + #diskio.include_devices: [] + +#------------------------------ Aerospike Module ----------------------------- #- module: aerospike # metricsets: ["namespace"] -# enabled: false -# period: 30s +# enabled: true +# period: 10s # hosts: ["localhost:3000"] # -##------------------------------- Apache Module ------------------------------- +#------------------------------- Apache Module ------------------------------- {% if apache_enabled | default(false) | bool %} + - module: apache metricsets: ["status"] - enabled: true period: 30s -# -# # Apache hosts + enabled: true + + # Apache hosts hosts: ["http://127.0.1.1:18181"] -# -# # Path to server status. Default server-status -# #server_status_path: "server-status" -# -# # Username of hosts. Empty by default -# #username: username -# -# # Password of hosts. Empty by default -# #password: password + + # Path to server status. Default server-status + #server_status_path: "server-status" + + # Username of hosts. Empty by default + #username: username + + # Password of hosts. Empty by default + #password: password {% endif %} -# #-------------------------------- Ceph Module -------------------------------- {% if ceph_restapi_enabled | default(false) | bool %} - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] - enabled: true period: 30s hosts: {{ ceph_stats_hosts | to_json }} + enabled: true + {% endif %} -# -##------------------------------ Couchbase Module ----------------------------- +#------------------------------ Couchbase Module ----------------------------- #- module: couchbase # metricsets: ["bucket", "cluster", "node"] -# period: 30s +# period: 10s # hosts: ["localhost:8091"] +# enabled: true # -##------------------------------- Docker Module ------------------------------- +#------------------------------- couchdb Module ------------------------------ +- module: couchdb + metricsets: ["server"] + period: 10s + hosts: ["localhost:5984"] + +#------------------------------- Docker Module ------------------------------- {% if docker_enabled | default(false) | bool %} - module: docker - metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] - enabled: true + metricsets: + - "container" + - "cpu" + - "diskio" + - "event" + - "healthcheck" + - "info" + #- "image" + - "memory" + - "network" hosts: ["unix:///var/run/docker.sock"] period: 30s + enabled: true + + # If set to true, replace dots in labels with `_`. + #labels.dedot: false + + # If set to true, collects metrics per core. + #cpu.cores: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: @@ -180,16 +210,16 @@ metricbeat.modules: #certificate: "/etc/pki/client/cert.pem" #key: "/etc/pki/client/cert.key" {% endif %} -# -##----------------------------- Dropwizard Module ----------------------------- +#----------------------------- Dropwizard Module ----------------------------- #- module: dropwizard # metricsets: ["collector"] -# period: 30s +# period: 10s # hosts: ["localhost:8080"] # metrics_path: /metrics/metrics # namespace: example +# enabled: true # -##---------------------------- Elasticsearch Module --------------------------- +#---------------------------- Elasticsearch Module --------------------------- {% if inventory_hostname in (groups['elastic-logstash'] | union(groups['kibana'])) %} - module: elasticsearch metricsets: @@ -203,15 +233,24 @@ metricbeat.modules: enabled: true period: 30s hosts: ["localhost:{{ elastic_port }}"] -{% endif %} + #username: "elastic" + #password: "changeme" + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] -##----------------------------- envoyproxy Module ----------------------------- + # Set to false to fetch all entries + #index_recovery.active_only: true + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false +{% endif %} +#----------------------------- envoyproxy Module ----------------------------- - module: envoyproxy metricsets: ["server"] period: 10s hosts: ["localhost:9901"] -##-------------------------------- Etcd Module -------------------------------- +#-------------------------------- Etcd Module -------------------------------- {% if etcd_enabled | default(false) | bool %} - module: etcd metricsets: ["leader", "self", "store"] @@ -219,100 +258,154 @@ metricbeat.modules: period: 30s hosts: ["localhost:2379"] {% endif %} -# -##------------------------------- Golang Module ------------------------------- + +#------------------------------- Golang Module ------------------------------- #- module: golang -# metricsets: ["expvar","heap"] -# period: 30s + #metricsets: + # - expvar + # - heap +# period: 10s # hosts: ["localhost:6060"] # heap.path: "/debug/vars" # expvar: # namespace: "example" # path: "/debug/vars" # -##------------------------------ Graphite Module ------------------------------ +#------------------------------ Graphite Module ------------------------------ #- module: graphite # metricsets: ["server"] # enabled: true -## protocol: "udp" -## templates: -## - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats -## namespace: "test" -## template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash -## delimiter: "_" -# -# -##------------------------------- HAProxy Module ------------------------------ + + # Host address to listen on. Default localhost. + #host: localhost + + # Listening port. Default 2003. + #port: 2003 + + # Protocol to listen on. This can be udp or tcp. Default udp. + #protocol: "udp" + + # Receive buffer size in bytes + #receive_buffer_size: 1024 + + #templates: + # - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats + # namespace: "test" + # template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash + # delimiter: "_" + + +#------------------------------- HAProxy Module ------------------------------ {% if haproxy_enabled | default(false) | bool %} - module: haproxy metricsets: ["info", "stat"] - enabled: true period: 30s hosts: [ {{ elastic_metricbeat_haproxy_monitoring_hosts }} ] + enabled: true {% endif %} -# -##-------------------------------- HTTP Module -------------------------------- -#- module: http -# metricsets: ["json"] -# period: 30s -# hosts: ["localhost:80"] -# namespace: "json_namespace" -# path: "/" -# #body: "" -# #method: "GET" -# #request.enabled: false -# #response.enabled: false -# #dedot.enabled: false -# -##------------------------------- Jolokia Module ------------------------------ +#-------------------------------- HTTP Module -------------------------------- +#- module: http + #metricsets: + # - json + #period: 10s + #hosts: ["localhost:80"] + #namespace: "json_namespace" + #path: "/" + #body: "" + #method: "GET" + #username: "user" + #password: "secret" + #request.enabled: false + #response.enabled: false + #json.is_array: false + #dedot.enabled: false + +- module: http + #metricsets: + # - server + host: "localhost" + port: "8080" + enabled: false + #paths: + # - path: "/foo" + # namespace: "foo" + # fields: # added to the the response in root. overwrites existing fields + # key: "value" + +#------------------------------- Jolokia Module ------------------------------ #- module: jolokia -# metricsets: ["jmx"] -# period: 30s -# hosts: ["localhost"] -# namespace: "metrics" -# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" -# jmx.mapping: -# jmx.application: -# jmx.instance: -# -##-------------------------------- Kafka Module ------------------------------- + #metricsets: ["jmx"] + #period: 10s + #hosts: ["localhost"] + #namespace: "metrics" + #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + #username: "user" + #password: "secret" + #jmx.mappings: + #- mbean: 'java.lang:type=Runtime' + # attributes: + # - attr: Uptime + # field: uptime + #- mbean: 'java.lang:type=Memory' + # attributes: + # - attr: HeapMemoryUsage + # field: memory.heap_usage + # - attr: NonHeapMemoryUsage + # field: memory.non_heap_usage + # GC Metrics - this depends on what is available on your JVM + #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + # attributes: + # - attr: CollectionTime + # field: gc.cms_collection_time + # - attr: CollectionCount + # field: gc.cms_collection_count + + #jmx.application: + #jmx.instance: + +#-------------------------------- Kafka Module ------------------------------- #- module: kafka -# metricsets: ["partition"] -# period: 30s -# hosts: ["localhost:9092"] -# -# #client_id: metricbeat -# #retries: 3 -# #backoff: 250ms -# -# # List of Topics to query metadata for. If empty, all topics will be queried. -# #topics: [] -# -# # Optional SSL. By default is off. -# # List of root certificates for HTTPS server verifications -# #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] -# -# # Certificate for SSL client authentication -# #ssl.certificate: "/etc/pki/client/cert.pem" -# -# # Client Certificate Key -# #ssl.key: "/etc/pki/client/cert.key" -# -# # SASL authentication -# #username: "" -# #password: "" -# -##------------------------------- Kibana Module ------------------------------- + #metricsets: ["consumergroup", "partition"] + #period: 10s + #hosts: ["localhost:9092"] + #enabled: true + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" + +#------------------------------- Kibana Module ------------------------------- {% if inventory_hostname in groups['kibana'] | default([]) %} - module: kibana metricsets: ["status"] - enabled: true period: 30s - hosts: ["localhost:{{ kibana_port }}"] + hosts: ["localhost:5601"] + basepath: "" + enabled: true + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false {% endif %} -# -##----------------------------- Kubernetes Module ----------------------------- +#----------------------------- Kubernetes Module ----------------------------- # Node metrics, from kubelet: #- module: kubernetes # metricsets: @@ -357,20 +450,20 @@ metricbeat.modules: # #host: node_name # #kube_config: ~/.kube/config # -## Kubernetes events +# Kubernetes events #- module: kubernetes # enabled: true # metricsets: # - event # -## Kubernetes API server +# Kubernetes API server #- module: kubernetes # enabled: true # metricsets: # - apiserver # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] -##--------------------------------- kvm Module -------------------------------- +#--------------------------------- kvm Module -------------------------------- {% if kvm_enabled | default(false) | bool %} - module: kvm metricsets: ["dommemstat"] @@ -385,16 +478,16 @@ metricbeat.modules: #timeout: 1s {% endif %} -##------------------------------ Logstash Module ------------------------------ +#------------------------------ Logstash Module ------------------------------ {% if inventory_hostname in groups['elastic-logstash'] | default([]) %} - module: logstash metricsets: ["node", "node_stats"] enabled: true - period: 30s + period: 10s hosts: ["localhost:9600"] {% endif %} -# -##------------------------------ Memcached Module ----------------------------- + +#------------------------------ Memcached Module ----------------------------- {% if memcached_enabled | default(false) | bool %} - module: memcached metricsets: ["stats"] @@ -402,12 +495,13 @@ metricbeat.modules: period: 30s hosts: ["{{ ansible_hostname }}:11211"] {% endif %} -# -##------------------------------- MongoDB Module ------------------------------ + +#------------------------------- MongoDB Module ------------------------------ #- module: mongodb -# metricsets: ["dbstats", "status"] -# period: 30s -# +# metricsets: ["dbstats", "status", "collstats", "metrics", "replstatus"] +# period: 10s +# enabled: true + # # The hosts must be passed as MongoDB URLs in the format: # # [mongodb://][user:pass@]host[:port]. # # The username and password can also be set using the respective configuration @@ -415,44 +509,77 @@ metricbeat.modules: # # password configuration options. # hosts: ["localhost:27017"] # +# # Optional SSL. By default is off. +# #ssl.enabled: true +# +# # Mode of verification of server certificate ('none' or 'full') +# #ssl.verification_mode: 'full' +# +# # List of root certificates for TLS server verifications +# #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] +# +# # Certificate for SSL client authentication +# #ssl.certificate: "/etc/pki/client/cert.pem" +# +# # Client Certificate Key +# #ssl.key: "/etc/pki/client/cert.key" +# # # Username to use when connecting to MongoDB. Empty by default. # #username: user # # # Password to use when connecting to MongoDB. Empty by default. # #password: pass # -##-------------------------------- Munin Module ------------------------------- +#-------------------------------- Munin Module ------------------------------- #- module: munin # metricsets: ["node"] # enabled: true # period: 10s # hosts: ["localhost:4949"] -# node.namespace: node -# -##-------------------------------- MySQL Module ------------------------------- + + # List of plugins to collect metrics from, by default it collects from + # all the available ones. + #munin.plugins: [] + + # If set to true, it sanitizes fields names in concordance with munin + # implementation (all characters that are not alphanumeric, or underscore + # are replaced by underscores). + #munin.sanitize: false + +#-------------------------------- MySQL Module ------------------------------- {% if (mysql_enabled | default(false) | bool) and galera_root_user is defined and galera_root_password is defined %} - module: mysql - metricsets: ["status"] + metricsets: + - "status" + # - "galera_status" enabled: true period: 30s -# -# # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" -# # The username and password can either be set in the DSN or using the username -# # and password config options. Those specified in the DSN take precedence. + + # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" + # The username and password can either be set in the DSN or using the username + # and password config options. Those specified in the DSN take precedence. hosts: ["{{ galera_root_user }}:{{ galera_root_password }}@tcp({{ ansible_hostname }}:3306)/"] -# -# # Username of hosts. Empty by default. + + # Username of hosts. Empty by default. username: {{ galera_root_user }} -# -# # Password of hosts. Empty by default. + + # Password of hosts. Empty by default. password: {{ galera_root_password }} -# -# # By setting raw to true, all raw fields from the status metricset will be added to the event. -# #raw: false -# + + # By setting raw to true, all raw fields from the status metricset will be added to the event. + #raw: false {% endif %} -# -##-------------------------------- Nginx Module ------------------------------- +#-------------------------------- Nats Module -------------------------------- +- module: nats + metricsets: ["connections", "routes", "stats", "subscriptions"] + period: 10s + hosts: ["localhost:8222"] + #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" + #routes.metrics_path: "/routez" + #subscriptions.metrics_path: "/subsz" + +#-------------------------------- Nginx Module ------------------------------- {% if nginx_enabled | default(false) | bool %} - module: nginx metricsets: ["stubstatus"] @@ -465,16 +592,19 @@ metricbeat.modules: # Path to server status. Default server-status server_status_path: "server-status" {% endif %} -# -##------------------------------- PHP_FPM Module ------------------------------ +#------------------------------- PHP_FPM Module ------------------------------ #- module: php_fpm -# metricsets: ["pool"] -# period: 30s +# metricsets: +# - pool +# #- process +# enabled: true +# period: 10s # status_path: "/status" # hosts: ["localhost:8080"] -# -##----------------------------- PostgreSQL Module ----------------------------- + +#----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql +# enabled: true # metricsets: # # Stats about every PostgreSQL database # - database @@ -498,8 +628,8 @@ metricbeat.modules: # # # Password to use when connecting to PostgreSQL. Empty by default. # #password: pass -# -##----------------------------- Prometheus Module ----------------------------- + +#----------------------------- Prometheus Module ----------------------------- {% if (prometheus_enabled | default(false) | bool) and (prometheus_config is defined) %} {% for prometheus in prometheus_config %} - module: prometheus @@ -521,22 +651,34 @@ metricbeat.modules: metrics_path: "/metrics" namespace: ceph {% endif %} -# -##------------------------------ RabbitMQ Module ------------------------------ + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + +#------------------------------ RabbitMQ Module ------------------------------ {% if (rabbitmq_enabled | default(false) | bool) and (rabbitmq_monitoring_password is defined) %} - module: rabbitmq - metricsets: ["node", "queue"] + metricsets: ["node", "queue", "connection"] enabled: true period: 30s hosts: [ {{ elastic_metricbeat_rabbitmq_monitoring_hosts }} ] username: {{ rabbitmq_monitoring_userid | default('monitoring') }} password: {{ rabbitmq_monitoring_password }} {% endif %} -# -##-------------------------------- Redis Module ------------------------------- + # Management path prefix, if `management.path_prefix` is set in RabbitMQ + # configuration, it has to be set to the same value. + #management_path_prefix: "" + + #username: guest + #password: guest + +#-------------------------------- Redis Module ------------------------------- #- module: redis # metricsets: ["info", "keyspace"] -# period: 30s +# enabled: true +# period: 10s # # # Redis hosts # hosts: ["127.0.0.1:6379"] @@ -557,21 +699,20 @@ metricbeat.modules: # #maxconn: 10 # # # Filters can be used to reduce the number of fields sent. -# #processors: -# # - include_fields: -# # fields: ["beat", "metricset", "redis.info.stats"] -# -# # Redis AUTH password. Empty by default. -# #password: foobared -# + #processors: + # - include_fields: + # fields: ["beat", "metricset", "redis.info.stats"] -##------------------------------- traefik Module ------------------------------ + # Redis AUTH password. Empty by default. + #password: foobared + +#------------------------------- traefik Module ------------------------------ - module: traefik metricsets: ["health"] period: 10s hosts: ["localhost:8080"] -##-------------------------------- uwsgi Module ------------------------------- +#-------------------------------- uwsgi Module ------------------------------- {% if uwsgi_enabled | default(false) | bool %} - module: uwsgi metricsets: ["status"] @@ -583,11 +724,12 @@ metricbeat.modules: hosts: ["tcp://127.0.0.1:9191"] {% endif %} {% endif %} -# -##------------------------------- vSphere Module ------------------------------ + +#------------------------------- vSphere Module ------------------------------ #- module: vsphere +# enabled: true # metricsets: ["datastore", "host", "virtualmachine"] -# period: 30s +# period: 10s # hosts: ["https://localhost/sdk"] # # username: "user" @@ -596,26 +738,32 @@ metricbeat.modules: # insecure: false # # Get custom fields when using virtualmachine metric set. Default false. # # get_custom_fields: false -# -# -##------------------------------- Windows Module ------------------------------ + +#------------------------------- Windows Module ------------------------------ #- module: windows # metricsets: ["perfmon"] -# period: 30s +# enabled: true +# period: 10s +# perfmon.ignore_non_existent_counters: true # perfmon.counters: +# # - instance_label: processor.name +# # instance_name: total +# # measurement_label: processor.time.total.pct +# # query: '\Processor Information(_Total)\% Processor Time' # #- module: windows # metricsets: ["service"] +# enabled: true # period: 60s # -##------------------------------ ZooKeeper Module ----------------------------- +#------------------------------ ZooKeeper Module ----------------------------- #- module: zookeeper -# metricsets: ["mntr"] -# period: 30s +# enabled: true +# metricsets: ["mntr", "server"] +# period: 10s # hosts: ["localhost:2181"] -# -# -# + + #================================ General ====================================== @@ -701,7 +849,7 @@ metricbeat.modules: # Sets the write buffer size. #buffer_size: 1MiB - # Maximum duration after which events are flushed, if the write buffer + # Maximum duration after which events are flushed if the write buffer # is not full yet. The default value is 1s. #flush.timeout: 1s @@ -715,7 +863,7 @@ metricbeat.modules: #codec: cbor #read: # Reader flush timeout, waiting for more events to become available, so - # to fill a complete batch, as required by the outputs. + # to fill a complete batch as required by the outputs. # If flush_timeout is 0, all available events are forwarded to the # outputs immediately. # The default value is 0s. @@ -838,19 +986,19 @@ metricbeat.modules: processors: - add_host_metadata: ~ -##============================= Elastic Cloud ================================== -# -## These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). -# -## The cloud.id setting overwrites the `output.elasticsearch.hosts` and -## `setup.kibana.host` options. -## You can find the `cloud.id` in the Elastic Cloud web UI. -##cloud.id: -# -## The cloud.auth setting overwrites the `output.elasticsearch.username` and -## `output.elasticsearch.password` settings. The format is `:`. -##cloud.auth: -# +#============================= Elastic Cloud ================================== + +# These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + #================================ Outputs ====================================== # Configure what output to use when sending the data collected by the beat. @@ -869,12 +1017,15 @@ processors: # # Set gzip compression level. # #compression_level: 0 # +# # Configure escaping HTML symbols in strings. +# #escape_html: false +# # # Optional protocol and basic auth credentials. # #protocol: "https" # #username: "elastic" # #password: "changeme" # -# # Dictionary of HTTP parameters to pass within the url with index operations. +# # Dictionary of HTTP parameters to pass within the URL with index operations. # #parameters: # #param1: value1 # #param2: value2 @@ -885,19 +1036,19 @@ processors: # # Optional index name. The default is "metricbeat" plus date # # and generates [metricbeat-]YYYY.MM.DD keys. # # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. -# #index: "metricbeat-%{[beat.version]}-%{+yyyy.MM.dd}" +# #index: "metricbeat-%{[agent.version]}-%{+yyyy.MM.dd}" # # # Optional ingest node pipeline. By default no pipeline will be used. # #pipeline: "" # -# # Optional HTTP Path +# # Optional HTTP path # #path: "/elasticsearch" # # # Custom HTTP headers to add to each request # #headers: # # X-My-Header: Contents of the header # -# # Proxy server url +# # Proxy server URL # #proxy_url: http://proxy:3128 # # # The number of times a particular Elasticsearch index operation is attempted. If @@ -909,55 +1060,64 @@ processors: # # The default is 50. # #bulk_max_size: 50 # -# # Configure http request timeout before failing an request to Elasticsearch. +# # The number of seconds to wait before trying to reconnect to Elasticsearch +# # after a network error. After waiting backoff.init seconds, the Beat +# # tries to reconnect. If the attempt fails, the backoff timer is increased +# # exponentially up to backoff.max. After a successful connection, the backoff +# # timer is reset. The default is 1s. +# #backoff.init: 1s +# +# # The maximum number of seconds to wait before attempting to connect to +# # Elasticsearch after a network error. The default is 60s. +# #backoff.max: 60s +# +# # Configure HTTP request timeout before failing a request to Elasticsearch. # #timeout: 90 # # # Use SSL settings for HTTPS. # #ssl.enabled: true # # # Configure SSL verification mode. If `none` is configured, all server hosts -# # and certificates will be accepted. In this mode, SSL based connections are +# # and certificates will be accepted. In this mode, SSL-based connections are # # susceptible to man-in-the-middle attacks. Use only for testing. Default is # # `full`. # #ssl.verification_mode: full # -# # List of supported/valid TLS versions. By default all TLS versions 1.0 up to +# # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # # 1.2 are enabled. # #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] # -# # SSL configuration. By default is off. # # List of root certificates for HTTPS server verifications # #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # # Certificate for SSL client authentication # #ssl.certificate: "/etc/pki/client/cert.pem" # -# # Client Certificate Key +# # Client certificate key # #ssl.key: "/etc/pki/client/cert.key" # -# # Optional passphrase for decrypting the Certificate Key. +# # Optional passphrase for decrypting the certificate key. # #ssl.key_passphrase: '' # # # Configure cipher suites to be used for SSL connections # #ssl.cipher_suites: [] # -# # Configure curve types for ECDHE based cipher suites +# # Configure curve types for ECDHE-based cipher suites # #ssl.curve_types: [] # # # Configure what types of renegotiation are supported. Valid options are # # never, once, and freely. Default is never. # #ssl.renegotiation: never -# -# + + #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count) }} - #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] @@ -966,7 +1126,7 @@ processors: # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats - # The Kafka event key setting. Use format string to create unique event key. + # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' @@ -987,28 +1147,38 @@ processors: #username: '' #password: '' - # Kafka version metricbeat is assumed to run against. Defaults to the oldest - # supported stable version (currently version 0.8.2.0) - #version: 0.8.2 + # Kafka version metricbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' - # Metadata update configuration. Metadata do contain leader information - # deciding which broker to use when publishing. + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 - # Waiting time between retries during leader elections. Default is 250ms. + # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m + # Strategy for fetching the topics metadata from the broker. Default is true. + #full: true + # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. + # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. @@ -1037,6 +1207,10 @@ processors: # default is gzip. #compression: gzip + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + # The maximum permitted size of JSON-encoded messages. Bigger messages will be # dropped. The default value is 1000000 (bytes). This value should be equal to # or less than the broker's message.max.bytes. @@ -1052,7 +1226,7 @@ processors: # purposes. The default is "beats". #client_id: beats - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. @@ -1065,7 +1239,7 @@ processors: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -1081,7 +1255,7 @@ processors: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -1093,20 +1267,24 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Redis servers to connect to. If load balancing is enabled, the + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] - # The Redis port to use if hosts does not contain a port number. The default - # is 6379. - #port: 6379 - # The name of the Redis list or channel the events are published to. The # default is metricbeat. #key: metricbeat - # The password to authenticate with. The default is no authentication. + # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. @@ -1140,6 +1318,17 @@ processors: # until all events are published. The default is 3. #max_retries: 3 + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + # The maximum number of events to bulk in a single Redis request or pipeline. # The default is 2048. #bulk_max_size: 2048 @@ -1194,6 +1383,14 @@ processors: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/metricbeat" @@ -1223,11 +1420,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false #================================= Paths ====================================== @@ -1262,10 +1459,29 @@ processors: #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('metricbeat') }} -#=============================== Template ====================================== +#============================== Template ===================================== {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} + +#============================== Setup ILM ===================================== -#================================ Kibana ======================================= +# Configure Index Lifecycle Management Index Lifecycle Management creates a +# write alias and adds additional settings to the template. +# The elasticsearch.output.index setting will be replaced with the write alias +# if ILM is enabled. + +# Enabled ILM support. Valid values are true, false, and auto. The beat will +# detect availabilty of Index Lifecycle Management in Elasticsearch and enable +# or disable ILM support. +#setup.ilm.enabled: auto + +# Configure the ILM write alias name. +#setup.ilm.rollover_alias: "metricbeat" + +# Configure rollover index pattern. +#setup.ilm.pattern: "{now/d}-000001" + + +#============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %} @@ -1273,10 +1489,10 @@ processors: #================================ Logging ====================================== {{ elk_macros.beat_logging('metricbeat') }} -#============================== Xpack Monitoring =============================== +#============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} -#================================ HTTP Endpoint ================================ +#================================ HTTP Endpoint ====================================== # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1295,3 +1511,8 @@ processors: # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false diff --git a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index aa67a6f4..43fb9b8f 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -25,13 +25,13 @@ packetbeat.interfaces.type: af_packet # large enough for almost all networks and interface types. If you sniff on a # physical network interface, the optimal setting is the MTU size. On virtual # interfaces, however, it's safer to accept the default value. -packetbeat.interfaces.snaplen: 65535 +#packetbeat.interfaces.snaplen: 65535 # The maximum size of the shared memory buffer to use between the kernel and # user space. A bigger buffer usually results in lower CPU usage, but consumes # more memory. This setting is only available for the af_packet sniffer type. # The default is 30 MB. -packetbeat.interfaces.buffer_size_mb: 30 +#packetbeat.interfaces.buffer_size_mb: 30 # Packetbeat automatically generates a BPF for capturing only the traffic on # ports where it expects to find known protocols. Use this settings to tell @@ -99,8 +99,6 @@ packetbeat.protocols: #transaction_timeout: 10s - type: cassandra - # Enable cassandra monitoring. Default: false - enabled: false #Cassandra port for traffic monitoring. ports: [9042] @@ -134,7 +132,7 @@ packetbeat.protocols: - type: dns # Enable DNS monitoring. Default: true - enabled: true + #enabled: true # Configure the ports where to listen for DNS traffic. You can disable # the DNS protocol by commenting out the list of ports. @@ -164,6 +162,7 @@ packetbeat.protocols: - type: http # Enable HTTP monitoring. Default: true + #enabled: true {% set used_ports = [53, 443, 2049, 3306, 5432, 5672, 6379, 9042, 9090, 11211, 27017] %} {% set ports = [] %} {% for item in heartbeat_services %} @@ -173,7 +172,6 @@ packetbeat.protocols: {% endif %} {% endfor %} {% endfor %} - enabled: true # Configure the ports where to listen for HTTP traffic. You can disable # the HTTP protocol by commenting out the list of ports. @@ -196,9 +194,22 @@ packetbeat.protocols: send_all_headers: true # The list of content types for which Packetbeat includes the full HTTP - # payload in the response field. + # payload. If the request's or response's Content-Type matches any on this + # list, the full body will be included under the request or response field. #include_body_for: [] + # The list of content types for which Packetbeat includes the full HTTP + # request payload. + #include_request_body_for: [] + + # The list of content types for which Packetbeat includes the full HTTP + # response payload. + #include_response_body_for: [] + + # Whether the body of a request must be decoded when a content-encoding + # or transfer-encoding has been applied. + #decode_body: true + # If the Cookie or Set-Cookie headers are sent, this option controls whether # they are split into individual values. #split_cookie: false @@ -226,7 +237,7 @@ packetbeat.protocols: - type: memcache # Enable memcache monitoring. Default: true - enabled: true + #enabled: true # Configure the ports where to listen for memcache traffic. You can disable # the Memcache protocol by commenting out the list of ports. @@ -275,11 +286,11 @@ packetbeat.protocols: - type: mysql # Enable mysql monitoring. Default: true - enabled: true + #enabled: true # Configure the ports where to listen for MySQL traffic. You can disable # the MySQL protocol by commenting out the list of ports. - ports: [3306] + ports: [3306,3307] # If this option is enabled, the raw message of the request (`request` field) # is sent to Elasticsearch. The default is false. @@ -440,15 +451,26 @@ packetbeat.protocols: - type: tls # Enable TLS monitoring. Default: true - enabled: true + #enabled: true # Configure the ports where to listen for TLS traffic. You can disable # the TLS protocol by commenting out the list of ports. - ports: [443] + ports: + - 443 # HTTPS + - 993 # IMAPS + - 995 # POP3S + - 5223 # XMPP over SSL + - 8443 + - 8883 # Secure MQTT + - 9243 # Elasticsearch + + # List of hash algorithms to use to calculate certificates' fingerprints. + # Valid values are `sha1`, `sha256` and `md5`. + #fingerprints: [sha1] # If this option is enabled, the client and server certificates and # certificate chains are sent to Elasticsearch. The default is true. - send_certificates: true + #send_certificates: true # If this option is enabled, the raw certificates will be stored # in PEM format under the `raw` key. The default is false. @@ -456,33 +478,17 @@ packetbeat.protocols: #=========================== Monitored processes ============================== -# Configure the processes to be monitored and how to find them. If a process is -# monitored then Packetbeat attempts to use it's name to fill in the `proc` and -# `client_proc` fields. -# The processes can be found by searching their command line by a given string. -# -# Process matching is optional and can be enabled by uncommenting the following -# lines. -# -#packetbeat.procs: -# enabled: false -# monitored: -# - process: mysqld -# cmdline_grep: mysqld -# -# - process: pgsql -# cmdline_grep: postgres -# -# - process: nginx -# cmdline_grep: nginx -# -# - process: app -# cmdline_grep: gunicorn +# Packetbeat can enrich events with information about the process associated +# the socket that sent or received the packet if Packetbeat is monitoring +# traffic from the host machine. By default process enrichment is disabled. +# This feature works on Linux and Windows. +packetbeat.procs.enabled: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#packetbeat.ignore_outgoing: true +# If you want to ignore transactions created by the server on which the shipper +# is installed you can enable this option. This option is useful to remove +# duplicates if shippers are installed on multiple servers. Default value is +# false. +packetbeat.ignore_outgoing: false #================================ General ====================================== @@ -568,7 +574,7 @@ packetbeat.protocols: # Sets the write buffer size. #buffer_size: 1MiB - # Maximum duration after which events are flushed, if the write buffer + # Maximum duration after which events are flushed if the write buffer # is not full yet. The default value is 1s. #flush.timeout: 1s @@ -582,7 +588,7 @@ packetbeat.protocols: #codec: cbor #read: # Reader flush timeout, waiting for more events to become available, so - # to fill a complete batch, as required by the outputs. + # to fill a complete batch as required by the outputs. # If flush_timeout is 0, all available events are forwarded to the # outputs immediately. # The default value is 0s. @@ -736,12 +742,15 @@ processors: # # Set gzip compression level. # #compression_level: 0 # +# # Configure escaping HTML symbols in strings. +# #escape_html: false +# # # Optional protocol and basic auth credentials. # #protocol: "https" # #username: "elastic" # #password: "changeme" # -# # Dictionary of HTTP parameters to pass within the url with index operations. +# # Dictionary of HTTP parameters to pass within the URL with index operations. # #parameters: # #param1: value1 # #param2: value2 @@ -752,19 +761,19 @@ processors: # # Optional index name. The default is "packetbeat" plus date # # and generates [packetbeat-]YYYY.MM.DD keys. # # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. -# #index: "packetbeat-%{[beat.version]}-%{+yyyy.MM.dd}" +# #index: "packetbeat-%{[agent.version]}-%{+yyyy.MM.dd}" # # # Optional ingest node pipeline. By default no pipeline will be used. # #pipeline: "" # -# # Optional HTTP Path +# # Optional HTTP path # #path: "/elasticsearch" # # # Custom HTTP headers to add to each request # #headers: # # X-My-Header: Contents of the header # -# # Proxy server url +# # Proxy server URL # #proxy_url: http://proxy:3128 # # # The number of times a particular Elasticsearch index operation is attempted. If @@ -776,55 +785,64 @@ processors: # # The default is 50. # #bulk_max_size: 50 # -# # Configure http request timeout before failing an request to Elasticsearch. +# # The number of seconds to wait before trying to reconnect to Elasticsearch +# # after a network error. After waiting backoff.init seconds, the Beat +# # tries to reconnect. If the attempt fails, the backoff timer is increased +# # exponentially up to backoff.max. After a successful connection, the backoff +# # timer is reset. The default is 1s. +# #backoff.init: 1s +# +# # The maximum number of seconds to wait before attempting to connect to +# # Elasticsearch after a network error. The default is 60s. +# #backoff.max: 60s +# +# # Configure HTTP request timeout before failing a request to Elasticsearch. # #timeout: 90 # # # Use SSL settings for HTTPS. # #ssl.enabled: true # # # Configure SSL verification mode. If `none` is configured, all server hosts -# # and certificates will be accepted. In this mode, SSL based connections are +# # and certificates will be accepted. In this mode, SSL-based connections are # # susceptible to man-in-the-middle attacks. Use only for testing. Default is # # `full`. # #ssl.verification_mode: full # -# # List of supported/valid TLS versions. By default all TLS versions 1.0 up to +# # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # # 1.2 are enabled. # #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] # -# # SSL configuration. By default is off. # # List of root certificates for HTTPS server verifications # #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # # Certificate for SSL client authentication # #ssl.certificate: "/etc/pki/client/cert.pem" # -# # Client Certificate Key +# # Client certificate key # #ssl.key: "/etc/pki/client/cert.key" # -# # Optional passphrase for decrypting the Certificate Key. +# # Optional passphrase for decrypting the certificate key. # #ssl.key_passphrase: '' # # # Configure cipher suites to be used for SSL connections # #ssl.cipher_suites: [] # -# # Configure curve types for ECDHE based cipher suites +# # Configure curve types for ECDHE-based cipher suites # #ssl.curve_types: [] # # # Configure what types of renegotiation are supported. Valid options are # # never, once, and freely. Default is never. # #ssl.renegotiation: never - +# #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count) }} - #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true - # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] @@ -833,7 +851,7 @@ processors: # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats - # The Kafka event key setting. Use format string to create unique event key. + # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' @@ -859,30 +877,33 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false - # Metadata update configuration. Metadata do contain leader information - # deciding which broker to use when publishing. + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 - # Waiting time between retries during leader elections. Default is 250ms. + # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m + # Strategy for fetching the topics metadata from the broker. Default is true. + #full: true + # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. + # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. @@ -930,7 +951,7 @@ processors: # purposes. The default is "beats". #client_id: beats - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. @@ -943,7 +964,7 @@ processors: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -959,7 +980,7 @@ processors: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are @@ -976,23 +997,19 @@ processors: # Pretty print json event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false - # The list of Redis servers to connect to. If load balancing is enabled, the + # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] - # The Redis port to use if hosts does not contain a port number. The default - # is 6379. - #port: 6379 - # The name of the Redis list or channel the events are published to. The # default is packetbeat. #key: packetbeat - # The password to authenticate with. The default is no authentication. + # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. @@ -1093,11 +1110,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false # Path to the directory where to save the generated files. The option is # mandatory. @@ -1128,11 +1145,11 @@ processors: # Configure JSON encoding #codec.json: - # Pretty print json event + # Pretty-print JSON event #pretty: false - # Configure escaping html symbols in strings. - #escape_html: true + # Configure escaping HTML symbols in strings. + #escape_html: false #================================= Paths ====================================== @@ -1167,10 +1184,29 @@ processors: #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('packetbeat') }} -#=============================== Template ====================================== +#============================== Template ===================================== {{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} -#================================ Kibana ======================================= +#============================== Setup ILM ===================================== + +# Configure Index Lifecycle Management Index Lifecycle Management creates a +# write alias and adds additional settings to the template. +# The elasticsearch.output.index setting will be replaced with the write alias +# if ILM is enabled. + +# Enabled ILM support. Valid values are true, false, and auto. The beat will +# detect availabilty of Index Lifecycle Management in Elasticsearch and enable +# or disable ILM support. +#setup.ilm.enabled: auto + +# Configure the ILM write alias name. +#setup.ilm.rollover_alias: "packetbeat" + +# Configure rollover index pattern. +#setup.ilm.pattern: "{now/d}-000001" + + +#============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %} @@ -1178,7 +1214,7 @@ processors: #================================ Logging ====================================== {{ elk_macros.beat_logging('packetbeat') }} -#============================== Xpack Monitoring =============================== +#============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} #================================ HTTP Endpoint ====================================== @@ -1200,3 +1236,8 @@ processors: # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true + +#================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false diff --git a/elk_metrics_7x/roles/elastic_repositories/vars/ubuntu.yml b/elk_metrics_7x/roles/elastic_repositories/vars/ubuntu.yml index 90019c53..a1716e26 100644 --- a/elk_metrics_7x/roles/elastic_repositories/vars/ubuntu.yml +++ b/elk_metrics_7x/roles/elastic_repositories/vars/ubuntu.yml @@ -18,6 +18,6 @@ elastic_repo_distro_packages: # elk apt repo elastic_repo: - repo: 'deb https://artifacts.elastic.co/packages/6.x/apt stable main' + repo: 'deb https://artifacts.elastic.co/packages/7.x/apt stable main' state: "{{ ((elk_package_state | default('present')) == 'absent') | ternary('absent', 'present') }}" key_url: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" diff --git a/elk_metrics_7x/roles/elastic_retention/defaults/main.yml b/elk_metrics_7x/roles/elastic_retention/defaults/main.yml deleted file mode 100644 index 8b8e1da0..00000000 --- a/elk_metrics_7x/roles/elastic_retention/defaults/main.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -elastic_index_retention_algorithm: default - -### Elastic curator variables -## If any of these retention policy option are undefined a dynamic fact will be -## generated. -## These options are all in days. -# elastic_logstash_retention: 1 -# elastic_apm_retention: 1 -# elastic_auditbeat_retention: 1 -# elastic_filebeat_retention: 1 -# elastic_heartbeat_retention: 1 -# elastic_journalbeat_retention: 1 -# elastic_metricbeat_retention: 1 -# elastic_packetbeat_retention: 1 -# elastic_skydive_retention: 1 - -## These options are all in megabytes. -# elastic_logstash_size: 1024 -# elastic_apm_size: 1024 -# elastic_auditbeat_size: 1024 -# elastic_filebeat_size: 1024 -# elastic_heartbeat_size: 1024 -# elastic_journalbeat_size: 1024 -# elastic_metricbeat_size: 1024 -# elastic_packetbeat_size: 1024 -# elastic_skydive_size: 1024 - -## WHen a static retention policy option is not defined these options will be -## used for dynamic fact generation. -## -## Facts will be generated for the general retention using the total available -## storage from the ES data nodes, subtracting 25%. Using the weights, each -## index will be given a percentage of the total available storage. Indexes with -## higher weights are expected to use more storage. The list of hosts in a given -## index will be used to determine the number of days data can exist within an -## index before it's pruned. - -## Example: -# es cluster has 4TiB of storage -# filebeat is deployed to 100 hosts -# filebeat has a weight of 10 -# metricbeat is deployed to 125 hosts -# metricbeat has a weight of 2 -# -# es storage in MiB: 4194304 -# hosts and weighting total: (100 + 125) x (10 + 2) = 2700 -# filebeat pct: (100 x 10) / 2700 = 0.37 -# filebeat storage allowed: 0.37 * 4194304 = 1551892.48 MiB -# filebeat days allowed: 1551892.48 / (100 * 1024) = 15.1552 Days -# filebeat result: 15 days of retention or 1.5TiB of storage, whatever comes first -# metricbeat pct: (125 x 2) / 2700 = 0.09 -# metricbeat storage allowed: 0.09 * 4194304 = 377487.36 MiB -# metricbeat days allowed: 377487.36 / (125 * 1024) = 2.94912 Days -# metricbeat result: 2 days of retention or 38GiB of storage, whatever comes first - -elastic_beat_retention_policy_hosts: - logstash: - make_index: true - weight: 1 - hosts: "{{ groups['elastic-logstash'] | default([]) }}" - apm: - make_index: true - timeFieldName: '@timestamp' - weight: 1 - hosts: "{{ groups['apm-server'] | default([]) }}" - auditbeat: - timeFieldName: '@timestamp' - weight: 10 - hosts: "{{ groups['hosts'] | default([]) }}" - filebeat: - timeFieldName: '@timestamp' - weight: 10 - hosts: "{{ groups['hosts'] | default([]) }}" - syslog: - make_index: true - weight: 1 - hosts: "{{ groups['hosts'] | default([]) }}" - heartbeat: - timeFieldName: '@timestamp' - weight: 1 - hosts: "{{ groups['kibana'][:3] | default([]) }}" - journalbeat: - timeFieldName: '@timestamp' - weight: 3 - hosts: "{{ groups['hosts'] | default([]) }}" - metricbeat: - timeFieldName: '@timestamp' - weight: 2 - hosts: "{{ groups['all'] | default([]) }}" - packetbeat: - timeFieldName: '@timestamp' - weight: 1 - hosts: "{{ groups['hosts'] | default([]) }}" - monitorstack: - timeFieldName: '@timestamp' - weight: 1 - hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}" - skydive: - weight: 1 - hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}" - -# Refresh the elasticsearch retention policy local facts. -elastic_retention_refresh: false diff --git a/elk_metrics_7x/roles/elastic_retention/meta/main.yml b/elk_metrics_7x/roles/elastic_retention/meta/main.yml deleted file mode 100644 index ab69fa29..00000000 --- a/elk_metrics_7x/roles/elastic_retention/meta/main.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -galaxy_info: - author: OpenStack - description: Elastic v6.x retention role - company: Rackspace - license: Apache2 - min_ansible_version: 2.5 - platforms: - - name: Ubuntu - versions: - - trusty - - xenial - - bionic - categories: - - cloud - - development - - elasticsearch - - elastic-stack -dependencies: - - role: elastic_data_hosts diff --git a/elk_metrics_7x/roles/elastic_retention/tasks/main.yml b/elk_metrics_7x/roles/elastic_retention/tasks/main.yml deleted file mode 100644 index e80e8db3..00000000 --- a/elk_metrics_7x/roles/elastic_retention/tasks/main.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Ensure local facts directory exists - file: - dest: "/etc/ansible/facts.d" - state: directory - group: "root" - owner: "root" - mode: "0755" - recurse: no - -- name: Initialize local facts - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "retention" - option: cacheable - value: true - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Retention storage block - block: - - name: Query es storage - uri: - url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" - method: GET - register: elk_data - environment: - no_proxy: "{{ coordination_nodes[0].split(':')[0] }}" - until: - - elk_data is success and elk_data['json'] is defined - retries: 5 - delay: 30 - run_once: true - - - name: Set retention keys fact - set_fact: - es_storage_json: "{{ elk_data['json'] }}" - - - name: Load retention algo variables - include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" - tags: - - always - - - name: Set storage fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "retention" - option: "cluster_nodes" - value: "{{ groups['elastic-logstash'] | length }}" - - - name: Set retention policy keys fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "retention" - option: "elastic_beat_retention_policy_keys" - value: "{{ elastic_beat_retention_policy_hosts.keys() | list | sort }}" - - - name: Set size fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "retention" - option: "elastic_{{ item.key }}_size" - value: "{{ item.value }}" - with_dict: "{{ es_storage_per_index }}" - - - name: Set retention fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "retention" - option: "elastic_{{ item.key }}_retention" - value: "{{ item.value }}" - with_dict: "{{ es_days_per_index }}" - - - name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - when: - - (ansible_local['elastic']['retention']['cluster_nodes'] is undefined) or - ((groups['elastic-logstash'] | length) != (ansible_local['elastic']['retention']['cluster_nodes'] | int)) or - ((ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] is defined) and - ((ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) != (elastic_beat_retention_policy_hosts.keys() | list | sort))) or - (elastic_retention_refresh | bool) diff --git a/elk_metrics_7x/roles/elastic_retention/vars/calculate_index_retention_default.yml b/elk_metrics_7x/roles/elastic_retention/vars/calculate_index_retention_default.yml deleted file mode 100644 index 886cf5df..00000000 --- a/elk_metrics_7x/roles/elastic_retention/vars/calculate_index_retention_default.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set available storage fact. This tasks the total amount of storage found -# within the data nodes of the elasticsearch cluster and converts bytes to -# megabytes. -es_total_available_storage: "{{ ((es_storage_json['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}" - -# Set assumed buffer storage fact. This will result in 25% of the total -# available storage. -es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}" - -# Set usable buffer storage fact(s). This is the toal storage minus the buffer. -es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}" - -# This function will take the sum total of all hosts in the retention policy -# after weighting. Once the policy is set the sum total will be carved up into -# individual percentages of the total amount of usable storage after the buffer -# is calculated. -es_storage_per_index: |- - {%- set es_hash = {} %} - {%- set total_weight = (elastic_beat_retention_policy_hosts.values() | list | map(attribute='weight') | list | sum) %} - {%- set host_count = (elastic_beat_retention_policy_hosts.values() | list | map(attribute='hosts') | list | map('flatten') | list | length) %} - {%- set total_values = (total_weight | int) * (host_count | int) %} - {%- for key, value in elastic_beat_retention_policy_hosts.items() %} - {%- set value_pct = (((value.weight | int) * (value.hosts | length)) / (total_values | int)) %} - {%- set value_total = ((value_pct | float) * (es_usable_buffer_storage | int)) %} - {%- set _ = es_hash.__setitem__(key, value_total | int) %} - {%- endfor %} - {{ es_hash }} - -# The assumed number of days an index will be retained is based on the size of -# the given index. With the sizes all figured out in the function above this -# function will divide each retention size be a constant of 1024 and the number -# of hosts within a given collector segment. -es_days_per_index: |- - {%- set es_hash = {} %} - {%- for key, value in elastic_beat_retention_policy_hosts.items() %} - {%- if (es_storage_per_index[key] | int) > 0 %} - {%- set value_days = ((es_storage_per_index[key] | int) // ((value.hosts | length) * 1024)) %} - {%- set _ = es_hash.__setitem__(key, ((value_days | int) > 0) | ternary(value_days, 1) ) %} - {%- else %} - {%- set _ = es_hash.__setitem__(key, 1) %} - {%- endif %} - {%- endfor %} - {{ es_hash }} diff --git a/elk_metrics_7x/roles/elastic_rollup/defaults/main.yml b/elk_metrics_7x/roles/elastic_rollup/defaults/main.yml index f72162fd..6dad3bdd 100644 --- a/elk_metrics_7x/roles/elastic_rollup/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_rollup/defaults/main.yml @@ -14,3 +14,4 @@ # limitations under the License. elastic_allow_rollup_purge: false +days_until_rollup: 15 diff --git a/elk_metrics_7x/roles/elastic_rollup/meta/main.yml b/elk_metrics_7x/roles/elastic_rollup/meta/main.yml index 1c53bb47..ecb098b6 100644 --- a/elk_metrics_7x/roles/elastic_rollup/meta/main.yml +++ b/elk_metrics_7x/roles/elastic_rollup/meta/main.yml @@ -30,5 +30,3 @@ galaxy_info: - development - elasticsearch - elastic-stack -dependencies: - - role: elastic_retention diff --git a/elk_metrics_7x/roles/elastic_rollup/tasks/main.yml b/elk_metrics_7x/roles/elastic_rollup/tasks/main.yml index 96caefdd..a93f4a86 100644 --- a/elk_metrics_7x/roles/elastic_rollup/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_rollup/tasks/main.yml @@ -40,21 +40,6 @@ - name: Create rollup block block: - - name: Set min retention days fact - set_fact: - min_days_until_rollup: |- - {% set index_retention = [] %} - {% for item in ansible_play_hosts %} - {% set _ = index_retention.append(ansible_local['elastic']['retention']['elastic_' + index_name + '_retention'] | int) %} - {% endfor %} - {{ index_retention | min }} - run_once: true - - - name: Set retention days fact - set_fact: - days_until_rollup: "{{ ((min_days_until_rollup | int) > 1) | ternary(((min_days_until_rollup | int) - 1), min_days_until_rollup) }}" - run_once: true - - name: Create rollup job uri: url: "{{ item.url }}" diff --git a/elk_metrics_7x/roles/elasticsearch/templates/elasticsearch.yml.j2 b/elk_metrics_7x/roles/elasticsearch/templates/elasticsearch.yml.j2 index 3abdee83..c46ab471 100644 --- a/elk_metrics_7x/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/elk_metrics_7x/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,74 +1,102 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# # ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# cluster.name: {{ cluster_name }} +# # ------------------------------------ Node ------------------------------------ -node.name: {{ ansible_nodename }} -# node.rack: r1 +# +# Use a descriptive name for the node: +# +# ansible_nodename may be appropriate for your instance +# If you're having issues with bootstrap skipping, check this. +node.name: {{ inventory_hostname }} +# +# Add custom attributes to the node: # Set to true to enable machine learning on the node. node.ml: false +# # ----------------------------------- Paths ------------------------------------ +# # Path to directory where to store the data (separate multiple locations by comma): # -# path.data: /path/to/data path.data: /var/lib/elasticsearch # # Path to log files: # -# -# Path to log files: -# -# path.logs: /path/to/logs -#path.logs: /var/lib/elasticsearch/logs/ path.logs: /var/log/elasticsearch/ # # Path to shared filesystem repos # -# path.repo: ["/mount/backups", "/mount/longterm_backups"] -# - {% if elastic_shared_fs_repos is defined and elastic_shared_fs_repos|length > 0 %} path.repo: {{ elastic_shared_fs_repos | json_query("[*].path") | to_json }} {% endif %} - +# # Set the global default index store. More information on these settings can be # found here: # +# index.store.type: niofs - +# # ----------------------------------- Memory ----------------------------------- # # Lock the memory on startup: # bootstrap.memory_lock: {{ elastic_memory_lock }} # -# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory -# available on the system and that the owner of the process is allowed to use this limit. +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. # # Elasticsearch performs poorly when the system is swapping the memory. # # ---------------------------------- Network ----------------------------------- # # Set the bind address to a specific IP (IPv4 or IPv6): +# network.host: ["127.0.0.1", "{{ ansible_host }}", "{{ ansible_hostname }}"] {% if elasticsearch_publish_host is defined %} network.publish_host: "{{ elasticsearch_publish_host }}" {% endif %} +# # Set a custom port for HTTP: - +# http.port: {{ elastic_port }} +# +# For more information, consult the network module documentation. +# # --------------------------------- Discovery ---------------------------------- # -# Pass an initial list of hosts to perform discovery when new node is started: +# Pass an initial list of hosts to perform discovery when this node is started: # The default list of hosts is ["127.0.0.1", "[::1]"] # -# Node definitions can be seen here: -# -discovery.zen.ping.unicast.hosts: {{ zen_nodes | to_json }} -# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): -discovery.zen.minimum_master_nodes: {{ elasticsearch_master_node_count | default(((master_node_count | int) // 2) + 1) }} +discovery.seed_hosts: {{ zen_nodes | to_json }} +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +cluster.initial_master_nodes: {{ master_nodes | to_json }} +# +# For more information, consult the discovery and cluster formation module documentation. +# # The first set of nodes in the master_node_count are marked as such +# node.master: {{ elasticsearch_node_master | default(master_node) }} # Every node in the master list and every other node after will be a data node +# node.data: {{ elasticsearch_node_data | default(data_node) }} +# # Ingest nodes can execute pre-processing pipelines. To override automatic # determination, the option `elasticsearch_node_ingest` can be defined as a # Boolean which will enable or disable ingest nodes. When using automatic @@ -76,16 +104,14 @@ node.data: {{ elasticsearch_node_data | default(data_node) }} # # NOTE(cloudnull): The use of "search remote connect" will follow the enablement # of an ingest nodes. +# {% if elasticsearch_node_ingest is defined %} node.ingest: {{ elasticsearch_node_ingest }} -search.remote.connect: {{ elasticsearch_node_ingest }} +cluster.remote.connect: {{ elasticsearch_node_ingest }} {% else %} node.ingest: {{ data_node }} -search.remote.connect: {{ data_node }} +cluster.remote.connect: {{ data_node }} {% endif %} - -# For more information, see the documentation at: -# # # ---------------------------------- Gateway ----------------------------------- # @@ -93,15 +119,10 @@ search.remote.connect: {{ data_node }} # gateway.recover_after_nodes: {{ elasticsearch_master_node_count | default(((master_node_count | int) // 2) + 1) }} # -# For more information, see the documentation at: -# +# For more information, consult the gateway module documentation. # # ---------------------------------- Various ----------------------------------- # -# Disable starting multiple nodes on a single system: -# -# node.max_local_storage_nodes: 1 -# # Require explicit names when deleting indices: # action.destructive_requires_name: true @@ -111,8 +132,6 @@ action.destructive_requires_name: true # Thread pool settings. For more on this see the documentation at: # thread_pool: - index: - queue_size: {{ (processors | int) * 256 }} get: queue_size: {{ (processors | int) * 256 }} write: @@ -139,8 +158,9 @@ indices.recovery.max_bytes_per_sec: {{ elasticserch_interface_speed }}mb # ---------------------------------- X-Pack ------------------------------------ # X-Pack Monitoring -# https://www.elastic.co/guide/en/elasticsearch/reference/6.3/monitoring-settings.html +# xpack.monitoring.collection.enabled: true xpack.monitoring.collection.interval: 30s # Set to true to enable machine learning on the node. xpack.ml.enabled: false + diff --git a/elk_metrics_7x/site-elka.yml b/elk_metrics_7x/site-elka.yml index bc02fce9..0ffc365e 100644 --- a/elk_metrics_7x/site-elka.yml +++ b/elk_metrics_7x/site-elka.yml @@ -13,6 +13,4 @@ - import_playbook: installElastic.yml - import_playbook: installLogstash.yml -- import_playbook: installCurator.yml - import_playbook: installKibana.yml -- import_playbook: installAPMserver.yml diff --git a/elk_metrics_7x/templates/_macros.j2 b/elk_metrics_7x/templates/_macros.j2 index 8e280588..9147af4a 100644 --- a/elk_metrics_7x/templates/_macros.j2 +++ b/elk_metrics_7x/templates/_macros.j2 @@ -107,6 +107,9 @@ output.logstash: # Set gzip compression level. compression_level: 3 + # Configure escaping HTML symbols in strings. + #escape_html: false + # Optional maximum time to live for a connection to Logstash, after which the # connection will be re-established. A value of `0s` (the default) will # disable this feature. @@ -114,10 +117,10 @@ output.logstash: # Not yet supported for async connections (i.e. with the "pipelining" option set) #ttl: 30s - # Optional load balance the events between the Logstash hosts. Default is false. + # Optionally load-balance events between Logstash hosts. Default is false. loadbalance: true - # Number of batches to be sent asynchronously to logstash while processing + # Number of batches to be sent asynchronously to Logstash while processing # new batches. pipelining: 2 @@ -126,33 +129,30 @@ output.logstash: # if no error is encountered. slow_start: true - # The maximum number of events to bulk in a single Logstash request. The - # default is the number of cores multiplied by the number of threads, - # the resultant is then multiplied again by 128 which results in a the defined - # bulk max size. If the Beat sends single events, the events are collected - # into batches. If the Beat publishes a large batch of events (larger than - # the value specified by bulk_max_size), the batch is split. Specifying a - # larger batch size can improve performance by lowering the overhead of - # sending events. However big batch sizes can also increase processing times, - # which might result in API errors, killed connections, timed-out publishing - # requests, and, ultimately, lower throughput. Setting bulk_max_size to values - # less than or equal to 0 disables the splitting of batches. When splitting - # is disabled, the queue decides on the number of events to be contained in a - # batch. - bulk_max_size: {{ (processors | int) * 128 }} + # The number of seconds to wait before trying to reconnect to Logstash + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s -{% if named_index is defined %} - # Optional index name. The default index name is set to {{ named_index }} + # The maximum number of seconds to wait before attempting to connect to + # Logstash after a network error. The default is 60s. + #backoff.max: 60s + + # Optional index name. The default index name is set to journalbeat # in all lowercase. +{% if named_index is defined %} index: '{{ named_index }}' {% endif %} + # SOCKS5 proxy server URL #proxy_url: socks5://user:password@socks5-server:2233 # Resolve names locally when using a proxy server. Defaults to false. #proxy_use_local_resolver: false - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts @@ -161,7 +161,7 @@ output.logstash: # `full`. #ssl.verification_mode: full - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] @@ -172,7 +172,7 @@ output.logstash: # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" - # Client Certificate Key + # Client certificate key #ssl.key: "/etc/pki/client/cert.key" # Optional passphrase for decrypting the Certificate Key. @@ -181,12 +181,27 @@ output.logstash: # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] - # Configure curve types for ECDHE based cipher suites + # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + bulk_max_size: {{ (processors | int) * 128 }} + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s {%- endmacro %} {% macro setup_dashboards(beat_name) -%} @@ -254,10 +269,19 @@ setup.template.pattern: "{{ beat_name }}-%{[beat.version]}-*" # Path to fields.yml file to generate the template setup.template.fields: "${path.config}/fields.yml" +# Enable JSON template loading. If this is enabled, the fields.yml is ignored. +#setup.template.json.enabled: false + +# Path to the JSON template file +#setup.template.json.path: "${path.config}/template.json" + +# Name under which the template is stored in Elasticsearch +#setup.template.json.name: "" + # Overwrite existing template setup.template.overwrite: {{ host == data_nodes[0] }} -{% set shards = ((data_nodes | length) * 3) | int %} +{% set shards = 1 %} # Elasticsearch template settings setup.template.settings: @@ -443,6 +467,17 @@ xpack.monitoring.elasticsearch: # The default is 50. bulk_max_size: {{ (processors | int) * 64 }} + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + # Configure http request timeout before failing an request to Elasticsearch. timeout: 120 @@ -481,4 +516,7 @@ xpack.monitoring.elasticsearch: # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never + + #metrics.period: 10s + #state.period: 1m {%- endmacro %} diff --git a/elk_metrics_7x/templates/jvm.options.j2 b/elk_metrics_7x/templates/jvm.options.j2 index f2e76559..75c3482c 100644 --- a/elk_metrics_7x/templates/jvm.options.j2 +++ b/elk_metrics_7x/templates/jvm.options.j2 @@ -26,16 +26,9 @@ ################################################################ ## GC Configuration -{% if ((heap_size | int) > 6144) and (elastic_g1gc_enabled | bool) %} -XX:+UseG1GC -XX:MaxGCPauseMillis=400 -XX:InitiatingHeapOccupancyPercent=75 -{% else %} --XX:+UseParNewGC --XX:+UseConcMarkSweepGC --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly -{% endif %} ## optimizations diff --git a/elk_metrics_7x/templates/logstash-pipelines.yml.j2 b/elk_metrics_7x/templates/logstash-pipelines.yml.j2 index 000627c5..6bc6fcc7 100644 --- a/elk_metrics_7x/templates/logstash-pipelines.yml.j2 +++ b/elk_metrics_7x/templates/logstash-pipelines.yml.j2 @@ -2,7 +2,7 @@ # For more information on multiple pipelines, see the documentation: # https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html -- pipeline.id: "elk_metrics_6x" +- pipeline.id: "elk_metrics_7x" queue.type: "persisted" config.string: | input { @@ -498,7 +498,7 @@ hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} - index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" + index => "%{[@metadata][beat]}-%{[@metadata][version]}" } } else if [@metadata][beat] { elasticsearch { @@ -544,7 +544,7 @@ hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"] sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }} manage_template => {{ (data_node | bool) | lower }} - index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" + index => "%{[@metadata][beat]}-%{[@metadata][version]}" } } else if [@metadata][beat] { elasticsearch { diff --git a/elk_metrics_7x/tests/ansible-role-requirements.yml b/elk_metrics_7x/tests/ansible-role-requirements.yml index 85ba806d..6ecfc2d8 100644 --- a/elk_metrics_7x/tests/ansible-role-requirements.yml +++ b/elk_metrics_7x/tests/ansible-role-requirements.yml @@ -1,33 +1,33 @@ --- - name: apt_package_pinning scm: git - src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning + src: https://opendev.org/openstack/openstack-ansible-apt_package_pinning version: master - name: config_template scm: git - src: https://git.openstack.org/openstack/ansible-config_template + src: https://opendev.org/openstack/ansible-config_template version: master - name: nspawn_container_create scm: git - src: https://git.openstack.org/openstack/openstack-ansible-nspawn_container_create + src: https://opendev.org/openstack/openstack-ansible-nspawn_container_create version: master - name: nspawn_hosts scm: git - src: https://git.openstack.org/openstack/openstack-ansible-nspawn_hosts + src: https://opendev.org/openstack/openstack-ansible-nspawn_hosts version: master - name: plugins scm: git - src: https://git.openstack.org/openstack/openstack-ansible-plugins + src: https://opendev.org/openstack/openstack-ansible-plugins version: master - name: systemd_mount scm: git - src: https://git.openstack.org/openstack/ansible-role-systemd_mount + src: https://opendev.org/openstack/ansible-role-systemd_mount version: master - name: systemd_networkd scm: git - src: https://git.openstack.org/openstack/ansible-role-systemd_networkd + src: https://opendev.org/openstack/ansible-role-systemd_networkd version: master - name: systemd_service scm: git - src: https://git.openstack.org/openstack/ansible-role-systemd_service + src: https://opendev.org/openstack/ansible-role-systemd_service version: master diff --git a/elk_metrics_7x/tests/functional.yml b/elk_metrics_7x/tests/functional.yml index 9858d5fd..bfb9162d 100644 --- a/elk_metrics_7x/tests/functional.yml +++ b/elk_metrics_7x/tests/functional.yml @@ -25,7 +25,7 @@ ZUUL_PROJECT: "{{ zuul.project.short_name }}" ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}" ANSIBLE_HOST_KEY_CHECKING: "False" - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" ANSIBLE_ACTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/config_template/action" ANSIBLE_CONNECTION_PLUGINS: "${HOME}/ansible_venv/repositories/roles/plugins/connection" ANSIBLE_ROLES_PATH: "${HOME}/ansible_venv/repositories/roles" @@ -63,15 +63,15 @@ reload: "yes" sysctl_file: /etc/sysctl.d/99-elasticsearch.conf - - name: Create tmp elk_metrics_6x dir + - name: Create tmp elk_metrics_7x dir file: - path: "/tmp/elk-metrics-6x-logs" + path: "/tmp/elk-metrics-7x-logs" state: directory - name: Flush iptables rules command: "{{ item }}" args: - creates: "/tmp/elk-metrics-6x-logs/iptables.flushed" + creates: "/tmp/elk-metrics-7x-logs/iptables.flushed" with_items: - "iptables -F" - "iptables -X" @@ -82,7 +82,7 @@ - "iptables -P INPUT ACCEPT" - "iptables -P FORWARD ACCEPT" - "iptables -P OUTPUT ACCEPT" - - "touch /tmp/elk-metrics-6x-logs/iptables.flushed" + - "touch /tmp/elk-metrics-7x-logs/iptables.flushed" - name: First ensure apt cache is always refreshed apt: @@ -96,30 +96,30 @@ become_user: root command: "./bootstrap-embedded-ansible.sh" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x" - name: Run ansible-galaxy (tests) become: yes become_user: root command: "${HOME}/ansible_venv/bin/ansible-galaxy install --force --ignore-errors --roles-path=${HOME}/ansible_venv/repositories/roles -r ansible-role-requirements.yml" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x/tests" - - name: Run ansible-galaxy (elk_metrics_6x) + - name: Run ansible-galaxy (elk_metrics_7x) become: yes become_user: root command: "${HOME}/ansible_venv/bin/ansible-galaxy install --force --ignore-errors --roles-path=${HOME}/ansible_venv/repositories/roles -r ansible-role-requirements.yml" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x" - name: Run environment setup become: yes become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _key-setup.yml" environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test-container-setup.log" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x/tests" when: - ansible_service_mgr != 'systemd' or not (container_inventory | bool) @@ -129,9 +129,9 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i {{ inventory_file }} -e @test-vars.yml _container-setup.yml" environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-container-setup.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test-container-setup.log" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x/tests" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x/tests" when: - ansible_service_mgr == 'systemd' - container_inventory | bool @@ -147,15 +147,15 @@ become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml site.yml" environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-deployment.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test-deployment.log" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x" - name: Show cluster state become: yes become_user: root command: "${HOME}/ansible_venv/bin/ansible-playbook -i tests/{{ inventory_file }} -e @tests/test-vars.yml showElasticCluster.yml" environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test-show-cluster.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test-show-cluster.log" args: - chdir: "src/{{ current_test_repo }}/elk_metrics_6x" + chdir: "src/{{ current_test_repo }}/elk_metrics_7x" diff --git a/elk_metrics_7x/tests/manual-test.rc b/elk_metrics_7x/tests/manual-test.rc index 2222ac0c..72940d96 100644 --- a/elk_metrics_7x/tests/manual-test.rc +++ b/elk_metrics_7x/tests/manual-test.rc @@ -2,11 +2,11 @@ export ANSIBLE_HOST_KEY_CHECKING="False" export ANSIBLE_ROLES_PATH="${HOME}/ansible_venv/repositories/roles" export ANSIBLE_ACTION_PLUGINS="${HOME}/ansible_venv/repositories/roles/config_template/action" export ANSIBLE_CONNECTION_PLUGINS="${HOME}/ansible_venv/repositories/roles/plugins/connection" -export ANSIBLE_LOG_PATH="/tmp/elk-metrics-6x-logs/ansible-elk-test.log" +export ANSIBLE_LOG_PATH="/tmp/elk-metrics-7x-logs/ansible-elk-test.log" -if [[ ! -d "/tmp/elk-metrics-6x-logs" ]]; then - mkdir -pv "/tmp/elk-metrics-6x-logs" - chmod 0777 "/tmp/elk-metrics-6x-logs" +if [[ ! -d "/tmp/elk-metrics-7x-logs" ]]; then + mkdir -pv "/tmp/elk-metrics-7x-logs" + chmod 0777 "/tmp/elk-metrics-7x-logs" fi echo "To build a test environment run the following:" diff --git a/elk_metrics_7x/tests/post-run.yml b/elk_metrics_7x/tests/post-run.yml index a50c3539..d38a34cf 100644 --- a/elk_metrics_7x/tests/post-run.yml +++ b/elk_metrics_7x/tests/post-run.yml @@ -20,7 +20,7 @@ tasks: - name: Copy logs back to the executor synchronize: - src: "/tmp/elk-metrics-6x-logs" + src: "/tmp/elk-metrics-7x-logs" dest: "{{ zuul.executor.log_root }}/" mode: pull rsync_opts: diff --git a/elk_metrics_7x/tests/run-cleanup.sh b/elk_metrics_7x/tests/run-cleanup.sh index 478b4abb..5bad2ba4 100755 --- a/elk_metrics_7x/tests/run-cleanup.sh +++ b/elk_metrics_7x/tests/run-cleanup.sh @@ -18,7 +18,7 @@ set -e export TEST_DIR="$(readlink -f $(dirname ${0})/../../)" # Stop beat processes -pushd "${TEST_DIR}/elk_metrics_6x" +pushd "${TEST_DIR}/elk_metrics_7x" for i in $(ls -1 install*beat.yml); do LOWER_BEAT="$(echo "${i}" | tr '[:upper:]' '[:lower:]')" BEAT_PARTIAL="$(echo ${LOWER_BEAT} | awk -F'.' '{print $1}')" diff --git a/elk_metrics_7x/tests/run-setup.yml b/elk_metrics_7x/tests/run-setup.yml index bd5d179b..d60eb8b3 100644 --- a/elk_metrics_7x/tests/run-setup.yml +++ b/elk_metrics_7x/tests/run-setup.yml @@ -32,7 +32,7 @@ - name: Set current test repo (cross-repo) set_fact: - current_test_repo: "git.openstack.org/{{ osa_test_repo }}" + current_test_repo: "opendev.org/{{ osa_test_repo }}" when: - osa_test_repo is defined @@ -49,5 +49,5 @@ post_tasks: - name: Ensure the log directory exists file: - path: "/tmp/elk-metrics-6x-logs" + path: "/tmp/elk-metrics-7x-logs" state: directory diff --git a/elk_metrics_7x/tests/run-tests.sh b/elk_metrics_7x/tests/run-tests.sh index 559f3c09..a6a62821 100755 --- a/elk_metrics_7x/tests/run-tests.sh +++ b/elk_metrics_7x/tests/run-tests.sh @@ -26,21 +26,21 @@ pushd "${HOME}" popd popd -source "${TEST_DIR}/elk_metrics_6x/tests/manual-test.rc" +source "${TEST_DIR}/elk_metrics_7x/tests/manual-test.rc" -source "${TEST_DIR}/elk_metrics_6x/bootstrap-embedded-ansible.sh" +source "${TEST_DIR}/elk_metrics_7x/bootstrap-embedded-ansible.sh" deactivate ${HOME}/ansible_venv/bin/ansible-galaxy install --force \ --roles-path="${HOME}/ansible_venv/repositories/roles" \ - --role-file="${TEST_DIR}/elk_metrics_6x/tests/ansible-role-requirements.yml" + --role-file="${TEST_DIR}/elk_metrics_7x/tests/ansible-role-requirements.yml" -if [[ ! -e "${TEST_DIR}/elk_metrics_6x/tests/src" ]]; then - ln -s ${TEST_DIR}/../ ${TEST_DIR}/elk_metrics_6x/tests/src +if [[ ! -e "${TEST_DIR}/elk_metrics_7x/tests/src" ]]; then + ln -s ${TEST_DIR}/../ ${TEST_DIR}/elk_metrics_7x/tests/src fi ${HOME}/ansible_venv/bin/ansible-playbook -i 'localhost,' \ -vv \ -e ansible_connection=local \ -e test_clustered_elk=${CLUSTERED:-no} \ - ${TEST_DIR}/elk_metrics_6x/tests/test.yml + ${TEST_DIR}/elk_metrics_7x/tests/test.yml diff --git a/elk_metrics_7x/tests/testAPI.yml b/elk_metrics_7x/tests/testAPI.yml index 383db8bb..c90d95df 100644 --- a/elk_metrics_7x/tests/testAPI.yml +++ b/elk_metrics_7x/tests/testAPI.yml @@ -17,7 +17,7 @@ become: true environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" tasks: - name: Check for open TCP @@ -36,7 +36,7 @@ become: true environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" tasks: - name: Check http @@ -69,7 +69,7 @@ become: true environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" tasks: - name: Check http @@ -96,7 +96,7 @@ become: true environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" tasks: - name: Check http diff --git a/elk_metrics_7x/tests/testLayout.yml b/elk_metrics_7x/tests/testLayout.yml index 418a3457..eab638ce 100644 --- a/elk_metrics_7x/tests/testLayout.yml +++ b/elk_metrics_7x/tests/testLayout.yml @@ -18,7 +18,7 @@ become: true environment: - ANSIBLE_LOG_PATH: "/tmp/elk-metrics-6x-logs/ansible-elk-test.log" + ANSIBLE_LOG_PATH: "/tmp/elk-metrics-7x-logs/ansible-elk-test.log" vars: storage_node_count: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] diff --git a/elk_metrics_7x/vars/variables.yml b/elk_metrics_7x/vars/variables.yml index f751f2d7..48abc1cd 100644 --- a/elk_metrics_7x/vars/variables.yml +++ b/elk_metrics_7x/vars/variables.yml @@ -376,3 +376,38 @@ grafana_datasources: maxConcurrentShardRequests: 256 timeField: "@timestamp" timeInterval: ">60s" + +elastic_beats: + logstash: + make_index: true + hosts: "{{ groups['elastic-logstash'] | default([]) }}" + apm: + make_index: true + timeFieldName: '@timestamp' + hosts: "{{ groups['apm-server'] | default([]) }}" + auditbeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['hosts'] | default([]) }}" + filebeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['hosts'] | default([]) }}" + syslog: + make_index: true + hosts: "{{ groups['hosts'] | default([]) }}" + heartbeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['kibana'][:3] | default([]) }}" + journalbeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['hosts'] | default([]) }}" + metricbeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['all'] | default([]) }}" + packetbeat: + timeFieldName: '@timestamp' + hosts: "{{ groups['hosts'] | default([]) }}" + monitorstack: + timeFieldName: '@timestamp' + hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}" + skydive: + hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 92d2a223..9d72dba3 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -26,6 +26,20 @@ osa_test_repo: "openstack/openstack-ansible-ops" test_clustered_elk: false +- job: + name: "openstack-ansible-ops:elk_metrics_7x-ubuntu-bionic" + parent: base + nodeset: ubuntu-bionic + description: "Runs a gate test on the elk_metrics_7x project." + run: "elk_metrics_7x/tests/test.yml" + post-run: "elk_metrics_7x/tests/post-run.yml" + files: + - ^elk_metrics_7x/.* + - ^bootstrap-embedded-ansible/.* + vars: + osa_test_repo: "openstack/openstack-ansible-ops" + test_clustered_elk: false + - job: name: "openstack-ansible-ops:elk_metrics_6x-centos-7" parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial" diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index e710cf12..f89a651c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -26,6 +26,7 @@ - openstack-ansible-ops:elk_metrics_6x-ubuntu-trusty - openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial - openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic + - openstack-ansible-ops:elk_metrics_7x-ubuntu-bionic # - openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial-clustered # - openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic-clustered - openstack-ansible-ops:osquery-ubuntu-xenial