814622cc6c
The logstash and elasticsearch performance can be improved by using async index options, pulling back the refresh interval, and by not fingerprinting every document. * Async translog allows elasticsearch to using run fsync in the background instead of blocking * the refresh interval will now be 5x the number of replicas with a cap of 30. This integer is representitive of the seconds between index refresh calls which greatly lowers the load generated across the cluster. * All documents were fingerprinted before writting to the cluster. This was a costly operation as elasticsearch will do a forward lookup on all documents with a preset ID resulting in 100's, if not 1000's, of extra reads. The purpose of the fingerprint function is to limit repeading writes so to keep some of this functionality the fingerprint function is now only added to documents with messages. * G1 garbage collection is now enabled by default when the heap size is > 6GiB. Early versions of elasticsearch did not recommend this setting however its since stabalized in recent releases. * JVM options have been moved into the elasticsearch and logstash roles allowing these tasks to trigger service restarts when changes are made. Change-Id: I805129b207ad4db182ae6e59b6ec78eb3e246b54 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
364 lines
8.3 KiB
YAML
364 lines
8.3 KiB
YAML
---
|
|
# Option to define quarter memory
|
|
q_mem: "{{ (ansible_memtotal_mb | int) // 4 }}"
|
|
|
|
# Option to define half memory
|
|
h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}"
|
|
|
|
# Option to set persistent queue storage in gigabytes
|
|
q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_core | int) * 2 }}"
|
|
|
|
apm_port: 8200
|
|
elastic_port: 9200
|
|
elastic_hap_port: 9201
|
|
logstash_beat_input_port: 5044
|
|
logstash_syslog_input_port: 5140
|
|
logstash_syslog_input_mode: udp
|
|
kibana_port: 5601
|
|
kibana_nginx_port: 81
|
|
|
|
# This is the URL external services can use to communicate with the
|
|
# elasticsearch cluster.
|
|
elastic_vip_url: >-
|
|
{% if internal_lb_vip_address is defined %}
|
|
http://{{ internal_lb_vip_address ~ ':' ~ elastic_hap_port }}
|
|
{% else %}
|
|
http://{{ hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ elastic_port }}
|
|
{% endif %}
|
|
|
|
# Elasticsearch can query the itself and loadbalance requests across the cluster.
|
|
# This function is automatically enabled on non-data nodes however this setting
|
|
# can be used to override the default behaviour.
|
|
#elastic_sniffing_enabled: true
|
|
|
|
# Beat options
|
|
heartbeat_services:
|
|
- group: "{{ groups['galera_all'] | default([]) }}"
|
|
name: galera
|
|
ports:
|
|
- 9200
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['galera_all'] | default([]) }}"
|
|
name: galera
|
|
ports:
|
|
- 3306
|
|
type: tcp
|
|
- group: "{{ groups['repo_all'] | default([]) }}"
|
|
name: repo-git
|
|
ports:
|
|
- 9418
|
|
type: tcp
|
|
- group: "{{ groups['repo_all'] | default([]) }}"
|
|
name: repo-server
|
|
ports:
|
|
- 8181
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['repo_all'] | default([]) }}"
|
|
name: repo-acng
|
|
ports:
|
|
- 3142
|
|
type: http
|
|
method: HEAD
|
|
path: "/acng-report.html"
|
|
- group: "{{ groups['glance_api'] | default([]) }}"
|
|
name: glance-api
|
|
ports:
|
|
- 9292
|
|
type: http
|
|
method: HEAD
|
|
path: "/healthcheck"
|
|
- group: "{{ groups['glance_api'] | default([]) }}"
|
|
name: glance-registry
|
|
ports:
|
|
- 9191
|
|
type: http
|
|
method: HEAD
|
|
path: "/healthcheck"
|
|
- group: "{{ groups['gnocchi_all'] | default([]) }}"
|
|
name: gnocchi-api
|
|
ports:
|
|
- 8041
|
|
type: http
|
|
method: HEAD
|
|
path: "/healthcheck"
|
|
- group: "{{ groups['heat_api_cfn'] | default([]) }}"
|
|
name: heat-cfn-api
|
|
ports:
|
|
- 8000
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
check_response:
|
|
status: 300
|
|
- group: "{{ groups['heat_api'] | default([]) }}"
|
|
name: heat-api
|
|
ports:
|
|
- 8004
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
check_response:
|
|
status: 300
|
|
- group: "{{ groups['keystone_all'] | default([]) }}"
|
|
name: keystone-api
|
|
ports:
|
|
- 5000
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
check_response:
|
|
status: 300
|
|
- group: "{{ groups['neutron_server'] | default([]) }}"
|
|
name: neutron-server
|
|
ports:
|
|
- 9696
|
|
type: http
|
|
method: GET
|
|
path: "/"
|
|
- group: "{{ groups['nova_api_metadata'] | default([]) }}"
|
|
name: nova-api-metadata
|
|
ports:
|
|
- 8775
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['nova_api_os_compute'] | default([]) }}"
|
|
name: nova-api-compute
|
|
ports:
|
|
- 8774
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['nova_api_placement'] | default([]) }}"
|
|
name: nova-api-placement
|
|
ports:
|
|
- 8780
|
|
type: http
|
|
method: GET
|
|
path: "/"
|
|
- group: "{{ groups['nova_console'] | default([]) }}"
|
|
name: nova-console
|
|
ports:
|
|
- 6080
|
|
- 6082
|
|
- 6083
|
|
type: tcp
|
|
- group: "{{ groups['cinder_api'] | default([]) }}"
|
|
name: cinder-api
|
|
ports:
|
|
- 8776
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
check_response:
|
|
status: 300
|
|
- group: "{{ groups['horizon_all'] | default([]) }}"
|
|
name: horizon
|
|
ports:
|
|
- 80
|
|
- 443
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['sahara_api'] | default([]) }}"
|
|
name: sahara-api
|
|
ports:
|
|
- 8386
|
|
type: http
|
|
method: HEAD
|
|
path: "/healthcheck"
|
|
- group: "{{ groups['swift_proxy'] | default([]) }}"
|
|
name: swift-proxy
|
|
ports:
|
|
- 8080
|
|
type: http
|
|
method: HEAD
|
|
path: "/healthcheck"
|
|
- group: "{{ groups['aodh_api'] | default([]) }}"
|
|
name: aodh-api
|
|
ports:
|
|
- 8042
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['ironic_api'] | default([]) }}"
|
|
name: ironic-api
|
|
ports:
|
|
- 6385
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['rabbitmq_all'] | default([]) }}"
|
|
name: rabbitmq-management
|
|
ports:
|
|
- 15672
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['rabbitmq_all'] | default([]) }}"
|
|
name: rabbitmq-access
|
|
ports:
|
|
- 5672
|
|
- 5671
|
|
type: tcp
|
|
- group: "{{ groups['magnum_all'] | default([]) }}"
|
|
name: magnum-api
|
|
ports:
|
|
- 9511
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['trove_api'] | default([]) }}"
|
|
name: trove-api
|
|
ports:
|
|
- 8779
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['barbican_api'] | default([]) }}"
|
|
name: barbican-api
|
|
ports:
|
|
- 9311
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['designate_api'] | default([]) }}"
|
|
name: designate-api
|
|
ports:
|
|
- 9001
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['octavia_all'] | default([]) }}"
|
|
name: octavia-api
|
|
ports:
|
|
- 9876
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['tacker_all'] | default([]) }}"
|
|
name: tacker-api
|
|
ports:
|
|
- 9890
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
- group: "{{ groups['neutron_server'] | default([]) }}"
|
|
name: opendaylight
|
|
ports:
|
|
- 8180
|
|
- 8185
|
|
type: tcp
|
|
- group: "{{ groups['neutron_server'] | default([]) }}"
|
|
name: ceph-rgw
|
|
ports:
|
|
- 7980
|
|
type: http
|
|
method: HEAD
|
|
path: "/"
|
|
|
|
# Grafana
|
|
grafana_dashboards:
|
|
- dashboard_id: 5566
|
|
revision_id: 5
|
|
datasource: "metricbeat-Elasticsearch"
|
|
- dashboard_id: 5569
|
|
revision_id: 3
|
|
datasource: "filebeat-Elasticsearch"
|
|
|
|
grafana_datasources:
|
|
- name: "all-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: true
|
|
database: "*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|
|
- name: "auditbeat-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: false
|
|
database: "auditbeat-*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|
|
- name: "filebeat-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: false
|
|
database: "filebeat-*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|
|
- name: "heartbeat-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: false
|
|
database: "heartbeat-*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|
|
- name: "metricbeat-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: false
|
|
database: "metricbeat-*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|
|
- name: "packetbeat-Elasticsearch"
|
|
type: "elasticsearch"
|
|
access: "proxy"
|
|
url: "{{ elastic_vip_url }}"
|
|
basicAuth: false
|
|
basicAuthUser: ""
|
|
basicAuthPassword: ""
|
|
isDefault: false
|
|
database: "packetbeat-*"
|
|
jsonData:
|
|
esVersion: 56
|
|
keepCookies: []
|
|
maxConcurrentShardRequests: 256
|
|
timeField: "@timestamp"
|
|
timeInterval: ">10s"
|