openstack/openstack-ansible-ops
Code Issues Proposed changes

Add GeoIP capabilities into logstash

Browse Source 
The geoip capabilities are built into logstash and can help a deployer
better understand the workloads they're running within their
environments. This change adds geoip capabilities in line with the
recommended practice from elastic.

Link: https://www.elastic.co/blog/geoip-in-the-elastic-stack

Change-Id: I109879bd9aeeaa0312eb3303e2e7566181ecff91
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-06-15 09:27:08 -05:00
parent 977c2b9c58
commit 5537e8b378
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
6 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
elk_metrics_6x/templates/02-beats-input.conf.j2
View File

@ -3,3 +3,15 @@ input {
port => {{ logstash_beat_input_port }} port => {{ logstash_beat_input_port }}
} }
} }
filter {
if [source.ip] {
geoip {
source => "source.ip"
}
} else if [ip] {
geoip {
source => "ip"
}
}
}

4
elk_metrics_6x/templates/08-apache.conf
View File

@ -10,7 +10,9 @@ filter {
] ]
} }
} }
geoip {
source => "clientip"
}
if ![loglevel] { if ![loglevel] {
mutate { mutate {
add_field => { "logmessage" => "%{request}" } add_field => { "logmessage" => "%{request}" }

4
elk_metrics_6x/templates/10-mysql.conf
View File

@ -16,6 +16,10 @@ filter {
match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" }
} }
geoip {
source => "clientip"
}
date { date {
match => [ "timestamp", "UNIX" ] match => [ "timestamp", "UNIX" ]
} }

3
elk_metrics_6x/templates/19-nginx.conf
View File

@ -7,6 +7,9 @@ filter {
"message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}"
} }
} }
geoip {
source => "clientip"
}
} }
if "nginx-error" in [tags] { if "nginx-error" in [tags] {
grok { grok {

2
elk_metrics_6x/templates/99-elasticsearch-output.conf.j2
View File

@ -2,7 +2,7 @@ output {
elasticsearch { elasticsearch {
hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }}
sniffing => {{ (not data_node | bool) | lower }} sniffing => {{ (not data_node | bool) | lower }}
manage_template => {{ (not data_node | bool) | lower }} manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
} }
} }

11
elk_metrics_6x/templates/elasticsearch.yml.j2
View File

@ -99,9 +99,12 @@ thread_pool:
search: search:
size: {{ thread_pool_size }} size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 64 }} queue_size: {{ thread_pool_size * 64 }}
index: search:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 128 }}
bulk:
size: {{ thread_pool_size }} size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }} queue_size: {{ thread_pool_size * 256 }}
index:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
bulk:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 512 }}