openstack/openstack-ansible-ops
Code Issues Proposed changes

Add GeoIP capabilities into logstash

Browse Source 
The geoip capabilities are built into logstash and can help a deployer
better understand the workloads they're running within their
environments. This change adds geoip capabilities in line with the
recommended practice from elastic.

Link: https://www.elastic.co/blog/geoip-in-the-elastic-stack

Change-Id: I109879bd9aeeaa0312eb3303e2e7566181ecff91
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-06-15 09:27:08 -05:00
parent 977c2b9c58
commit 5537e8b378
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
6 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
elk_metrics_6x/templates/02-beats-input.conf.j2
View File

@ -3,3 +3,15 @@ input {
port => {{ logstash_beat_input_port }}
}
}
filter {
if [source.ip] {
geoip {
source => "source.ip"
}
} else if [ip] {
geoip {
source => "ip"
}
}
}

6
elk_metrics_6x/templates/08-apache.conf
View File

@ -6,11 +6,13 @@ filter {
"message" => [
"%{COMMONAPACHELOG}",
"\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}",
"%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}"
"%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}"
]
}
}
geoip {
source => "clientip"
}
if ![loglevel] {
mutate {
add_field => { "logmessage" => "%{request}" }

4
elk_metrics_6x/templates/10-mysql.conf
View File

@ -16,6 +16,10 @@ filter {
match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" }
}
geoip {
source => "clientip"
}
date {
match => [ "timestamp", "UNIX" ]
}

3
elk_metrics_6x/templates/19-nginx.conf
View File

@ -7,6 +7,9 @@ filter {
"message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}"
}
}
geoip {
source => "clientip"
}
}
if "nginx-error" in [tags] {
grok {

2
elk_metrics_6x/templates/99-elasticsearch-output.conf.j2
View File

@ -2,7 +2,7 @@ output {
elasticsearch {
hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }}
sniffing => {{ (not data_node | bool) | lower }}
manage_template => {{ (not data_node | bool) | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}

11
elk_metrics_6x/templates/elasticsearch.yml.j2
View File

@ -99,9 +99,12 @@ thread_pool:
search:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 64 }}
index:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 128 }}
bulk:
search:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
index:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
bulk:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 512 }}