openstack/openstack-ansible-ops
Code Issues Proposed changes

Merge "Add GeoIP capabilities into logstash"

Browse Source
This commit is contained in:
Zuul 2018-06-15 22:50:49 +00:00 committed by Gerrit Code Review
commit 8ce7794be9
6 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
elk_metrics_6x/templates/02-beats-input.conf.j2
View File

@ -3,3 +3,15 @@ input {
port => {{ logstash_beat_input_port }} port => {{ logstash_beat_input_port }}
} }
} }
filter {
if [source.ip] {
geoip {
source => "source.ip"
}
} else if [ip] {
geoip {
source => "ip"
}
}
}

6
elk_metrics_6x/templates/08-apache.conf
View File

@ -6,11 +6,13 @@ filter {
"message" => [ "message" => [
"%{COMMONAPACHELOG}", "%{COMMONAPACHELOG}",
"\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}",
"%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}"
] ]
} }
} }
geoip {
source => "clientip"
}
if ![loglevel] { if ![loglevel] {
mutate { mutate {
add_field => { "logmessage" => "%{request}" } add_field => { "logmessage" => "%{request}" }

4
elk_metrics_6x/templates/10-mysql.conf
View File

@ -16,6 +16,10 @@ filter {
match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" }
} }
geoip {
source => "clientip"
}
date { date {
match => [ "timestamp", "UNIX" ] match => [ "timestamp", "UNIX" ]
} }

3
elk_metrics_6x/templates/19-nginx.conf
View File

@ -7,6 +7,9 @@ filter {
"message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}"
} }
} }
geoip {
source => "clientip"
}
} }
if "nginx-error" in [tags] { if "nginx-error" in [tags] {
grok { grok {

2
elk_metrics_6x/templates/99-elasticsearch-output.conf.j2
View File

@ -2,7 +2,7 @@ output {
elasticsearch { elasticsearch {
hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }}
sniffing => {{ (not data_node | bool) | lower }} sniffing => {{ (not data_node | bool) | lower }}
manage_template => {{ (not data_node | bool) | lower }} manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
} }
} }

11
elk_metrics_6x/templates/elasticsearch.yml.j2
View File

@ -99,9 +99,12 @@ thread_pool:
search: search:
size: {{ thread_pool_size }} size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 64 }} queue_size: {{ thread_pool_size * 64 }}
index: search:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 128 }}
bulk:
size: {{ thread_pool_size }} size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }} queue_size: {{ thread_pool_size * 256 }}
index:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
bulk:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 512 }}