From 5537e8b3784b318c55c797c786413b3a5a847cb1 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 15 Jun 2018 09:27:08 -0500 Subject: [PATCH] Add GeoIP capabilities into logstash The geoip capabilities are built into logstash and can help a deployer better understand the workloads they're running within their environments. This change adds geoip capabilities in line with the recommended practice from elastic. Link: https://www.elastic.co/blog/geoip-in-the-elastic-stack Change-Id: I109879bd9aeeaa0312eb3303e2e7566181ecff91 Signed-off-by: Kevin Carter --- elk_metrics_6x/templates/02-beats-input.conf.j2 | 12 ++++++++++++ elk_metrics_6x/templates/08-apache.conf | 6 ++++-- elk_metrics_6x/templates/10-mysql.conf | 4 ++++ elk_metrics_6x/templates/19-nginx.conf | 3 +++ .../templates/99-elasticsearch-output.conf.j2 | 2 +- elk_metrics_6x/templates/elasticsearch.yml.j2 | 11 +++++++---- 6 files changed, 31 insertions(+), 7 deletions(-) diff --git a/elk_metrics_6x/templates/02-beats-input.conf.j2 b/elk_metrics_6x/templates/02-beats-input.conf.j2 index 567d509c..6c979811 100644 --- a/elk_metrics_6x/templates/02-beats-input.conf.j2 +++ b/elk_metrics_6x/templates/02-beats-input.conf.j2 @@ -3,3 +3,15 @@ input { port => {{ logstash_beat_input_port }} } } + +filter { + if [source.ip] { + geoip { + source => "source.ip" + } + } else if [ip] { + geoip { + source => "ip" + } + } +} diff --git a/elk_metrics_6x/templates/08-apache.conf b/elk_metrics_6x/templates/08-apache.conf index ba1ad58f..59b0de89 100644 --- a/elk_metrics_6x/templates/08-apache.conf +++ b/elk_metrics_6x/templates/08-apache.conf @@ -6,11 +6,13 @@ filter { "message" => [ "%{COMMONAPACHELOG}", "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", - "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" + "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" ] } } - + geoip { + source => "clientip" + } if ![loglevel] { mutate { add_field => { "logmessage" => "%{request}" } diff --git a/elk_metrics_6x/templates/10-mysql.conf b/elk_metrics_6x/templates/10-mysql.conf index 861b240b..c977c005 100644 --- a/elk_metrics_6x/templates/10-mysql.conf +++ b/elk_metrics_6x/templates/10-mysql.conf @@ -16,6 +16,10 @@ filter { match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } } + geoip { + source => "clientip" + } + date { match => [ "timestamp", "UNIX" ] } diff --git a/elk_metrics_6x/templates/19-nginx.conf b/elk_metrics_6x/templates/19-nginx.conf index 0c3f5ba9..8a68484c 100644 --- a/elk_metrics_6x/templates/19-nginx.conf +++ b/elk_metrics_6x/templates/19-nginx.conf @@ -7,6 +7,9 @@ filter { "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" } } + geoip { + source => "clientip" + } } if "nginx-error" in [tags] { grok { diff --git a/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 b/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 index 139cdc87..dd6175ae 100644 --- a/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 +++ b/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 @@ -2,7 +2,7 @@ output { elasticsearch { hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }} sniffing => {{ (not data_node | bool) | lower }} - manage_template => {{ (not data_node | bool) | lower }} + manage_template => {{ (data_node | bool) | lower }} index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" } } diff --git a/elk_metrics_6x/templates/elasticsearch.yml.j2 b/elk_metrics_6x/templates/elasticsearch.yml.j2 index 2e2700d9..a22e74d9 100644 --- a/elk_metrics_6x/templates/elasticsearch.yml.j2 +++ b/elk_metrics_6x/templates/elasticsearch.yml.j2 @@ -99,9 +99,12 @@ thread_pool: search: size: {{ thread_pool_size }} queue_size: {{ thread_pool_size * 64 }} - index: - size: {{ thread_pool_size }} - queue_size: {{ thread_pool_size * 128 }} - bulk: + search: size: {{ thread_pool_size }} queue_size: {{ thread_pool_size * 256 }} + index: + size: {{ thread_pool_size }} + queue_size: {{ thread_pool_size * 256 }} + bulk: + size: {{ thread_pool_size }} + queue_size: {{ thread_pool_size * 512 }}