From 063518d0720d56a5b70ba277f1c645decb554c9a Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Mon, 24 Sep 2018 17:06:51 +0000
Subject: [PATCH] Update Monasca Log Transformer config
* Convert all log levels to lowercase
* Make number of threads configurable
* Improve naming
* Update path and log options for Logstash 2.4
* Change worker count from string to integer for the Kafka output plugin supplied
with Logstash 2.4.
Partially-Implements: blueprint monasca-roles
Change-Id: I9379969ee639dff0b525d7a1a03df59bcf18b98b
---
.../log-transformer.conf.j2 | 16 +++++++++++++++-
.../monasca-log-transformer.json.j2 | 2 +-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
index c571623317..ac88a0d525 100644
--- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
@@ -5,7 +5,9 @@ input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_raw_logs_topic }}"
- group_id => "transformer-logstash-consumer"
+ group_id => "log_transformer"
+ consumer_id => "log_transformer_{{ ansible_hostname }}"
+ consumer_threads => "{{ monasca_log_pipeline_threads }}"
}
}
@@ -15,11 +17,23 @@ filter {
match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
}
+
+ # OpenStack log levels are uppercase, and syslog are lowercase.
+ # Furthermore, syslog has more log levels that OpenStack. To avoid
+ # mapping syslog log levels to OpenStack log levels, we standardise
+ # on the syslog style here.
+ if [log][dimensions][log_level] {
+ mutate {
+ lowercase => [ "[log][dimensions][log_level]" ]
+ }
+ }
}
output {
kafka {
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
+ client_id => "log_transformer_{{ ansible_hostname }}"
+ workers => {{ monasca_log_pipeline_threads|int }}
}
}
diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
index 94cb1cf375..109326485c 100644
--- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
+++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
@@ -1,5 +1,5 @@
{
- "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+ "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-transformer.conf",