From 0c573062fc25e208bfa1206146fb31b401c8b7e5 Mon Sep 17 00:00:00 2001
From: Krzysztof Klimonda <kklimonda@cloudferro.com>
Date: Tue, 28 May 2019 12:05:48 +0000
Subject: [PATCH] Make fluentd-elasticsearch configuration more robust
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Enable reconnect_on_error option so that ES plugin re-establishes
a new session to the ES cluster on errors. Also, enable buffering
to the file, so that the buffer survives container restarts.

Co-Authored-By: Michal Nasiadka <mnasiadka@gmail.com>
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1830724
Change-Id: Ia40685b9d4fc02194e03c8791ddeb3d29d7f07f6
---
 ansible/roles/common/defaults/main.yml                      | 1 +
 ansible/roles/common/templates/conf/output/00-local.conf.j2 | 6 ++++++
 ansible/roles/common/templates/conf/output/01-es.conf.j2    | 3 +++
 ansible/roles/common/templates/fluentd.json.j2              | 5 +++++
 4 files changed, 15 insertions(+)

diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index d8c6ce9b0d..4d2b46041c 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -84,6 +84,7 @@ fluentd_default_volumes:
   - "{{ node_config_directory }}/fluentd/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
+  - "fluentd_data:/var/lib/fluentd/data/"
 kolla_toolbox_extra_volumes: "{{ default_extra_volumes }}"
 cron_extra_volumes: "{{ default_extra_volumes }}"
 fluentd_extra_volumes: "{{ default_extra_volumes }}"
diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index f9753eb9d6..550749c7ec 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -27,6 +27,9 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
   </store>
 {% elif enable_monasca | bool %}
     type copy
@@ -75,6 +78,9 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
   </store>
 {% elif enable_monasca | bool %}
     type copy
diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2
index 178acf407a..38500e8e94 100644
--- a/ansible/roles/common/templates/conf/output/01-es.conf.j2
+++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2
@@ -19,5 +19,8 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/openstack.*
     </store>
 </match>
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 40d2d14fb5..9cef1f8a68 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -65,6 +65,11 @@
             "path": "/var/log/kolla/swift",
             "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
             "recurse": true
+        },
+        {
+            "path": "/var/lib/fluentd/data",
+            "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
+            "recurse": true
         }
     ]