Make fluentd-elasticsearch configuration more robust

Enable reconnect_on_error option so that ES plugin re-establishes
a new session to the ES cluster on errors. Also, enable buffering
to the file, so that the buffer survives container restarts.

Co-Authored-By: Michal Nasiadka <mnasiadka@gmail.com>
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1830724
Change-Id: Ia40685b9d4fc02194e03c8791ddeb3d29d7f07f6

ansible/roles/common/defaults/main.yml

@@ -84,6 +84,7 @@ fluentd_default_volumes:
   - "{{ node_config_directory }}/fluentd/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
+  - "fluentd_data:/var/lib/fluentd/data/"
 kolla_toolbox_extra_volumes: "{{ default_extra_volumes }}"
 cron_extra_volumes: "{{ default_extra_volumes }}"
 fluentd_extra_volumes: "{{ default_extra_volumes }}"

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -27,6 +27,9 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
   </store>
 {% elif enable_monasca | bool %}
     type copy
@@ -75,6 +78,9 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
   </store>
 {% elif enable_monasca | bool %}
     type copy

ansible/roles/common/templates/conf/output/01-es.conf.j2

@@ -19,5 +19,8 @@
        logstash_format true
        logstash_prefix {{ kibana_log_prefix }}
        flush_interval 15s
+       reconnect_on_error true
+       buffer_type file
+       buffer_path /var/lib/fluentd/data/elasticsearch.buffer/openstack.*
     </store>
 </match>

ansible/roles/common/templates/fluentd.json.j2

@@ -65,6 +65,11 @@
             "path": "/var/log/kolla/swift",
             "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
             "recurse": true
+        },
+        {
+            "path": "/var/lib/fluentd/data",
+            "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
+            "recurse": true
         }
     ]