From 6a2369f381cb95d9367550e44a3ecefe8f4c936a Mon Sep 17 00:00:00 2001 From: Dawud Date: Wed, 17 Jul 2024 20:26:53 +0100 Subject: [PATCH] Add size limits to Fluentd buffers This change adds size limits to the Fluentd buffers to prevent them from becoming too large for OpenSearch and Elasticsearch. These values have been tested and prevent the buffer becoming too big for OpenSearch. Closes-Bug: #2079988 Change-Id: I06e34ceb26bf792f24ee4030a267c94f9ee22e3b --- ansible/roles/common/defaults/main.yml | 12 ++++++++++++ .../common/templates/conf/output/00-local.conf.j2 | 4 ++++ .../roles/common/templates/conf/output/01-es.conf.j2 | 2 ++ .../templates/conf/output/03-opensearch.conf.j2 | 2 ++ .../notes/fluentd-buffers-86acb335b1cf3126.yaml | 9 +++++++++ 5 files changed, 29 insertions(+) create mode 100644 releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index 9fae431b15..084655fe52 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -139,6 +139,18 @@ cron_logrotate_schedule: "daily" # Enable the additional watch timer fluentd_enable_watch_timer: "false" +# Set limits for queue size and chunk size +# We need to ensure that the bulk_message_request_threshold is set below the +# default maximum content length for the OpenSearch bulk API (100MB). By +# default the bulk_message_request_threshold is unlimited, which can lead to +# large payloads being sent and subsequently rejected by the OpenSearch API. +fluentd_bulk_message_request_threshold: "20M" + +# The fluentd buffer chunk limit size is the maximum size of a single chunk in +# the buffer. This should be set to a value that is less than the maximum size +# of the bulk_message_request_threshold. +fluentd_buffer_chunk_limit_size: "8M" + fluentd_input_openstack_services: - name: aodh enabled: "{{ enable_aodh | bool }}" diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 16612be648..62c7965bfa 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -39,10 +39,12 @@ reconnect_on_error true request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} {% elif log_direct_to_opensearch %} @@ -70,10 +72,12 @@ reconnect_on_error true request_timeout {{ fluentd_opensearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} {% endif %} diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 index 4443ed24fc..91d011391b 100644 --- a/ansible/roles/common/templates/conf/output/01-es.conf.j2 +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -24,10 +24,12 @@ reconnect_on_error true request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/elasticsearch.buffer/openstack.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} diff --git a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 index e40b3f98cb..6f4beb2d64 100644 --- a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 +++ b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 @@ -24,10 +24,12 @@ reconnect_on_error true request_timeout {{ fluentd_opensearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/opensearch.buffer/openstack.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} diff --git a/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml new file mode 100644 index 0000000000..39bb7401e0 --- /dev/null +++ b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Reduce the size of the fluentd buffers to avoid getting HTTP 413 errors + when sending logs to opensearch/elasticsearch. The values chosen were based + on what seemed a sensible size. These can be customised by editing the + ``fluentd_bulk_message_request_threshold`` and + ``fluentd_buffer_chunk_limit_size`` variables. `LP#2079988 + `__