diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index 9fae431b15..084655fe52 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -139,6 +139,18 @@ cron_logrotate_schedule: "daily"
# Enable the additional watch timer
fluentd_enable_watch_timer: "false"
+# Set limits for queue size and chunk size
+# We need to ensure that the bulk_message_request_threshold is set below the
+# default maximum content length for the OpenSearch bulk API (100MB). By
+# default the bulk_message_request_threshold is unlimited, which can lead to
+# large payloads being sent and subsequently rejected by the OpenSearch API.
+fluentd_bulk_message_request_threshold: "20M"
+
+# The fluentd buffer chunk limit size is the maximum size of a single chunk in
+# the buffer. This should be set to a value that is less than the maximum size
+# of the bulk_message_request_threshold.
+fluentd_buffer_chunk_limit_size: "8M"
+
fluentd_input_openstack_services:
- name: aodh
enabled: "{{ enable_aodh | bool }}"
diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index 16612be648..62c7965bfa 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -39,10 +39,12 @@
reconnect_on_error true
request_timeout {{ fluentd_elasticsearch_request_timeout }}
suppress_type_name true
+ bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }}
@type file
path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
flush_interval 15s
+ chunk_limit_size {{ fluentd_buffer_chunk_limit_size }}
{% elif log_direct_to_opensearch %}
@@ -70,10 +72,12 @@
reconnect_on_error true
request_timeout {{ fluentd_opensearch_request_timeout }}
suppress_type_name true
+ bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }}
@type file
path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.*
flush_interval 15s
+ chunk_limit_size {{ fluentd_buffer_chunk_limit_size }}
{% endif %}
diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2
index 4443ed24fc..91d011391b 100644
--- a/ansible/roles/common/templates/conf/output/01-es.conf.j2
+++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2
@@ -24,10 +24,12 @@
reconnect_on_error true
request_timeout {{ fluentd_elasticsearch_request_timeout }}
suppress_type_name true
+ bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }}
@type file
path /var/lib/fluentd/data/elasticsearch.buffer/openstack.*
flush_interval 15s
+ chunk_limit_size {{ fluentd_buffer_chunk_limit_size }}
diff --git a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2
index e40b3f98cb..6f4beb2d64 100644
--- a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2
+++ b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2
@@ -24,10 +24,12 @@
reconnect_on_error true
request_timeout {{ fluentd_opensearch_request_timeout }}
suppress_type_name true
+ bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }}
@type file
path /var/lib/fluentd/data/opensearch.buffer/openstack.*
flush_interval 15s
+ chunk_limit_size {{ fluentd_buffer_chunk_limit_size }}
diff --git a/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml
new file mode 100644
index 0000000000..39bb7401e0
--- /dev/null
+++ b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+ - |
+ Reduce the size of the fluentd buffers to avoid getting HTTP 413 errors
+ when sending logs to opensearch/elasticsearch. The values chosen were based
+ on what seemed a sensible size. These can be customised by editing the
+ ``fluentd_bulk_message_request_threshold`` and
+ ``fluentd_buffer_chunk_limit_size`` variables. `LP#2079988
+ `__