diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index 9fae431b15..084655fe52 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -139,6 +139,18 @@ cron_logrotate_schedule: "daily" # Enable the additional watch timer fluentd_enable_watch_timer: "false" +# Set limits for queue size and chunk size +# We need to ensure that the bulk_message_request_threshold is set below the +# default maximum content length for the OpenSearch bulk API (100MB). By +# default the bulk_message_request_threshold is unlimited, which can lead to +# large payloads being sent and subsequently rejected by the OpenSearch API. +fluentd_bulk_message_request_threshold: "20M" + +# The fluentd buffer chunk limit size is the maximum size of a single chunk in +# the buffer. This should be set to a value that is less than the maximum size +# of the bulk_message_request_threshold. +fluentd_buffer_chunk_limit_size: "8M" + fluentd_input_openstack_services: - name: aodh enabled: "{{ enable_aodh | bool }}" diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 16612be648..62c7965bfa 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -39,10 +39,12 @@ reconnect_on_error true request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} {% elif log_direct_to_opensearch %} @@ -70,10 +72,12 @@ reconnect_on_error true request_timeout {{ fluentd_opensearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} {% endif %} diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2 index 4443ed24fc..91d011391b 100644 --- a/ansible/roles/common/templates/conf/output/01-es.conf.j2 +++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2 @@ -24,10 +24,12 @@ reconnect_on_error true request_timeout {{ fluentd_elasticsearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/elasticsearch.buffer/openstack.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} diff --git a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 index e40b3f98cb..6f4beb2d64 100644 --- a/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 +++ b/ansible/roles/common/templates/conf/output/03-opensearch.conf.j2 @@ -24,10 +24,12 @@ reconnect_on_error true request_timeout {{ fluentd_opensearch_request_timeout }} suppress_type_name true + bulk_message_request_threshold {{ fluentd_bulk_message_request_threshold }} @type file path /var/lib/fluentd/data/opensearch.buffer/openstack.* flush_interval 15s + chunk_limit_size {{ fluentd_buffer_chunk_limit_size }} diff --git a/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml new file mode 100644 index 0000000000..39bb7401e0 --- /dev/null +++ b/releasenotes/notes/fluentd-buffers-86acb335b1cf3126.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Reduce the size of the fluentd buffers to avoid getting HTTP 413 errors + when sending logs to opensearch/elasticsearch. The values chosen were based + on what seemed a sensible size. These can be customised by editing the + ``fluentd_bulk_message_request_threshold`` and + ``fluentd_buffer_chunk_limit_size`` variables. `LP#2079988 + `__