From 14d1bda0b367d95a081e473666c6e440049e87d2 Mon Sep 17 00:00:00 2001 From: Simeon Gourlin Date: Tue, 19 Nov 2019 17:59:02 +0100 Subject: [PATCH] This add chunk_size related parameters for proxy-server and object-server. New parameters for proxy class are object_chunk_size and client_chunk_size, and for storage class: network_chunk_size, disk_chunk_size and auditor_disk_chunk_size. Change-Id: If2bbc9cea1f6e0959fd67d2e004b607a8e40beee --- manifests/proxy.pp | 12 ++++++++++++ manifests/storage/server.pp | 15 +++++++++++++++ .../notes/swift-chunk-sizes-b2d3a41363d560ac.yaml | 9 +++++++++ spec/classes/swift_proxy_spec.rb | 6 ++++++ templates/object-server.conf.erb | 9 +++++++-- 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/swift-chunk-sizes-b2d3a41363d560ac.yaml diff --git a/manifests/proxy.pp b/manifests/proxy.pp index 06aba73b..79796b0d 100644 --- a/manifests/proxy.pp +++ b/manifests/proxy.pp @@ -65,6 +65,14 @@ # (optional) If True, the proxy will log whenever it has to failover to a handoff node # Defaults to true. # +# [*object_chunk_size*] +# (optional) Chunk size to read from object servers. +# Default to 65536. +# +# [*client_chunk_size*] +# (optional) Chunk size to read from clients. +# Default to 65536. +# # [*max_containers_per_account*] # (optional) If set to a positive value, will limit container number per account. # Default to 0. @@ -158,6 +166,8 @@ class swift::proxy( $log_name = 'proxy-server', $cors_allow_origin = undef, $strict_cors_mode = true, + $object_chunk_size = 65536, + $client_chunk_size = 65536, $max_containers_per_account = 0, $max_containers_whitelist = $::os_service_default, $read_affinity = undef, @@ -235,6 +245,8 @@ class swift::proxy( 'app:proxy-server/set log_level': value => $log_level; 'app:proxy-server/set log_address': value => $log_address; 'app:proxy-server/log_handoffs': value => $log_handoffs; + 'app:proxy-server/object_chunk_size': value => $object_chunk_size; + 'app:proxy-server/client_chunk_size': value => $client_chunk_size; 'app:proxy-server/allow_account_management': value => $allow_account_management; 'app:proxy-server/account_autocreate': value => $account_autocreate; 'app:proxy-server/max_containers_per_account': value => $max_containers_per_account; diff --git a/manifests/storage/server.pp b/manifests/storage/server.pp index 68296727..65f2d9fe 100644 --- a/manifests/storage/server.pp +++ b/manifests/storage/server.pp @@ -143,6 +143,18 @@ # (optional) Prefix for data being sent to statsd. # Defaults to '' # +# [*network_chunk_size*] +# (optional) Size of chunks to read/write over the network. +# Defaults to 65536. +# +# [*disk_chunk_size*] +# (optional) Size of chunks to read/write to disk. +# Defaults to 65536. +# +# [*auditor_disk_chunk_size*] +# (optional) Object-auditor size of chunks to read/write to disk. +# Defaults to undef. +# # [*splice*] # (optional) Use splice for zero-copy object GETs. This requires Linux Kernel # version 3.0 or greater. @@ -181,6 +193,9 @@ define swift::storage::server( $log_statsd_default_sample_rate = '1.0', $log_statsd_sample_rate_factor = '1.0', $log_statsd_metric_prefix = '', + $network_chunk_size = 65536, + $disk_chunk_size = 65536, + $auditor_disk_chunk_size = undef, $splice = false, ) { diff --git a/releasenotes/notes/swift-chunk-sizes-b2d3a41363d560ac.yaml b/releasenotes/notes/swift-chunk-sizes-b2d3a41363d560ac.yaml new file mode 100644 index 00000000..883d116d --- /dev/null +++ b/releasenotes/notes/swift-chunk-sizes-b2d3a41363d560ac.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + On proxy class, added two new parameters object_chunk_size and + client_chunk_size, that can be used to configure chunk size to + read from object servers and from clients side. + On storage::server class, added three new parameters network_chunk_size, + disk_chunk_size and auditor_disk_chunk_size, that can be used to + configure chunk size to read/write over the network and to the disk. diff --git a/spec/classes/swift_proxy_spec.rb b/spec/classes/swift_proxy_spec.rb index 1f5c3a25..c5458fb9 100644 --- a/spec/classes/swift_proxy_spec.rb +++ b/spec/classes/swift_proxy_spec.rb @@ -63,6 +63,8 @@ describe 'swift::proxy' do it { should contain_swift_proxy_config('app:proxy-server/set log_level').with_value('INFO') } it { should contain_swift_proxy_config('app:proxy-server/set log_address').with_value('/dev/log') } it { should contain_swift_proxy_config('app:proxy-server/log_handoffs').with_value('true') } + it { should contain_swift_proxy_config('app:proxy-server/object_chunk_size').with_value('65536') } + it { should contain_swift_proxy_config('app:proxy-server/client_chunk_size').with_value('65536') } it { should contain_swift_proxy_config('app:proxy-server/allow_account_management').with_value('true') } it { should contain_swift_proxy_config('app:proxy-server/account_autocreate').with_value('true') } it { should contain_swift_proxy_config('app:proxy-server/max_containers_per_account').with_value(0) } @@ -106,6 +108,8 @@ describe 'swift::proxy' do :account_autocreate => false, :log_level => 'DEBUG', :log_name => 'swift-proxy-server', + :object_chunk_size => '8192', + :client_chunk_size => '8192', :max_containers_per_account => 10, :max_containers_whitelist => 'project1,project2', :read_affinity => 'r1z1=100, r1=200', @@ -136,6 +140,8 @@ describe 'swift::proxy' do it { should contain_swift_proxy_config('app:proxy-server/set log_level').with_value('DEBUG') } it { should contain_swift_proxy_config('app:proxy-server/set log_address').with_value('/dev/log') } it { should contain_swift_proxy_config('app:proxy-server/log_handoffs').with_value('true') } + it { should contain_swift_proxy_config('app:proxy-server/object_chunk_size').with_value('8192') } + it { should contain_swift_proxy_config('app:proxy-server/client_chunk_size').with_value('8192') } it { should contain_swift_proxy_config('app:proxy-server/allow_account_management').with_value('false') } it { should contain_swift_proxy_config('app:proxy-server/account_autocreate').with_value('false') } it { should contain_swift_proxy_config('app:proxy-server/max_containers_per_account').with_value(10) } diff --git a/templates/object-server.conf.erb b/templates/object-server.conf.erb index b4c2cce0..90126bb2 100644 --- a/templates/object-server.conf.erb +++ b/templates/object-server.conf.erb @@ -17,13 +17,15 @@ log_udp_host = <%= @log_udp_host -%> log_udp_port = <%= @log_udp_port -%> <% end %> workers = <%= @workers %> -<%- if @statsd_enabled -%> +<% if @statsd_enabled -%> log_statsd_host = <%=@log_statsd_host%> log_statsd_port = <%=@log_statsd_port%> log_statsd_default_sample_rate = <%=@log_statsd_default_sample_rate%> log_statsd_sample_rate_factor = <%=@log_statsd_sample_rate_factor%> log_statsd_metric_prefix = <%=@log_statsd_metric_prefix%> -<%- end -%> +<% end -%> +network_chunk_size = <%= @network_chunk_size%> +disk_chunk_size = <%= @disk_chunk_size%> [pipeline:main] pipeline = <%= @pipeline.to_a.join(' ') %> @@ -44,5 +46,8 @@ concurrency = <%= @replicator_concurrency %> concurrency = <%= @updater_concurrency %> [object-auditor] +<% if @auditor_disk_chunk_size -%> +disk_chunk_size = <%= @auditor_disk_chunk_size %> +<% end -%> [object-reconstructor]