From d8a8b70547c2ec9bec98db012dc330f78568628d Mon Sep 17 00:00:00 2001 From: Javier Pena Date: Thu, 10 Sep 2015 15:51:52 +0200 Subject: [PATCH] Add support for swift-object-expirer service The swift-object-expirer service was not supported by puppet-swift. Adding support by creating a new class (swift::objectexpirer), and its associated custom type/provider. Change-Id: I498ffe525a7316c0091e4c9d8b7d9658234231f6 --- README.md | 9 ++ .../ini_setting.rb | 10 ++ .../type/swift_object_expirer_config.rb | 57 +++++++++ manifests/objectexpirer.pp | 120 ++++++++++++++++++ manifests/params.pp | 4 + spec/acceptance/basic_swift_spec.rb | 3 + spec/classes/swift_objectexpirer_spec.rb | 115 +++++++++++++++++ .../ini_setting_spec.rb | 61 +++++++++ .../type/swift_object_expirer_config_spec.rb | 43 +++++++ 9 files changed, 422 insertions(+) create mode 100644 lib/puppet/provider/swift_object_expirer_config/ini_setting.rb create mode 100644 lib/puppet/type/swift_object_expirer_config.rb create mode 100644 manifests/objectexpirer.pp create mode 100644 spec/classes/swift_objectexpirer_spec.rb create mode 100644 spec/unit/provider/swift_object_expirer_config/ini_setting_spec.rb create mode 100644 spec/unit/type/swift_object_expirer_config_spec.rb diff --git a/README.md b/README.md index b34a8c9c..ed05e67f 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,15 @@ The byte size that dd uses when it creates the file system. ####`seek` The size of the file system that will be created. Defaults to 25000. +### Class: swift::objectexpirer +Class that will set Swift object expirer, for scheduled deletion of objects. + +```puppet +class { 'swift::objectexpirer': } +``` + +It is assumed that the object expirer service will usually be installed in a proxy node. On Red Hat-based distributions, if the class is included in a non-proxy node, the openstack-swift-proxy package will need to be installed. + ### Verifying installation This modules ships with a simple Ruby script that validates whether or not your swift cluster is functional. diff --git a/lib/puppet/provider/swift_object_expirer_config/ini_setting.rb b/lib/puppet/provider/swift_object_expirer_config/ini_setting.rb new file mode 100644 index 00000000..cfc5e1fb --- /dev/null +++ b/lib/puppet/provider/swift_object_expirer_config/ini_setting.rb @@ -0,0 +1,10 @@ +Puppet::Type.type(:swift_object_expirer_config).provide( + :ini_setting, + :parent => Puppet::Type.type(:openstack_config).provider(:ini_setting) +) do + + def self.file_path + '/etc/swift/object-expirer.conf' + end + +end diff --git a/lib/puppet/type/swift_object_expirer_config.rb b/lib/puppet/type/swift_object_expirer_config.rb new file mode 100644 index 00000000..413373fb --- /dev/null +++ b/lib/puppet/type/swift_object_expirer_config.rb @@ -0,0 +1,57 @@ +Puppet::Type.newtype(:swift_object_expirer_config) do + + ensurable + + newparam(:name, :namevar => true) do + desc 'Section/setting name to manage from /etc/swift/object-expirer.conf' + newvalues(/\S+\/\S+/) + end + + newproperty(:value) do + desc 'The value of the setting to be defined.' + munge do |value| + value = value.to_s.strip + value.capitalize! if value =~ /^(true|false)$/i + value + end + + def is_to_s( currentvalue ) + if resource.secret? + return '[old secret redacted]' + else + return currentvalue + end + end + + def should_to_s( newvalue ) + if resource.secret? + return '[new secret redacted]' + else + return newvalue + end + end + end + + newparam(:secret, :boolean => true) do + desc 'Whether to hide the value from Puppet logs. Defaults to `false`.' + newvalues(:true, :false) + defaultto false + end + + newparam(:ensure_absent_val) do + desc 'A value that is specified as the value property will behave as if ensure => absent was specified' + defaultto('') + end + + # Require the package providing object-expirer to be present + if Facter['osfamily'].value == 'Debian' + autorequire(:package) do + 'swift-object-expirer' + end + elsif Facter['osfamily'].value == 'RedHat' + autorequire(:package) do + 'openstack-swift-proxy' + end + end + +end diff --git a/manifests/objectexpirer.pp b/manifests/objectexpirer.pp new file mode 100644 index 00000000..ecf0dcd5 --- /dev/null +++ b/manifests/objectexpirer.pp @@ -0,0 +1,120 @@ +# Class swift::objectexpirer +# +# == Parameters +# [*enabled*] +# (optional) Should the service be enabled. +# Defaults to true +# +# [*manage_service*] +# (optional) Whether the service should be managed by Puppet. +# Defaults to true. +# +# [*package_ensure*] +# (optional) Value of package resource parameter 'ensure'. +# Defaults to 'present'. +# +# [*pipeline*] +# (optional) The list of elements of the object expirer pipeline. +# Defaults to ['catch_errors', 'cache', 'proxy-server'] +# +# [*auto_create_account_prefix*] +# (optional) Prefix to use when automatically creating accounts. +# Defaults to '.'. +# +# [*concurrency*] +# (optional) Number of replication workers to spawn. +# Defaults to 1. +# +# [*expiring_objects_account_name*] +# (optional) Account name used for expiring objects. +# Defaults to 'expiring_objects'. +# +# [*interval*] +# (optional) Minimum time for a pass to take, in seconds. +# Defaults to 300. +# +# [*process*] +# (optional) Which part of the work defined by $processes +# will this instance take. +# Defaults to 0. +# +# [*processes*] +# (optional) How many parts to divide the work into, one part per +# process. 0 means a single process will do all work. +# Defaults to 0. +# +# [*reclaim_age*] +# (optional) Time elapsed in seconds before an object can be +# reclaimed. +# Defaults to 604800 (1 week). +# +# [*recon_cache_path*] +# (optional) Directory where stats for a few items will be stored. +# Defaults to '/var/cache/swift'. +# +# [*report_interval*] +# (optional) Report interval, in seconds. +# Defaults to 300. +# + +class swift::objectexpirer( + $manage_service = true, + $enabled = true, + $package_ensure = 'present', + $pipeline = ['catch_errors', 'cache', 'proxy-server'], + $auto_create_account_prefix = '.', + $concurrency = 1, + $expiring_objects_account_name = 'expiring_objects', + $interval = 300, + $process = 0, + $processes = 0, + $reclaim_age = 604800, + $recon_cache_path = '/var/cache/swift', + $report_interval = 300, +) { + + include ::swift::params + + Swift_config<| |> ~> Service['swift-object-expirer'] + Swift_object_expirer_config<||> ~> Service['swift-object-expirer'] + + # On Red Hat platforms, it may be defined already, + # because it is part of openstack-swift-proxy + if $::swift::params::object_expirer_package_name != $::swift::params::proxy_package_name { + package { 'swift-object-expirer': + ensure => $package_ensure, + name => $::swift::params::object_expirer_package_name, + tag => ['openstack', 'swift-package'], + } + } + + swift_object_expirer_config { + 'pipeline:main/pipeline': value => join($pipeline, ' '); + 'object-expirer/auto_create_account_prefix': value => $auto_create_account_prefix; + 'object-expirer/concurrency': value => $concurrency; + 'object-expirer/expiring_objects_account_name': value => $expiring_objects_account_name; + 'object-expirer/interval': value => $interval; + 'object-expirer/process': value => $process; + 'object-expirer/processes': value => $processes; + 'object-expirer/reclaim_age': value => $reclaim_age; + 'object-expirer/recon_cache_path': value => $recon_cache_path; + 'object-expirer/report_interval': value => $report_interval; + } + + if $manage_service { + if $enabled { + $service_ensure = 'running' + } else { + $service_ensure = 'stopped' + } + } + + service { 'swift-object-expirer': + ensure => $service_ensure, + name => $::swift::params::object_expirer_service_name, + enable => $enabled, + provider => $::swift::params::service_provider, + tag => 'swift-service', + } +} + diff --git a/manifests/params.pp b/manifests/params.pp index feaa197d..a4aa65d8 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -12,6 +12,8 @@ class swift::params { $object_auditor_service_name = 'swift-object-auditor' $object_replicator_service_name = 'swift-object-replicator' $object_updater_service_name = 'swift-object-updater' + $object_expirer_package_name = 'swift-object-expirer' + $object_expirer_service_name = 'swift-object-expirer' $container_package_name = 'swift-container' $container_service_name = 'swift-container' $container_auditor_service_name = 'swift-container-auditor' @@ -39,6 +41,8 @@ class swift::params { $object_auditor_service_name = 'openstack-swift-object-auditor' $object_replicator_service_name = 'openstack-swift-object-replicator' $object_updater_service_name = 'openstack-swift-object-updater' + $object_expirer_package_name = 'openstack-swift-proxy' + $object_expirer_service_name = 'openstack-swift-object-expirer' $container_package_name = 'openstack-swift-container' $container_service_name = 'openstack-swift-container' $container_auditor_service_name = 'openstack-swift-container-auditor' diff --git a/spec/acceptance/basic_swift_spec.rb b/spec/acceptance/basic_swift_spec.rb index 52abfe41..150ad7fd 100644 --- a/spec/acceptance/basic_swift_spec.rb +++ b/spec/acceptance/basic_swift_spec.rb @@ -116,6 +116,9 @@ describe 'basic swift' do class { '::swift::proxy::authtoken': admin_password => 'a_big_secret', } + class {'::swift::objectexpirer': + interval => 600, + } class { [ '::swift::proxy::healthcheck', '::swift::proxy::cache', '::swift::proxy::tempauth', '::swift::proxy::dlo' ]: diff --git a/spec/classes/swift_objectexpirer_spec.rb b/spec/classes/swift_objectexpirer_spec.rb new file mode 100644 index 00000000..43d7ea34 --- /dev/null +++ b/spec/classes/swift_objectexpirer_spec.rb @@ -0,0 +1,115 @@ +require 'spec_helper' + +describe 'swift::objectexpirer' do + + let :default_params do + { :manage_service => true, + :enabled => true, + :package_ensure => 'present', + :pipeline => ['catch_errors', 'cache', 'proxy-server'], + :auto_create_account_prefix => '.', + :concurrency => 1, + :expiring_objects_account_name => 'expiring_objects', + :interval => 300, + :process => 0, + :processes => 0, + :reclaim_age => 604800, + :recon_cache_path => '/var/cache/swift', + :report_interval => 300 } + end + + let :params do + {} + end + + + shared_examples_for 'swift-object-expirer' do + let (:p) { default_params.merge!(params) } + + it 'configures object-expirer.conf' do + is_expected.to contain_swift_object_expirer_config( + 'pipeline:main/pipeline').with_value(p[:pipeline].join(' ')) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/auto_create_account_prefix').with_value(p[:auto_create_account_prefix]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/concurrency').with_value(p[:concurrency]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/expiring_objects_account_name').with_value(p[:expiring_objects_account_name]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/interval').with_value(p[:interval]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/process').with_value(p[:process]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/processes').with_value(p[:processes]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/reclaim_age').with_value(p[:reclaim_age]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/recon_cache_path').with_value(p[:recon_cache_path]) + is_expected.to contain_swift_object_expirer_config( + 'object-expirer/report_interval').with_value(p[:report_interval]) + end + + it 'configures object-expirer service' do + is_expected.to contain_service('swift-object-expirer').with( + :ensure => (p[:manage_service] && p[:enabled]) ? 'running' : 'stopped', + :name => platform_params[:service_name], + :provider => platform_params[:service_provider], + :enable => p[:enabled] + ) + end + + end + + context 'on Debian platforms' do + let :facts do + { :operatingsystem => 'Ubuntu', + :osfamily => 'Debian' } + end + + let :platform_params do + { :object_expirer_package_name => 'swift-object-expirer', + :service_name => 'swift-object-expirer', + :service_provider => 'upstart' } + end + + it_configures 'swift-object-expirer' + + context 'when overridding parameters' do + before do + params.merge!( + :interval => '600', + :reclaim_age => '10000', + :concurrency => '3', + ) + end + + it_configures 'swift-object-expirer' + end + end + + context 'on RedHat platforms' do + let :facts do + { :osfamily => 'RedHat', + :operatingsystem => 'RedHat' } + end + + let :platform_params do + { :object_expirer_package_name => 'openstack-swift-proxy', + :service_name => 'openstack-swift-object-expirer', + :service_provider => nil } + end + + it_configures 'swift-object-expirer' + context 'when overridding parameters' do + before do + params.merge!( + :interval => '600', + :reclaim_age => '10000', + :concurrency => '3', + ) + end + + it_configures 'swift-object-expirer' + end + end +end diff --git a/spec/unit/provider/swift_object_expirer_config/ini_setting_spec.rb b/spec/unit/provider/swift_object_expirer_config/ini_setting_spec.rb new file mode 100644 index 00000000..730b0383 --- /dev/null +++ b/spec/unit/provider/swift_object_expirer_config/ini_setting_spec.rb @@ -0,0 +1,61 @@ +$LOAD_PATH.push( + File.join( + File.dirname(__FILE__), + '..', + '..', + '..', + 'fixtures', + 'modules', + 'inifile', + 'lib') +) + +require 'spec_helper' + +provider_class = Puppet::Type.type(:swift_object_expirer_config).provider(:ini_setting) + +describe provider_class do + + it 'should default to the default setting when no other one is specified' do + resource = Puppet::Type::Swift_object_expirer_config.new( + { + :name => 'DEFAULT/foo', + :value => 'bar' + } + ) + provider = provider_class.new(resource) + expect(provider.section).to eq('DEFAULT') + expect(provider.setting).to eq('foo') + end + + it 'should allow setting to be set explicitly' do + resource = Puppet::Type::Swift_object_expirer_config.new( + { + :name => 'dude/foo', + :value => 'bar' + } + ) + provider = provider_class.new(resource) + expect(provider.section).to eq('dude') + expect(provider.setting).to eq('foo') + end + + it 'should ensure absent when is specified as a value' do + resource = Puppet::Type::Swift_object_expirer_config.new( + {:name => 'dude/foo', :value => ''} + ) + provider = provider_class.new(resource) + provider.exists? + expect(resource[:ensure]).to eq :absent + end + + it 'should ensure absent when value matches ensure_absent_val' do + resource = Puppet::Type::Swift_object_expirer_config.new( + {:name => 'dude/foo', :value => 'foo', :ensure_absent_val => 'foo' } + ) + provider = provider_class.new(resource) + provider.exists? + expect(resource[:ensure]).to eq :absent + end + +end diff --git a/spec/unit/type/swift_object_expirer_config_spec.rb b/spec/unit/type/swift_object_expirer_config_spec.rb new file mode 100644 index 00000000..b4229775 --- /dev/null +++ b/spec/unit/type/swift_object_expirer_config_spec.rb @@ -0,0 +1,43 @@ +require 'puppet' +require 'puppet/type/swift_object_expirer_config' + +describe 'Puppet::Type.type(:swift_object_expirer_config)' do + before :each do + # If we do not remove the type, if will only be processed once on + # initialization, thus it will only read the fact on startup, no + # matter how we try to stub it. + Puppet::Type.rmtype(:swift_object_expirer_config) + Facter.fact(:osfamily).stubs(:value).returns(platform_params[:osfamily]) + @swift_object_expirer_config = Puppet::Type.type(:swift_object_expirer_config).new(:name => 'DEFAULT/foo', :value => 'bar') + end + + shared_examples_for 'swift-object-expirer' do + it 'should autorequire the package that installs the file' do + catalog = Puppet::Resource::Catalog.new + package = Puppet::Type.type(:package).new(:name => platform_params[:package_name]) + catalog.add_resource package, @swift_object_expirer_config + dependency = @swift_object_expirer_config.autorequire + expect(dependency.size).to eq(1) + expect(dependency[0].target).to eq(@swift_object_expirer_config) + expect(dependency[0].source).to eq(package) + end + end + + context 'on Debian platforms' do + let :platform_params do + { :package_name => 'swift-object-expirer', + :osfamily => 'Debian' } + end + + it_behaves_like 'swift-object-expirer' + end + + context 'on RedHat platforms' do + let :platform_params do + { :package_name => 'openstack-swift-proxy', + :osfamily => 'RedHat'} + end + + it_behaves_like 'swift-object-expirer' + end +end