Add support for swift-object-expirer service

The swift-object-expirer service was not supported by
puppet-swift. Adding support by creating a new class
(swift::objectexpirer), and its associated custom type/provider.

Change-Id: I498ffe525a7316c0091e4c9d8b7d9658234231f6

README.md

@@ -236,6 +236,15 @@ The byte size that dd uses when it creates the file system.
 ####`seek`
 The size of the file system that will be created.  Defaults to 25000.
 
+### Class: swift::objectexpirer
+Class that will set Swift object expirer, for scheduled deletion of objects.
+
+```puppet
+class { 'swift::objectexpirer': }
+```
+
+It is assumed that the object expirer service will usually be installed in a proxy node. On Red Hat-based distributions, if the class is included in a non-proxy node, the openstack-swift-proxy package will need to be installed.
+
 ### Verifying installation
 
 This modules ships with a simple Ruby script that validates whether or not your swift cluster is functional.

lib/puppet/provider/swift_object_expirer_config/ini_setting.rb

@@ -0,0 +1,10 @@
+Puppet::Type.type(:swift_object_expirer_config).provide(
+  :ini_setting,
+  :parent => Puppet::Type.type(:openstack_config).provider(:ini_setting)
+) do
+
+  def self.file_path
+    '/etc/swift/object-expirer.conf'
+  end
+
+end

lib/puppet/type/swift_object_expirer_config.rb

@@ -0,0 +1,57 @@
+Puppet::Type.newtype(:swift_object_expirer_config) do
+
+  ensurable
+
+  newparam(:name, :namevar => true) do
+    desc 'Section/setting name to manage from /etc/swift/object-expirer.conf'
+    newvalues(/\S+\/\S+/)
+  end
+
+  newproperty(:value) do
+    desc 'The value of the setting to be defined.'
+    munge do |value|
+      value = value.to_s.strip
+      value.capitalize! if value =~ /^(true|false)$/i
+      value
+    end
+
+    def is_to_s( currentvalue )
+      if resource.secret?
+        return '[old secret redacted]'
+      else
+        return currentvalue
+      end
+    end
+
+    def should_to_s( newvalue )
+      if resource.secret?
+        return '[new secret redacted]'
+      else
+        return newvalue
+      end
+    end
+  end
+
+  newparam(:secret, :boolean => true) do
+    desc 'Whether to hide the value from Puppet logs. Defaults to `false`.'
+    newvalues(:true, :false)
+    defaultto false
+  end
+
+  newparam(:ensure_absent_val) do
+    desc 'A value that is specified as the value property will behave as if ensure => absent was specified'
+    defaultto('<SERVICE DEFAULT>')
+  end
+
+  # Require the package providing object-expirer to be present
+  if Facter['osfamily'].value == 'Debian'
+    autorequire(:package) do
+      'swift-object-expirer'
+    end
+  elsif Facter['osfamily'].value == 'RedHat'
+    autorequire(:package) do
+      'openstack-swift-proxy'
+    end
+  end
+
+end

manifests/objectexpirer.pp

@@ -0,0 +1,120 @@
+# Class swift::objectexpirer
+#
+# == Parameters
+#  [*enabled*]
+#    (optional) Should the service be enabled.
+#    Defaults to true
+#
+#  [*manage_service*]
+#    (optional) Whether the service should be managed by Puppet.
+#    Defaults to true.
+#
+#  [*package_ensure*]
+#    (optional) Value of package resource parameter 'ensure'.
+#    Defaults to 'present'.
+#
+#  [*pipeline*]
+#    (optional) The list of elements of the object expirer pipeline.
+#    Defaults to ['catch_errors', 'cache', 'proxy-server']
+#
+#  [*auto_create_account_prefix*]
+#    (optional) Prefix to use when automatically creating accounts.
+#    Defaults to '.'.
+#
+#  [*concurrency*]
+#    (optional) Number of replication workers to spawn.
+#    Defaults to 1.
+#
+#  [*expiring_objects_account_name*]
+#    (optional) Account name used for expiring objects.
+#    Defaults to 'expiring_objects'.
+#
+#  [*interval*]
+#    (optional) Minimum time for a pass to take, in seconds.
+#    Defaults to 300.
+#
+#  [*process*]
+#    (optional) Which part of the work defined by $processes
+#    will this instance take.
+#    Defaults to 0.
+#
+#  [*processes*]
+#    (optional) How many parts to divide the work into, one part per
+#    process. 0 means a single process will do all work.
+#    Defaults to 0.
+#
+#  [*reclaim_age*]
+#    (optional) Time elapsed in seconds before an object can be
+#    reclaimed.
+#    Defaults to 604800 (1 week).
+#
+#  [*recon_cache_path*]
+#    (optional) Directory where stats for a few items will be stored.
+#    Defaults to '/var/cache/swift'.
+#
+#  [*report_interval*]
+#    (optional) Report interval, in seconds.
+#    Defaults to 300.
+#
+
+class swift::objectexpirer(
+  $manage_service                = true,
+  $enabled                       = true,
+  $package_ensure                = 'present',
+  $pipeline                      = ['catch_errors', 'cache', 'proxy-server'],
+  $auto_create_account_prefix    = '.',
+  $concurrency                   = 1,
+  $expiring_objects_account_name = 'expiring_objects',
+  $interval                      = 300,
+  $process                       = 0,
+  $processes                     = 0,
+  $reclaim_age                   = 604800,
+  $recon_cache_path              = '/var/cache/swift',
+  $report_interval               = 300,
+) {
+
+  include ::swift::params
+
+  Swift_config<| |> ~> Service['swift-object-expirer']
+  Swift_object_expirer_config<||> ~> Service['swift-object-expirer']
+
+  # On Red Hat platforms, it may be defined already,
+  # because it is part of openstack-swift-proxy
+  if $::swift::params::object_expirer_package_name != $::swift::params::proxy_package_name {
+    package { 'swift-object-expirer':
+      ensure => $package_ensure,
+      name   => $::swift::params::object_expirer_package_name,
+      tag    => ['openstack', 'swift-package'],
+    }
+  }
+
+  swift_object_expirer_config {
+    'pipeline:main/pipeline':                       value => join($pipeline, ' ');
+    'object-expirer/auto_create_account_prefix':    value => $auto_create_account_prefix;
+    'object-expirer/concurrency':                   value => $concurrency;
+    'object-expirer/expiring_objects_account_name': value => $expiring_objects_account_name;
+    'object-expirer/interval':                      value => $interval;
+    'object-expirer/process':                       value => $process;
+    'object-expirer/processes':                     value => $processes;
+    'object-expirer/reclaim_age':                   value => $reclaim_age;
+    'object-expirer/recon_cache_path':              value => $recon_cache_path;
+    'object-expirer/report_interval':               value => $report_interval;
+  }
+
+  if $manage_service {
+    if $enabled {
+      $service_ensure = 'running'
+    } else {
+      $service_ensure = 'stopped'
+    }
+  }
+
+  service { 'swift-object-expirer':
+    ensure   => $service_ensure,
+    name     => $::swift::params::object_expirer_service_name,
+    enable   => $enabled,
+    provider => $::swift::params::service_provider,
+    tag      => 'swift-service',
+  }
+}
+

manifests/params.pp

@@ -12,6 +12,8 @@ class swift::params {
       $object_auditor_service_name       = 'swift-object-auditor'
       $object_replicator_service_name    = 'swift-object-replicator'
       $object_updater_service_name       = 'swift-object-updater'
+      $object_expirer_package_name       = 'swift-object-expirer'
+      $object_expirer_service_name       = 'swift-object-expirer'
       $container_package_name            = 'swift-container'
       $container_service_name            = 'swift-container'
       $container_auditor_service_name    = 'swift-container-auditor'
@@ -39,6 +41,8 @@ class swift::params {
       $object_auditor_service_name       = 'openstack-swift-object-auditor'
       $object_replicator_service_name    = 'openstack-swift-object-replicator'
       $object_updater_service_name       = 'openstack-swift-object-updater'
+      $object_expirer_package_name       = 'openstack-swift-proxy'
+      $object_expirer_service_name       = 'openstack-swift-object-expirer'
       $container_package_name            = 'openstack-swift-container'
       $container_service_name            = 'openstack-swift-container'
       $container_auditor_service_name    = 'openstack-swift-container-auditor'

spec/acceptance/basic_swift_spec.rb

@@ -116,6 +116,9 @@ describe 'basic swift' do
       class { '::swift::proxy::authtoken':
         admin_password => 'a_big_secret',
       }
+      class {'::swift::objectexpirer':
+        interval => 600,
+      }
       class {
         [ '::swift::proxy::healthcheck', '::swift::proxy::cache',
         '::swift::proxy::tempauth', '::swift::proxy::dlo' ]:

spec/classes/swift_objectexpirer_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+describe 'swift::objectexpirer' do
+
+  let :default_params do
+    { :manage_service                => true,
+      :enabled                       => true,
+      :package_ensure                => 'present',
+      :pipeline                      => ['catch_errors', 'cache', 'proxy-server'],
+      :auto_create_account_prefix    => '.',
+      :concurrency                   => 1,
+      :expiring_objects_account_name => 'expiring_objects',
+      :interval                      => 300,
+      :process                       => 0,
+      :processes                     => 0,
+      :reclaim_age                   => 604800,
+      :recon_cache_path              => '/var/cache/swift',
+      :report_interval               => 300 }
+  end
+
+  let :params do
+    {}
+  end
+
+
+  shared_examples_for 'swift-object-expirer' do
+    let (:p) { default_params.merge!(params) }
+
+    it 'configures object-expirer.conf' do
+      is_expected.to contain_swift_object_expirer_config(
+        'pipeline:main/pipeline').with_value(p[:pipeline].join(' '))
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/auto_create_account_prefix').with_value(p[:auto_create_account_prefix])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/concurrency').with_value(p[:concurrency])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/expiring_objects_account_name').with_value(p[:expiring_objects_account_name])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/interval').with_value(p[:interval])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/process').with_value(p[:process])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/processes').with_value(p[:processes])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/reclaim_age').with_value(p[:reclaim_age])
+      is_expected.to contain_swift_object_expirer_config(
+       'object-expirer/recon_cache_path').with_value(p[:recon_cache_path])
+      is_expected.to contain_swift_object_expirer_config(
+        'object-expirer/report_interval').with_value(p[:report_interval])
+    end
+
+    it 'configures object-expirer service' do
+      is_expected.to contain_service('swift-object-expirer').with(
+        :ensure    => (p[:manage_service] && p[:enabled]) ? 'running' : 'stopped',
+        :name      => platform_params[:service_name],
+        :provider  => platform_params[:service_provider],
+        :enable    => p[:enabled]
+      )
+    end
+
+  end
+
+  context 'on Debian platforms' do
+    let :facts do
+      { :operatingsystem  => 'Ubuntu',
+        :osfamily         => 'Debian' }
+    end
+
+    let :platform_params do
+      { :object_expirer_package_name => 'swift-object-expirer',
+        :service_name                => 'swift-object-expirer',
+        :service_provider            => 'upstart' }
+    end
+
+    it_configures 'swift-object-expirer'
+
+    context 'when overridding parameters' do
+      before do
+        params.merge!(
+          :interval        => '600',
+          :reclaim_age     => '10000',
+          :concurrency     => '3',
+        )
+      end
+
+      it_configures 'swift-object-expirer'
+    end
+  end
+
+  context 'on RedHat platforms' do
+        let :facts do
+          { :osfamily        => 'RedHat',
+            :operatingsystem => 'RedHat' }
+        end
+
+        let :platform_params do
+          { :object_expirer_package_name => 'openstack-swift-proxy',
+            :service_name                => 'openstack-swift-object-expirer',
+            :service_provider            => nil }
+        end
+
+    it_configures 'swift-object-expirer'
+    context 'when overridding parameters' do
+      before do
+        params.merge!(
+          :interval        => '600',
+          :reclaim_age     => '10000',
+          :concurrency     => '3',
+        )
+      end
+
+      it_configures 'swift-object-expirer'
+    end
+  end
+end

spec/unit/provider/swift_object_expirer_config/ini_setting_spec.rb

@@ -0,0 +1,61 @@
+$LOAD_PATH.push(
+  File.join(
+    File.dirname(__FILE__),
+    '..',
+    '..',
+    '..',
+    'fixtures',
+    'modules',
+    'inifile',
+    'lib')
+)
+
+require 'spec_helper'
+
+provider_class = Puppet::Type.type(:swift_object_expirer_config).provider(:ini_setting)
+
+describe provider_class do
+
+  it 'should default to the default setting when no other one is specified' do
+    resource = Puppet::Type::Swift_object_expirer_config.new(
+      {
+        :name => 'DEFAULT/foo',
+        :value => 'bar'
+      }
+    )
+    provider = provider_class.new(resource)
+    expect(provider.section).to eq('DEFAULT')
+    expect(provider.setting).to eq('foo')
+  end
+
+  it 'should allow setting to be set explicitly' do
+    resource = Puppet::Type::Swift_object_expirer_config.new(
+      {
+        :name => 'dude/foo',
+        :value => 'bar'
+      }
+    )
+    provider = provider_class.new(resource)
+    expect(provider.section).to eq('dude')
+    expect(provider.setting).to eq('foo')
+  end
+
+  it 'should ensure absent when <SERVICE DEFAULT> is specified as a value' do
+    resource = Puppet::Type::Swift_object_expirer_config.new(
+      {:name => 'dude/foo', :value => '<SERVICE DEFAULT>'}
+    )
+    provider = provider_class.new(resource)
+    provider.exists?
+    expect(resource[:ensure]).to eq :absent
+  end
+
+  it 'should ensure absent when value matches ensure_absent_val' do
+    resource = Puppet::Type::Swift_object_expirer_config.new(
+      {:name => 'dude/foo', :value => 'foo', :ensure_absent_val => 'foo' }
+    )
+    provider = provider_class.new(resource)
+    provider.exists?
+    expect(resource[:ensure]).to eq :absent
+  end
+
+end

spec/unit/type/swift_object_expirer_config_spec.rb

@@ -0,0 +1,43 @@
+require 'puppet'
+require 'puppet/type/swift_object_expirer_config'
+
+describe 'Puppet::Type.type(:swift_object_expirer_config)' do
+  before :each do
+    # If we do not remove the type, if will only be processed once on
+    # initialization, thus it will only read the fact on startup, no
+    # matter how we try to stub it.
+    Puppet::Type.rmtype(:swift_object_expirer_config)
+    Facter.fact(:osfamily).stubs(:value).returns(platform_params[:osfamily])
+    @swift_object_expirer_config = Puppet::Type.type(:swift_object_expirer_config).new(:name => 'DEFAULT/foo', :value => 'bar')
+  end
+
+  shared_examples_for 'swift-object-expirer' do
+    it 'should autorequire the package that installs the file' do
+      catalog = Puppet::Resource::Catalog.new
+      package = Puppet::Type.type(:package).new(:name => platform_params[:package_name])
+      catalog.add_resource package, @swift_object_expirer_config
+      dependency = @swift_object_expirer_config.autorequire
+      expect(dependency.size).to eq(1)
+      expect(dependency[0].target).to eq(@swift_object_expirer_config)
+      expect(dependency[0].source).to eq(package)
+    end
+  end
+
+  context 'on Debian platforms' do
+    let :platform_params do
+      { :package_name => 'swift-object-expirer',
+        :osfamily     => 'Debian' }
+    end
+
+    it_behaves_like 'swift-object-expirer'
+  end
+
+  context 'on RedHat platforms' do
+    let :platform_params do
+      { :package_name => 'openstack-swift-proxy',
+        :osfamily     => 'RedHat'}
+    end
+
+    it_behaves_like 'swift-object-expirer'
+  end
+end